Merge pull request #453 from officialasishkumar/sparse-checkout-kcl-m…

…od-add

Sparse checkout - kcl mod add --package

pkg/client/client.go

@@ -283,17 +283,17 @@ const PKG_NAME_PATTERN = "%s_%s"
 // 2. in the vendor subdirectory of the current package.
 // 3. the dependency is from the local path.
 func (c *KpmClient) getDepStorePath(search_path string, d *pkg.Dependency, isVendor bool) string {
-
 	storePkgName := d.GenPathSuffix()
-
 	if d.IsFromLocal() {
 		return d.GetLocalFullPath(search_path)
 	} else {
+		path := ""
 		if isVendor {
-			return filepath.Join(search_path, "vendor", storePkgName)
+			path = filepath.Join(search_path, "vendor", storePkgName)
 		} else {
-			return filepath.Join(c.homePath, storePkgName)
+			path = filepath.Join(c.homePath, storePkgName)
 		}
+		return path
 	}
 }
 
@@ -747,7 +747,7 @@ func (c *KpmClient) AddDepWithOpts(kclPkg *pkg.KclPkg, opt *opt.AddOptions) (*pk
 	c.noSumCheck = opt.NoSumCheck
 	kclPkg.NoSumCheck = opt.NoSumCheck
 
-	// 1. get the name and version of the repository from the input arguments.
+	// 1. get the name and version of the repository/package from the input arguments.
 	d, err := pkg.ParseOpt(&opt.RegistryOpts)
 	if err != nil {
 		return nil, err
@@ -1101,7 +1101,16 @@ func (c *KpmClient) Download(dep *pkg.Dependency, homePath, localPath string) (*
 			return nil, err
 		}
 
-		dep.LocalFullPath = localPath
+		if dep.GetPackage() != "" {
+			localFullPath, err := utils.FindPackage(localPath, dep.GetPackage())
+			if err != nil {
+				return nil, err
+			}
+			dep.LocalFullPath = localFullPath
+			dep.Name = dep.GetPackage()
+		} else {
+			dep.LocalFullPath = localPath
+		}
 		// Creating symbolic links in a global cache is not an optimal solution.
 		// This allows kclvm to locate the package by default.
 		// This feature is unstable and will be removed soon.
@@ -1111,7 +1120,7 @@ func (c *KpmClient) Download(dep *pkg.Dependency, homePath, localPath string) (*
 		// }
 		dep.FullName = dep.GenDepFullName()
 
-		modFile, err := c.LoadModFile(localPath)
+		modFile, err := c.LoadModFile(dep.LocalFullPath)
 		if err != nil {
 			return nil, err
 		}

pkg/client/client_test.go

@@ -9,6 +9,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"sort"
 	"strings"
@@ -139,6 +140,129 @@ func TestDownloadLatestOci(t *testing.T) {
 	assert.Equal(t, utils.DirExists(filepath.Join(getTestDir("download"), "helloworld")), false)
 }
 
+func TestDownloadGitWithPackage(t *testing.T) {
+	testPath := filepath.Join(getTestDir("download"), "a_random_name")
+
+	defer func() {
+		err := os.RemoveAll(getTestDir("download"))
+		if err != nil {
+			t.Errorf("Failed to remove directory: %v", err)
+		}
+	}()
+
+	err := os.MkdirAll(testPath, 0755)
+	assert.Equal(t, err, nil)
+
+	depFromGit := pkg.Dependency{
+		Name:    "k8s",
+		Version: "",
+		Source: downloader.Source{
+			Git: &downloader.Git{
+				Url:     "https://github.com/kcl-lang/modules.git",
+				Commit:  "bdd4d00a88bc3534ae50affa8328df2927fd2171",
+				Package: "add-ndots",
+			},
+		},
+	}
+
+	kpmcli, err := NewKpmClient()
+	assert.Equal(t, err, nil)
+
+	dep, err := kpmcli.Download(&depFromGit, "", testPath)
+
+	assert.Equal(t, err, nil)
+	assert.Equal(t, dep.Source.Git.Package, "add-ndots")
+}
+
+func TestModandLockFilesWithGitPackageDownload(t *testing.T) {
+	testPkgPath := getTestDir("test_mod_file_package")
+
+	if runtime.GOOS == "windows" {
+		testPkgPath = filepath.Join(testPkgPath, "test_pkg_win")
+	} else {
+		testPkgPath = filepath.Join(testPkgPath, "test_pkg")
+	}
+
+	kpmcli, err := NewKpmClient()
+	assert.Equal(t, err, nil)
+
+	kclPkg, err := kpmcli.LoadPkgFromPath(testPkgPath)
+	assert.Equal(t, err, nil)
+
+	opts := opt.AddOptions{
+		LocalPath: testPkgPath,
+		RegistryOpts: opt.RegistryOptions{
+			Git: &opt.GitOptions{
+				Url:     "https://github.com/kcl-lang/modules.git",
+				Commit:  "ee03122b5f45b09eb48694422fc99a0772f6bba8",
+				Package: "agent",
+			},
+		},
+	}
+
+	_, err = kpmcli.AddDepWithOpts(kclPkg, &opts)
+	assert.Equal(t, err, nil)
+
+	testPkgPathMod := filepath.Join(testPkgPath, "kcl.mod")
+	testPkgPathModExpect := filepath.Join(testPkgPath, "expect.mod")
+	testPkgPathModLock := filepath.Join(testPkgPath, "kcl.mod.lock")
+	testPkgPathModLockExpect := filepath.Join(testPkgPath, "expect.mod.lock")
+
+	modContent, err := os.ReadFile(testPkgPathMod)
+	assert.Equal(t, err, nil)
+
+	modExpectContent, err := os.ReadFile(testPkgPathModExpect)
+	assert.Equal(t, err, nil)
+
+	modContentStr := string(modContent)
+	modExpectContentStr := string(modExpectContent)
+
+	for _, str := range []*string{&modContentStr, &modExpectContentStr} {
+		*str = strings.ReplaceAll(*str, " ", "")
+		*str = strings.ReplaceAll(*str, "\r\n", "")
+		*str = strings.ReplaceAll(*str, "\n", "")
+
+		sumRegex := regexp.MustCompile(`sum\s*=\s*"[^"]+"`)
+		*str = sumRegex.ReplaceAllString(*str, "")
+
+		*str = strings.TrimRight(*str, ", \t\r\n")
+	}
+
+	assert.Equal(t, modExpectContentStr, modContentStr)
+
+	modLockContent, err := os.ReadFile(testPkgPathModLock)
+	assert.Equal(t, err, nil)
+
+	modLockExpectContent, err := os.ReadFile(testPkgPathModLockExpect)
+	assert.Equal(t, err, nil)
+
+	modLockContentStr := string(modLockContent)
+	modLockExpectContentStr := string(modLockExpectContent)
+
+	for _, str := range []*string{&modLockContentStr, &modLockExpectContentStr} {
+		*str = strings.ReplaceAll(*str, " ", "")
+		*str = strings.ReplaceAll(*str, "\r\n", "")
+		*str = strings.ReplaceAll(*str, "\n", "")
+
+		sumRegex := regexp.MustCompile(`sum\s*=\s*"[^"]+"`)
+		*str = sumRegex.ReplaceAllString(*str, "")
+
+		*str = strings.TrimRight(*str, ", \t\r\n")
+	}
+
+	fmt.Println(modLockContentStr)
+
+	assert.Equal(t, modLockExpectContentStr, modLockContentStr)
+
+	defer func() {
+		err = os.Truncate(testPkgPathMod, 0)
+		assert.Equal(t, err, nil)
+
+		err = os.Truncate(testPkgPathModLock, 0)
+		assert.Equal(t, err, nil)
+	}()
+}
+
 func TestDependencyGraph(t *testing.T) {
 	testDir := getTestDir("test_dependency_graph")
 	assert.Equal(t, utils.DirExists(filepath.Join(testDir, "kcl.mod.lock")), false)
@@ -889,7 +1013,7 @@ func TestUpdateWithKclModlock(t *testing.T) {
 	err = kpmcli.UpdateDeps(kclPkg)
 	assert.Equal(t, err, nil)
 	got_lock_file := filepath.Join(dest_testDir, "kcl.mod.lock")
-	got_content, err := os.ReadFile(got_lock_file)
+	got_content, err := os.ReadFile(got_lock_file) // help
 	assert.Equal(t, err, nil)
 
 	expected_path := filepath.Join(dest_testDir, "expected")

pkg/client/test_data/test_mod_file_package/test_pkg/expect.mod

@@ -0,0 +1,4 @@
+[package]
+
+[dependencies]
+agent = { git = "https://github.com/kcl-lang/modules.git", commit = "ee03122b5f45b09eb48694422fc99a0772f6bba8", package = "agent" }

pkg/client/test_data/test_mod_file_package/test_pkg/expect.mod.lock

@@ -0,0 +1,12 @@
+[dependencies]
+  [dependencies.agent]
+    name = "agent"
+    full_name = "agent_ee03122b5f45b09eb48694422fc99a0772f6bba8"
+    version = "0.1.0"
+    url = "https://github.com/kcl-lang/modules.git"
+    commit = "ee03122b5f45b09eb48694422fc99a0772f6bba8"
+    package = "agent"
+  [dependencies.k8s]
+    name = "k8s"
+    full_name = "k8s_1.28"
+    version = "1.28"

pkg/client/test_data/test_mod_file_package/test_pkg_win/expect.mod

@@ -0,0 +1,4 @@
+[package]
+
+[dependencies]
+agent = { git = "https://github.com/kcl-lang/modules.git", commit = "ee03122b5f45b09eb48694422fc99a0772f6bba8", package = "agent" }

pkg/client/test_data/test_mod_file_package/test_pkg_win/expect.mod.lock

@@ -0,0 +1,12 @@
+[dependencies]
+  [dependencies.agent]
+    name = "agent"
+    full_name = "agent_ee03122b5f45b09eb48694422fc99a0772f6bba8"
+    version = "0.1.0"
+    url = "https://github.com/kcl-lang/modules.git"
+    commit = "ee03122b5f45b09eb48694422fc99a0772f6bba8"
+    package = "agent"
+  [dependencies.k8s]
+    name = "k8s"
+    full_name = "k8s_1.28"
+    version = "1.28"

pkg/cmd/cmd_add.go

@@ -45,6 +45,10 @@ func NewAddCmd(kpmcli *client.KpmClient) *cli.Command {
 				Name:  "rename",
 				Usage: "rename the package name in kcl.mod.lock",
 			},
+			&cli.StringSliceFlag{
+				Name:  "package",
+				Usage: "package name to use in case of git",
+			},
 		},
 
 		Action: func(c *cli.Context) error {
@@ -198,6 +202,12 @@ func parseGitRegistryOptions(c *cli.Context) (*opt.RegistryOptions, *reporter.Kp
 		return nil, err
 	}
 
+	gitPackage, err := onlyOnceOption(c, "package")
+
+	if err != (*reporter.KpmEvent)(nil) {
+		return nil, err
+	}
+
 	if gitUrl == "" {
 		return nil, reporter.NewErrorEvent(reporter.InvalidGitUrl, fmt.Errorf("the argument 'git' is required"))
 	}
@@ -208,9 +218,10 @@ func parseGitRegistryOptions(c *cli.Context) (*opt.RegistryOptions, *reporter.Kp
 
 	return &opt.RegistryOptions{
 		Git: &opt.GitOptions{
-			Url:    gitUrl,
-			Tag:    gitTag,
-			Commit: gitCommit,
+			Url:     gitUrl,
+			Tag:     gitTag,
+			Commit:  gitCommit,
+			Package: gitPackage,
 		},
 	}, nil
 }

pkg/downloader/source.go

@@ -39,6 +39,7 @@ type Git struct {
 	Commit  string `toml:"commit,omitempty"`
 	Tag     string `toml:"git_tag,omitempty"`
 	Version string `toml:"version,omitempty"`
+	Package string `toml:"package,omitempty"`
 }
 
 type Registry struct {
@@ -211,6 +212,13 @@ func (git *Git) ToFilePath() (string, error) {
 	), nil
 }
 
+func (git *Git) GetPackage() string {
+	if(git == nil) {
+		return ""
+	}
+	return git.Package
+}
+
 func (oci *Oci) ToFilePath() (string, error) {
 	if oci == nil {
 		return "", fmt.Errorf("oci source is nil")

pkg/downloader/toml.go

@@ -69,6 +69,7 @@ const TAG_PATTERN = "tag = \"%s\""
 const GIT_COMMIT_PATTERN = "commit = \"%s\""
 const GIT_BRANCH_PATTERN = "branch = \"%s\""
 const VERSION_PATTERN = "version = \"%s\""
+const GIT_PACKAGE = "package = \"%s\""
 const SEPARATOR = ", "
 
 func (git *Git) MarshalTOML() string {
@@ -94,6 +95,12 @@ func (git *Git) MarshalTOML() string {
 		sb.WriteString(SEPARATOR)
 		sb.WriteString(fmt.Sprintf(VERSION_PATTERN, git.Version))
 	}
+
+	if len(git.Package) != 0 {
+		sb.WriteString(SEPARATOR)
+		sb.WriteString(fmt.Sprintf(GIT_PACKAGE, git.Package))
+	}
+
 	return sb.String()
 }
 
@@ -175,6 +182,7 @@ const GIT_URL_FLAG = "git"
 const TAG_FLAG = "tag"
 const GIT_COMMIT_FLAG = "commit"
 const GIT_BRANCH_FLAG = "branch"
+const GIT_PACKAGE_FLAG = "package"
 
 func (git *Git) UnmarshalModTOML(data interface{}) error {
 	meta, ok := data.(map[string]interface{})
@@ -198,6 +206,10 @@ func (git *Git) UnmarshalModTOML(data interface{}) error {
 		git.Branch = v
 	}
 
+	if v, ok := meta[GIT_PACKAGE_FLAG].(string); ok {
+		git.Package = v
+	}
+
 	return nil
 }
 

pkg/opt/opt.go

@@ -378,10 +378,11 @@ func ParseLocalPathOptions(localPath string) (*LocalOptions, *reporter.KpmEvent)
 }
 
 type GitOptions struct {
-	Url    string
-	Branch string
-	Commit string
-	Tag    string
+	Url     string
+	Branch  string
+	Commit  string
+	Tag     string
+	Package string
 }
 
 func (opts *GitOptions) Validate() error {