From e8cb7387f793943199cbb87302b20323dd11b422 Mon Sep 17 00:00:00 2001 From: zongz Date: Thu, 14 Sep 2023 16:28:43 +0800 Subject: [PATCH 1/4] refactor: add struct 'KpmClient' to manage the settings of kpm --- kpm.go | 28 +- .../test_data_add_deps/main.k => main.k | 0 pkg/api/kpm_pkg.go | 7 +- pkg/api/kpm_pkg_test.go | 31 +- pkg/api/kpm_run.go | 44 +- pkg/api/kpm_run_test.go | 17 - pkg/client/client.go | 1019 +++++++++++++++++ pkg/client/client_test.go | 668 +++++++++++ pkg/client/test_data/expected/kcl.mod | 8 + .../test_data/expected}/kcl.mod.lock | 0 .../test_data/expected/kcl.mod.reverse.lock | 16 + .../test_data/expected/kcl.reverse.mod} | 0 .../resolve_deps/kpm_home/kcl1/kcl.mod | 0 .../resolve_deps/kpm_home/kcl1/kcl.mod.lock | 0 .../resolve_deps/kpm_home/kcl1/main.k | 0 .../resolve_deps/kpm_home/kcl2/kcl.mod | 0 .../resolve_deps/kpm_home/kcl2/kcl.mod.lock | 0 .../resolve_deps/kpm_home/kcl2/main.k | 0 .../resolve_deps/my_kcl_compile/main.k | 0 .../test_data/resolve_metadata/kcl.mod | 0 .../test_data/resolve_metadata/kcl.mod.lock | 0 .../test_data/tar_kcl_pkg/kcl.mod | 0 .../test_data/tar_kcl_pkg/kcl.mod.lock | 0 .../test_data/test_data_add_deps/kcl.mod | 8 + .../test_data/test_data_add_deps/kcl.mod.lock | 16 + .../test_data/test_data_add_deps}/main.k | 0 .../test_data/test_init_empty_mod/kcl.mod | 5 + .../test_init_empty_mod/kcl.mod.lock | 0 .../test_data/test_init_empty_mod}/main.k | 0 .../test_pkg_with_vendor/kcl2/kcl.mod | 0 .../test_pkg_with_vendor/kcl2/kcl.mod.lock | 0 .../test_pkg_with_vendor/kcl2/main.k | 1 + pkg/cmd/cmd_add.go | 41 +- pkg/cmd/cmd_init.go | 6 +- pkg/cmd/cmd_login.go | 7 +- pkg/cmd/cmd_logout.go | 9 +- pkg/cmd/cmd_metadata.go | 6 +- pkg/cmd/cmd_pkg.go | 7 +- pkg/cmd/cmd_pull.go | 117 +- pkg/cmd/cmd_push.go | 40 +- pkg/cmd/cmd_push_test.go | 5 +- pkg/cmd/cmd_run.go | 13 +- pkg/constants/constants.go | 22 +- pkg/git/git.go | 8 +- pkg/oci/oci.go | 39 +- pkg/opt/opt.go | 54 +- pkg/opt/opt_test.go | 26 - pkg/package/modfile.go | 195 +--- pkg/package/modfile_test.go | 79 -- pkg/package/package.go | 485 +------- pkg/package/package_test.go | 524 +-------- pkg/reporter/reporter.go | 2 + pkg/settings/settings.go | 5 +- pkg/settings/settings_test.go | 4 +- .../test_data/test_check_tar_path/test.tar | Bin pkg/utils/utils.go | 33 + pkg/utils/utils_test.go | 17 + .../kpm_pull_with_oci_url/test_suite.stdout | 4 +- .../test_suite.stdout | 4 +- .../kpm_pull_with_pkg_name/test_suite.stdout | 4 +- .../test_suite.stdout | 4 +- .../test_suite.stderr | 3 +- .../test_suite.stdout | 2 +- 63 files changed, 2032 insertions(+), 1601 deletions(-) rename pkg/package/test_data/test_data_add_deps/main.k => main.k (100%) create mode 100644 pkg/client/client.go create mode 100644 pkg/client/client_test.go create mode 100644 pkg/client/test_data/expected/kcl.mod rename pkg/{package/test_data/test_data_add_deps => client/test_data/expected}/kcl.mod.lock (100%) create mode 100644 pkg/client/test_data/expected/kcl.mod.reverse.lock rename pkg/{package/test_data/test_data_add_deps/kcl.mod => client/test_data/expected/kcl.reverse.mod} (100%) rename pkg/{package => client}/test_data/resolve_deps/kpm_home/kcl1/kcl.mod (100%) rename pkg/{package => client}/test_data/resolve_deps/kpm_home/kcl1/kcl.mod.lock (100%) rename pkg/{package => client}/test_data/resolve_deps/kpm_home/kcl1/main.k (100%) rename pkg/{package => client}/test_data/resolve_deps/kpm_home/kcl2/kcl.mod (100%) rename pkg/{package => client}/test_data/resolve_deps/kpm_home/kcl2/kcl.mod.lock (100%) rename pkg/{package => client}/test_data/resolve_deps/kpm_home/kcl2/main.k (100%) rename pkg/{package => client}/test_data/resolve_deps/my_kcl_compile/main.k (100%) rename pkg/{package => client}/test_data/resolve_metadata/kcl.mod (100%) rename pkg/{package => client}/test_data/resolve_metadata/kcl.mod.lock (100%) rename pkg/{package => client}/test_data/tar_kcl_pkg/kcl.mod (100%) rename pkg/{package => client}/test_data/tar_kcl_pkg/kcl.mod.lock (100%) create mode 100644 pkg/client/test_data/test_data_add_deps/kcl.mod create mode 100644 pkg/client/test_data/test_data_add_deps/kcl.mod.lock rename pkg/{package/test_data/test_init_empty_mod => client/test_data/test_data_add_deps}/main.k (100%) create mode 100644 pkg/client/test_data/test_init_empty_mod/kcl.mod rename pkg/{package => client}/test_data/test_init_empty_mod/kcl.mod.lock (100%) rename pkg/{package/test_data/test_pkg_with_vendor/kcl2 => client/test_data/test_init_empty_mod}/main.k (100%) rename pkg/{package => client}/test_data/test_pkg_with_vendor/kcl2/kcl.mod (100%) rename pkg/{package => client}/test_data/test_pkg_with_vendor/kcl2/kcl.mod.lock (100%) create mode 100644 pkg/client/test_data/test_pkg_with_vendor/kcl2/main.k rename pkg/{api => utils}/test_data/test_check_tar_path/test.tar (100%) diff --git a/kpm.go b/kpm.go index 267a027e..be564eba 100644 --- a/kpm.go +++ b/kpm.go @@ -6,17 +6,17 @@ import ( "os" "github.com/urfave/cli/v2" + "kcl-lang.io/kpm/pkg/client" "kcl-lang.io/kpm/pkg/cmd" "kcl-lang.io/kpm/pkg/reporter" - "kcl-lang.io/kpm/pkg/settings" "kcl-lang.io/kpm/pkg/version" ) func main() { reporter.InitReporter() - setting := settings.GetSettings() - if setting.ErrorEvent != nil { - reporter.Fatal(setting.ErrorEvent) + kpmcli, err := client.NewKpmClient() + if err != nil { + reporter.Fatal(err) } app := cli.NewApp() app.Name = "kpm" @@ -24,20 +24,20 @@ func main() { app.Version = version.GetVersionInStr() app.UsageText = "kpm [arguments]..." app.Commands = []*cli.Command{ - cmd.NewInitCmd(), - cmd.NewAddCmd(), - cmd.NewPkgCmd(), - cmd.NewMetadataCmd(), + cmd.NewInitCmd(kpmcli), + cmd.NewAddCmd(kpmcli), + cmd.NewPkgCmd(kpmcli), + cmd.NewMetadataCmd(kpmcli), // todo: The following commands are bound to the oci registry. // Refactor them to compatible with the other registry. - cmd.NewRunCmd(), - cmd.NewLoginCmd(setting), - cmd.NewLogoutCmd(setting), - cmd.NewPushCmd(setting), - cmd.NewPullCmd(), + cmd.NewRunCmd(kpmcli), + cmd.NewLoginCmd(kpmcli), + cmd.NewLogoutCmd(kpmcli), + cmd.NewPushCmd(kpmcli), + cmd.NewPullCmd(kpmcli), } - err := app.Run(os.Args) + err = app.Run(os.Args) if err != nil { reporter.Fatal(err) } diff --git a/pkg/package/test_data/test_data_add_deps/main.k b/main.k similarity index 100% rename from pkg/package/test_data/test_data_add_deps/main.k rename to main.k diff --git a/pkg/api/kpm_pkg.go b/pkg/api/kpm_pkg.go index 89c3fe16..d4ff4da0 100644 --- a/pkg/api/kpm_pkg.go +++ b/pkg/api/kpm_pkg.go @@ -6,6 +6,7 @@ import ( "kcl-lang.io/kcl-go/pkg/kcl" "kcl-lang.io/kcl-go/pkg/spec/gpyrpc" + "kcl-lang.io/kpm/pkg/client" "kcl-lang.io/kpm/pkg/errors" pkg "kcl-lang.io/kpm/pkg/package" ) @@ -57,7 +58,11 @@ func GetKclPackage(pkgPath string) (*KclPackage, error) { // // 'pkg_path' is the path of dependencies download by kpm. func (pkg *KclPackage) UpdateDependencyInPath(pkg_path string) error { - return pkg.pkg.ResolveDepsMetadata(pkg_path, true) + kpmcli, err := client.NewKpmClient() + if err != nil { + return err + } + return kpmcli.ResolvePkgDepsMetadata(pkg.pkg, true) } // GetPkgName returns the name of the package. diff --git a/pkg/api/kpm_pkg_test.go b/pkg/api/kpm_pkg_test.go index 87e2d4bd..99bd70e9 100644 --- a/pkg/api/kpm_pkg_test.go +++ b/pkg/api/kpm_pkg_test.go @@ -5,18 +5,18 @@ import ( "testing" "gotest.tools/v3/assert" + "kcl-lang.io/kpm/pkg/client" ) func TestPackageApi(t *testing.T) { pkg_path := filepath.Join(getTestDir("test_kpm_package"), "kcl_pkg") kcl_pkg_path, err := GetKclPkgPath() - + assert.Equal(t, err, nil) + kpmcli, err := client.NewKpmClient() assert.Equal(t, err, nil) pkg, err := GetKclPackage(pkg_path) assert.Equal(t, err, nil) - err = pkg.pkg.ResolveDepsMetadata(kcl_pkg_path, true) - assert.Equal(t, err, nil) - + err = kpmcli.ResolvePkgDepsMetadata(pkg.pkg, true) assert.Equal(t, err, nil) assert.Equal(t, pkg.GetPkgName(), "kcl_pkg") assert.Equal(t, pkg.GetVersion(), "0.0.1") @@ -60,12 +60,12 @@ func TestPackageApi(t *testing.T) { func TestGetAllSchemaTypesMappingNamed(t *testing.T) { pkg_path := filepath.Join(getTestDir("test_kpm_package"), "kcl_pkg") - kcl_pkg_path, err := GetKclPkgPath() - - assert.Equal(t, err, nil) pkg, err := GetKclPackage(pkg_path) assert.Equal(t, err, nil) - err = pkg.pkg.ResolveDepsMetadata(kcl_pkg_path, true) + kpmcli, err := client.NewKpmClient() + assert.Equal(t, err, nil) + + err = kpmcli.ResolvePkgDepsMetadata(pkg.pkg, true) assert.Equal(t, err, nil) schemas, err := pkg.GetSchemaTypeMappingNamed("SchemaWithSameName") @@ -84,14 +84,12 @@ func TestGetAllSchemaTypesMappingNamed(t *testing.T) { } func TestGetSchemaTypeMappingWithFilters(t *testing.T) { - pkg_path := filepath.Join(getTestDir("test_kpm_package"), "kcl_pkg") - kcl_pkg_path, err := GetKclPkgPath() - - assert.Equal(t, err, nil) pkg, err := GetKclPackage(pkg_path) assert.Equal(t, err, nil) - err = pkg.pkg.ResolveDepsMetadata(kcl_pkg_path, true) + kpmcli, err := client.NewKpmClient() + assert.Equal(t, err, nil) + err = kpmcli.ResolvePkgDepsMetadata(pkg.pkg, true) assert.Equal(t, err, nil) filterFunc := func(kt *KclType) bool { @@ -138,12 +136,11 @@ func TestGetSchemaTypeMappingWithFilters(t *testing.T) { func TestGetSchemaTypeUnderEmptyDir(t *testing.T) { pkg_path := filepath.Join(getTestDir("test_kpm_package"), "no_kcl_files") - kcl_pkg_path, err := GetKclPkgPath() - - assert.Equal(t, err, nil) pkg, err := GetKclPackage(pkg_path) assert.Equal(t, err, nil) - err = pkg.pkg.ResolveDepsMetadata(kcl_pkg_path, true) + kpmcli, err := client.NewKpmClient() + assert.Equal(t, err, nil) + err = kpmcli.ResolvePkgDepsMetadata(pkg.pkg, true) assert.Equal(t, err, nil) schemas, err := pkg.GetSchemaTypeMappingNamed("SchemaInMain") assert.Equal(t, err, nil) diff --git a/pkg/api/kpm_run.go b/pkg/api/kpm_run.go index 6a0edf23..d5a012ae 100644 --- a/pkg/api/kpm_run.go +++ b/pkg/api/kpm_run.go @@ -7,6 +7,8 @@ import ( "strings" "kcl-lang.io/kcl-go/pkg/kcl" + "kcl-lang.io/kpm/pkg/client" + "kcl-lang.io/kpm/pkg/constants" "kcl-lang.io/kpm/pkg/env" "kcl-lang.io/kpm/pkg/errors" "kcl-lang.io/kpm/pkg/oci" @@ -32,8 +34,6 @@ func RunTar(tarPath string, opts *opt.CompileOptions) (string, error) { return compileResult.GetRawYamlResult(), nil } -const KCL_PKG_TAR = "*.tar" - // RunOci will compile the kcl package from an OCI reference. func RunOci(ociRef, version string, opts *opt.CompileOptions) (string, error) { compileResult, compileErr := RunOciPkg(ociRef, version, opts) @@ -85,23 +85,6 @@ func RunWithOpt(opts *opt.CompileOptions) (*kcl.KCLResultList, error) { return kcl.RunWithOpts(*opts.Option) } -// absTarPath checks whether path 'tarPath' exists and whether path 'tarPath' ends with '.tar' -// And after checking, absTarPath return the abs path for 'tarPath'. -func absTarPath(tarPath string) (string, error) { - absTarPath, err := filepath.Abs(tarPath) - if err != nil { - return "", errors.InternalBug - } - - if filepath.Ext(absTarPath) != ".tar" { - return "", errors.InvalidKclPacakgeTar - } else if !utils.DirExists(absTarPath) { - return "", errors.KclPacakgeTarNotFound - } - - return absTarPath, nil -} - // getAbsInputPath will return the abs path of the file path described by '--input'. // If the path exists after 'inputPath' is computed as a full path, it will be returned. // If not, the kpm checks whether the full path of 'pkgPath/inputPath' exists, @@ -165,11 +148,13 @@ func RunPkgWithOpt(opts *opt.CompileOptions) (*kcl.KCLResultList, error) { // Calculate the absolute path of entry file described by '--input'. compiler := runner.NewCompilerWithOpts(opts) + kpmcli, err := client.NewKpmClient() + if err != nil { + return nil, err + } + // Call the kcl compiler. - compileResult, err := kclPkg.Compile( - globalPkgPath, - compiler, - ) + compileResult, err := kpmcli.Compile(kclPkg, compiler) if err != nil { return nil, reporter.NewErrorEvent(reporter.CompileFailed, err, "failed to compile the kcl package") @@ -192,7 +177,7 @@ func RunCurrentPkg(opts *opt.CompileOptions) (*kcl.KCLResultList, error) { // RunTarPkg will compile the kcl package from a kcl package tar. func RunTarPkg(tarPath string, opts *opt.CompileOptions) (*kcl.KCLResultList, error) { - absTarPath, err := absTarPath(tarPath) + absTarPath, err := utils.AbsTarPath(tarPath) if err != nil { return nil, err } @@ -216,7 +201,12 @@ func RunTarPkg(tarPath string, opts *opt.CompileOptions) (*kcl.KCLResultList, er // RunOciPkg will compile the kcl package from an OCI reference. func RunOciPkg(ociRef, version string, opts *opt.CompileOptions) (*kcl.KCLResultList, error) { - ociOpts, err := opt.ParseOciOptionFromString(ociRef, version) + kpmcli, err := client.NewKpmClient() + if err != nil { + return nil, err + } + + ociOpts, err := kpmcli.ParseOciOptionFromString(ociRef, version) if err != nil { return nil, err @@ -233,14 +223,14 @@ func RunOciPkg(ociRef, version string, opts *opt.CompileOptions) (*kcl.KCLResult localPath := ociOpts.AddStoragePathSuffix(tmpDir) // 2. Pull the tar. - err = oci.Pull(localPath, ociOpts.Reg, ociOpts.Repo, ociOpts.Tag) + err = oci.Pull(localPath, ociOpts.Reg, ociOpts.Repo, ociOpts.Tag, kpmcli.GetSettings()) if err != (*reporter.KpmEvent)(nil) { return nil, err } // 3.Get the (*.tar) file path. - matches, err := filepath.Glob(filepath.Join(localPath, KCL_PKG_TAR)) + matches, err := filepath.Glob(filepath.Join(localPath, constants.KCL_PKG_TAR)) if err != nil || len(matches) != 1 { return nil, errors.FailedPull } diff --git a/pkg/api/kpm_run_test.go b/pkg/api/kpm_run_test.go index ccfe0278..d64d3f15 100644 --- a/pkg/api/kpm_run_test.go +++ b/pkg/api/kpm_run_test.go @@ -41,23 +41,6 @@ func TestGetAbsInputPath(t *testing.T) { assert.Equal(t, path, "") } -func TestAbsTarPath(t *testing.T) { - pkgPath := getTestDir("test_check_tar_path") - expectAbsTarPath, _ := filepath.Abs(filepath.Join(pkgPath, "test.tar")) - - abs, err := absTarPath(filepath.Join(pkgPath, "test.tar")) - assert.Equal(t, err, nil) - assert.Equal(t, abs, expectAbsTarPath) - - abs, err = absTarPath(filepath.Join(pkgPath, "no_exist.tar")) - assert.NotEqual(t, err, nil) - assert.Equal(t, abs, "") - - abs, err = absTarPath(filepath.Join(pkgPath, "invalid_tar")) - assert.NotEqual(t, err, nil) - assert.Equal(t, abs, "") -} - func TestRunPkgInPath(t *testing.T) { pkgPath := getTestDir("test_run_pkg_in_path") opts := opt.DefaultCompileOptions() diff --git a/pkg/client/client.go b/pkg/client/client.go new file mode 100644 index 00000000..6b84fec7 --- /dev/null +++ b/pkg/client/client.go @@ -0,0 +1,1019 @@ +package client + +import ( + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "strings" + + "github.com/otiai10/copy" + "kcl-lang.io/kcl-go/pkg/kcl" + "kcl-lang.io/kpm/pkg/constants" + "kcl-lang.io/kpm/pkg/env" + "kcl-lang.io/kpm/pkg/errors" + "kcl-lang.io/kpm/pkg/git" + "kcl-lang.io/kpm/pkg/oci" + "kcl-lang.io/kpm/pkg/opt" + pkg "kcl-lang.io/kpm/pkg/package" + "kcl-lang.io/kpm/pkg/reporter" + "kcl-lang.io/kpm/pkg/runner" + "kcl-lang.io/kpm/pkg/settings" + "kcl-lang.io/kpm/pkg/utils" +) + +// KpmClient is the client of kpm. +type KpmClient struct { + // The writer of the log. + logWriter io.Writer + // The home path of kpm for global configuration file and kcl package storage path. + homePath string + // The settings of kpm loaded from the global configuration file. + settings settings.Settings +} + +// NewKpmClient will create a new kpm client with default settings. +func NewKpmClient() (*KpmClient, error) { + settings := settings.GetSettings() + + if settings.ErrorEvent != (*reporter.KpmEvent)(nil) { + return nil, settings.ErrorEvent + } + + homePath, err := env.GetAbsPkgPath() + if err != nil { + return nil, err + } + + return &KpmClient{ + logWriter: os.Stdout, + settings: *settings, + homePath: homePath, + }, nil +} + +// SetHomePath will set the home path of kpm. +func (c *KpmClient) SetHomePath(homePath string) { + c.homePath = homePath +} + +// AcquirePackageCacheLock will acquire the lock of the package cache. +func (c *KpmClient) AcquirePackageCacheLock() error { + return c.settings.AcquirePackageCacheLock(c.logWriter) +} + +// ReleasePackageCacheLock will release the lock of the package cache. +func (c *KpmClient) ReleasePackageCacheLock() error { + return c.settings.ReleasePackageCacheLock() +} + +// GetSettings will return the settings of kpm client. +func (c *KpmClient) GetSettings() *settings.Settings { + return &c.settings +} + +// ResolveDepsIntoMap will calculate the map of kcl package name and local storage path of the external packages. +func (c *KpmClient) ResolveDepsIntoMap(kclPkg *pkg.KclPkg) (map[string]string, error) { + err := c.ResolvePkgDepsMetadata(kclPkg, true) + if err != nil { + return nil, err + } + + var pkgMap map[string]string = make(map[string]string) + for name, d := range kclPkg.Dependencies.Deps { + pkgMap[name] = d.GetLocalFullPath(kclPkg.HomePath) + } + + return pkgMap, nil +} + +// ResolveDepsMetadata will calculate the local storage path of the external package, +// and check whether the package exists locally. +// If the package does not exist, it will re-download to the local. +func (c *KpmClient) ResolvePkgDepsMetadata(kclPkg *pkg.KclPkg, update bool) error { + var searchPath string + if kclPkg.IsVendorMode() { + // In the vendor mode, the search path is the vendor subdirectory of the current package. + err := c.VendorDeps(kclPkg) + if err != nil { + return err + } + searchPath = kclPkg.LocalVendorPath() + } else { + // Otherwise, the search path is the $KCL_PKG_PATH. + searchPath = c.homePath + } + + for name, d := range kclPkg.Dependencies.Deps { + searchFullPath := filepath.Join(searchPath, d.FullName) + if !update { + if utils.DirExists(searchFullPath) { + // Find it and update the local path of the dependency. + d.LocalFullPath = searchFullPath + kclPkg.Dependencies.Deps[name] = d + } + } else { + if utils.DirExists(searchFullPath) && utils.CheckPackageSum(d.Sum, searchFullPath) { + // Find it and update the local path of the dependency. + d.LocalFullPath = searchFullPath + kclPkg.Dependencies.Deps[name] = d + } else if d.IsFromLocal() && !utils.DirExists(d.GetLocalFullPath(kclPkg.HomePath)) { + return reporter.NewErrorEvent(reporter.DependencyNotFound, fmt.Errorf("dependency '%s' not found in '%s'", d.Name, searchFullPath)) + } else if d.IsFromLocal() && utils.DirExists(d.GetLocalFullPath(kclPkg.HomePath)) { + sum, err := utils.HashDir(d.GetLocalFullPath(kclPkg.HomePath)) + if err != nil { + return reporter.NewErrorEvent(reporter.CalSumFailed, err, fmt.Sprintf("failed to calculate checksum for '%s' in '%s'", d.Name, searchFullPath)) + } + d.Sum = sum + kclPkg.Dependencies.Deps[name] = d + } else { + // Otherwise, re-vendor it. + if kclPkg.IsVendorMode() { + err := c.VendorDeps(kclPkg) + if err != nil { + return err + } + } else { + // Or, re-download it. + err := c.AddDepToPkg(kclPkg, &d) + if err != nil { + return err + } + } + // After re-downloading or re-vendoring, + // re-resolving is required to update the dependent paths. + err := c.ResolvePkgDepsMetadata(kclPkg, update) + if err != nil { + return err + } + return nil + } + } + } + + return nil +} + +// ResolveDepsMetadataInJsonStr will calculate the local storage path of the external package, +// and check whether the package exists locally. If the package does not exist, it will re-download to the local. +// Finally, the calculated metadata of the dependent packages is serialized into a json string and returned. +func (c *KpmClient) ResolveDepsMetadataInJsonStr(kclPkg *pkg.KclPkg, update bool) (string, error) { + // 1. Calculate the dependency path, check whether the dependency exists + // and re-download the dependency that does not exist. + err := c.ResolvePkgDepsMetadata(kclPkg, update) + if err != nil { + return "", err + } + + // 2. Serialize to JSON + jsonData, err := json.Marshal(kclPkg.Dependencies) + if err != nil { + return "", errors.InternalBug + } + + return string(jsonData), nil +} + +// Compile will call kcl compiler to compile the current kcl package and its dependent packages. +func (c *KpmClient) Compile(kclPkg *pkg.KclPkg, kclvmCompiler *runner.Compiler) (*kcl.KCLResultList, error) { + pkgMap, err := c.ResolveDepsIntoMap(kclPkg) + if err != nil { + return nil, err + } + + // Fill the dependency path. + for dName, dPath := range pkgMap { + if !filepath.IsAbs(dPath) { + dPath = filepath.Join(c.homePath, dPath) + } + kclvmCompiler.AddDepPath(dName, dPath) + } + + return kclvmCompiler.Run() +} + +// CompileWithOpts will compile the kcl program with the compile options. +func (c *KpmClient) CompileWithOpts(opts *opt.CompileOptions) (*kcl.KCLResultList, error) { + pkgPath, err := filepath.Abs(opts.PkgPath()) + if err != nil { + return nil, errors.InternalBug + } + + kclPkg, err := pkg.LoadKclPkg(pkgPath) + if err != nil { + return nil, fmt.Errorf("kpm: failed to load package, please check the package path '%s' is valid", pkgPath) + } + + kclPkg.SetVendorMode(opts.IsVendor()) + + globalPkgPath, err := env.GetAbsPkgPath() + if err != nil { + return nil, err + } + + err = kclPkg.ValidateKpmHome(globalPkgPath) + if err != (*reporter.KpmEvent)(nil) { + return nil, err + } + + if len(opts.Entries()) > 0 { + // add entry from '--input' + for _, entry := range opts.Entries() { + if filepath.IsAbs(entry) { + opts.Merge(kcl.WithKFilenames(entry)) + } else { + opts.Merge(kcl.WithKFilenames(filepath.Join(opts.PkgPath(), entry))) + } + } + // add entry from 'kcl.mod' + } else if len(kclPkg.GetEntryKclFilesFromModFile()) > 0 { + opts.Merge(*kclPkg.GetKclOpts()) + } else if !opts.HasSettingsYaml() { + // no entry + opts.Merge(kcl.WithKFilenames(opts.PkgPath())) + } + opts.Merge(kcl.WithWorkDir(opts.PkgPath())) + + // Calculate the absolute path of entry file described by '--input'. + compiler := runner.NewCompilerWithOpts(opts) + + // Call the kcl compiler. + compileResult, err := c.Compile(kclPkg, compiler) + + if err != nil { + return nil, reporter.NewErrorEvent(reporter.CompileFailed, err, "failed to compile the kcl package") + } + + return compileResult, nil +} + +// CompilePkgWithOpts will compile the kcl package with the compile options. +func (c *KpmClient) CompilePkgWithOpts(kclPkg *pkg.KclPkg, opts *opt.CompileOptions) (*kcl.KCLResultList, error) { + opts.SetPkgPath(kclPkg.HomePath) + if len(opts.Entries()) > 0 { + // add entry from '--input' + for _, entry := range opts.Entries() { + if filepath.IsAbs(entry) { + opts.Merge(kcl.WithKFilenames(entry)) + } else { + opts.Merge(kcl.WithKFilenames(filepath.Join(opts.PkgPath(), entry))) + } + } + // add entry from 'kcl.mod' + } else if len(kclPkg.GetEntryKclFilesFromModFile()) > 0 { + opts.Merge(*kclPkg.GetKclOpts()) + } else if !opts.HasSettingsYaml() { + // no entry + opts.Merge(kcl.WithKFilenames(opts.PkgPath())) + } + opts.Merge(kcl.WithWorkDir(opts.PkgPath())) + // Calculate the absolute path of entry file described by '--input'. + compiler := runner.NewCompilerWithOpts(opts) + // Call the kcl compiler. + compileResult, err := c.Compile(kclPkg, compiler) + + if err != nil { + return nil, reporter.NewErrorEvent(reporter.CompileFailed, err, "failed to compile the kcl package") + } + + return compileResult, nil +} + +// CompileTarPkg will compile the kcl package from the tar package. +func (c *KpmClient) CompileTarPkg(tarPath string, opts *opt.CompileOptions) (*kcl.KCLResultList, error) { + absTarPath, err := utils.AbsTarPath(tarPath) + if err != nil { + return nil, err + } + // Extract the tar package to a directory with the same name. + // e.g. + // 'xxx/xxx/xxx/test.tar' will be extracted to the directory 'xxx/xxx/xxx/test'. + destDir := strings.TrimSuffix(absTarPath, filepath.Ext(absTarPath)) + err = utils.UnTarDir(absTarPath, destDir) + if err != nil { + return nil, err + } + + opts.SetPkgPath(destDir) + // The directory after extracting the tar package is taken as the root directory of the package, + // and kclvm is called to compile the kcl program under the 'destDir'. + // e.g. + // if the tar path is 'xxx/xxx/xxx/test.tar', + // the 'xxx/xxx/xxx/test' will be taken as the root path of the kcl package to compile. + return c.CompileWithOpts(opts) +} + +// CompileOciPkg will compile the kcl package from the OCI reference or url. +func (c *KpmClient) CompileOciPkg(ociSource, version string, opts *opt.CompileOptions) (*kcl.KCLResultList, error) { + ociOpts, err := c.ParseOciOptionFromString(ociSource, version) + + if err != nil { + return nil, err + } + + // 1. Create the temporary directory to pull the tar. + tmpDir, err := os.MkdirTemp("", "") + if err != nil { + return nil, errors.InternalBug + } + // clean the temp dir. + defer os.RemoveAll(tmpDir) + + localPath := ociOpts.AddStoragePathSuffix(tmpDir) + + // 2. Pull the tar. + err = oci.Pull(localPath, ociOpts.Reg, ociOpts.Repo, ociOpts.Tag, c.GetSettings()) + + if err != (*reporter.KpmEvent)(nil) { + return nil, err + } + + // 3.Get the (*.tar) file path. + matches, err := filepath.Glob(filepath.Join(localPath, constants.KCL_PKG_TAR)) + if err != nil || len(matches) != 1 { + return nil, errors.FailedPull + } + + return c.CompileTarPkg(matches[0], opts) +} + +// InitEmptyPkg will initialize an empty kcl package. +func (c *KpmClient) InitEmptyPkg(kclPkg *pkg.KclPkg) error { + err := utils.CreateFileIfNotExist( + kclPkg.ModFile.GetModFilePath(), + kclPkg.ModFile.StoreModFile, + ) + if err != nil { + return err + } + + err = utils.CreateFileIfNotExist( + kclPkg.ModFile.GetModLockFilePath(), + kclPkg.LockDepsVersion, + ) + if err != nil { + return err + } + + // create the default kcl program. + err = createDefaultPkgIn(kclPkg.HomePath) + if err != nil { + return err + } + + return nil +} + +// AddDepWithOpts will add a dependency to the current kcl package. +func (c *KpmClient) AddDepWithOpts(kclPkg *pkg.KclPkg, opt *opt.AddOptions) (*pkg.KclPkg, error) { + // 1. get the name and version of the repository from the input arguments. + d, err := pkg.ParseOpt(&opt.RegistryOpts) + if err != nil { + return nil, err + } + + reporter.ReportEventTo( + reporter.NewEvent(reporter.Adding, fmt.Sprintf("adding dependency '%s'.", d.Name)), + c.logWriter, + ) + // 2. download the dependency to the local path. + err = c.AddDepToPkg(kclPkg, d) + if err != nil { + return nil, err + } + + // 3. update the kcl.mod and kcl.mod.lock. + err = kclPkg.UpdateModAndLockFile() + if err != nil { + return nil, err + } + + succeedMsgInfo := d.Name + if len(d.Version) != 0 { + succeedMsgInfo = fmt.Sprintf("%s:%s", d.Name, d.Version) + } + + reporter.ReportEventTo( + reporter.NewEvent( + reporter.Adding, + fmt.Sprintf("add dependency '%s' successfully.", succeedMsgInfo), + ), + c.logWriter, + ) + return kclPkg, nil +} + +// AddDepToPkg will add a dependency to the kcl package. +func (c *KpmClient) AddDepToPkg(kclPkg *pkg.KclPkg, d *pkg.Dependency) error { + + if !reflect.DeepEqual(kclPkg.ModFile.Dependencies.Deps[d.Name], *d) { + // the dep passed on the cli is different from the kcl.mod. + kclPkg.ModFile.Dependencies.Deps[d.Name] = *d + } + + // download all the dependencies. + changedDeps, err := c.downloadDeps(kclPkg.ModFile.Dependencies, kclPkg.Dependencies) + + if err != nil { + return err + } + + // Update kcl.mod and kcl.mod.lock + for k, v := range changedDeps.Deps { + kclPkg.ModFile.Dependencies.Deps[k] = v + kclPkg.Dependencies.Deps[k] = v + } + + return err +} + +// PackagePkg will package the current kcl package into a "*.tar" file in under the package path. +func (c *KpmClient) PackagePkg(kclPkg *pkg.KclPkg, vendorMode bool) (string, error) { + globalPkgPath, err := env.GetAbsPkgPath() + if err != nil { + return "", err + } + + err = kclPkg.ValidateKpmHome(globalPkgPath) + if err != (*reporter.KpmEvent)(nil) { + return "", err + } + + err = c.Package(kclPkg, kclPkg.DefaultTarPath(), vendorMode) + + if err != nil { + reporter.ExitWithReport("kpm: failed to package pkg " + kclPkg.GetPkgName() + ".") + return "", err + } + return kclPkg.DefaultTarPath(), nil +} + +// Package will package the current kcl package into a "*.tar" file into 'tarPath'. +func (c *KpmClient) Package(kclPkg *pkg.KclPkg, tarPath string, vendorMode bool) error { + // Vendor all the dependencies into the current kcl package. + if vendorMode { + err := c.VendorDeps(kclPkg) + if err != nil { + return errors.FailedToVendorDependency + } + } + + // Tar the current kcl package into a "*.tar" file. + err := utils.TarDir(kclPkg.HomePath, tarPath) + if err != nil { + return errors.FailedToPackage + } + return nil +} + +// VendorDeps will vendor all the dependencies of the current kcl package. +func (c *KpmClient) VendorDeps(kclPkg *pkg.KclPkg) error { + // Mkdir the dir "vendor". + vendorPath := kclPkg.LocalVendorPath() + err := os.MkdirAll(vendorPath, 0755) + if err != nil { + return err + } + + lockDeps := make([]pkg.Dependency, 0, len(kclPkg.Dependencies.Deps)) + + for _, d := range kclPkg.Dependencies.Deps { + lockDeps = append(lockDeps, d) + } + + // Traverse all dependencies in kcl.mod.lock. + for i := 0; i < len(lockDeps); i++ { + d := lockDeps[i] + if len(d.Name) == 0 { + return errors.InvalidDependency + } + vendorFullPath := filepath.Join(vendorPath, d.FullName) + // If the package already exists in the 'vendor', do nothing. + if utils.DirExists(vendorFullPath) && check(d, vendorFullPath) { + continue + } else { + // If not in the 'vendor', check the global cache. + cacheFullPath := filepath.Join(c.homePath, d.FullName) + if utils.DirExists(cacheFullPath) && check(d, cacheFullPath) { + // If there is, copy it into the 'vendor' directory. + err := copy.Copy(cacheFullPath, vendorFullPath) + if err != nil { + return errors.FailedToVendorDependency + } + } else if utils.DirExists(d.GetLocalFullPath(kclPkg.HomePath)) && check(d, d.GetLocalFullPath(kclPkg.HomePath)) { + // If there is, copy it into the 'vendor' directory. + err := copy.Copy(d.GetLocalFullPath(kclPkg.HomePath), vendorFullPath) + if err != nil { + return errors.FailedToVendorDependency + } + } else { + // re-download if not. + err = c.AddDepToPkg(kclPkg, &d) + if err != nil { + return errors.FailedToVendorDependency + } + // re-vendor again with new kcl.mod and kcl.mod.lock + err = c.VendorDeps(kclPkg) + if err != nil { + return errors.FailedToVendorDependency + } + return nil + } + } + } + + return nil +} + +// FillDepInfo will fill registry information for a dependency. +func (c *KpmClient) FillDepInfo(dep *pkg.Dependency) error { + if dep.Source.Oci != nil { + dep.Source.Oci.Reg = c.GetSettings().DefaultOciRegistry() + urlpath := utils.JoinPath(c.GetSettings().DefaultOciRepo(), dep.Name) + dep.Source.Oci.Repo = urlpath + } + return nil +} + +// FillDependenciesInfo will fill registry information for all dependencies in a kcl.mod. +func (c *KpmClient) FillDependenciesInfo(modFile *pkg.ModFile) error { + for k, v := range modFile.Deps { + err := c.FillDepInfo(&v) + if err != nil { + return err + } + modFile.Deps[k] = v + } + return nil +} + +// Download will download the dependency to the local path. +func (c *KpmClient) Download(dep *pkg.Dependency, localPath string) (*pkg.Dependency, error) { + if dep.Source.Git != nil { + _, err := c.DownloadFromGit(dep.Source.Git, localPath) + if err != nil { + return nil, err + } + dep.Version = dep.Source.Git.Tag + dep.LocalFullPath = localPath + dep.FullName = dep.GenDepFullName() + } + + if dep.Source.Oci != nil { + localPath, err := c.DownloadFromOci(dep.Source.Oci, localPath) + if err != nil { + return nil, err + } + dep.Version = dep.Source.Oci.Tag + dep.LocalFullPath = localPath + dep.FullName = dep.GenDepFullName() + } + + if dep.Source.Local != nil { + dep.LocalFullPath = dep.Source.Local.Path + } + + var err error + dep.Sum, err = utils.HashDir(dep.LocalFullPath) + if err != nil { + return nil, reporter.NewErrorEvent( + reporter.FailedHashPkg, + err, + fmt.Sprintf("failed to hash the kcl package '%s' in '%s'.", dep.Name, dep.LocalFullPath), + ) + } + + return dep, nil +} + +// DownloadFromGit will download the dependency from the git repository. +func (c *KpmClient) DownloadFromGit(dep *pkg.Git, localPath string) (string, error) { + reporter.ReportEventTo( + reporter.NewEvent( + reporter.DownloadingFromGit, + fmt.Sprintf("downloading '%s' with tag '%s'.", dep.Url, dep.Tag), + ), + c.logWriter, + ) + + _, err := git.Clone(dep.Url, dep.Tag, localPath, c.logWriter) + + if err != nil { + return localPath, reporter.NewErrorEvent( + reporter.FailedCloneFromGit, + err, + fmt.Sprintf("failed to clone from '%s' into '%s'.", dep.Url, localPath), + ) + } + + return localPath, err +} + +// DownloadFromOci will download the dependency from the oci repository. +func (c *KpmClient) DownloadFromOci(dep *pkg.Oci, localPath string) (string, error) { + ociClient, err := oci.NewOciClient(dep.Reg, dep.Repo, &c.settings) + if err != nil { + return "", err + } + ociClient.SetLogWriter(c.logWriter) + // Select the latest tag, if the tag, the user inputed, is empty. + var tagSelected string + if len(dep.Tag) == 0 { + tagSelected, err = ociClient.TheLatestTag() + if err != nil { + return "", err + } + + reporter.ReportEventTo( + reporter.NewEvent(reporter.SelectLatestVersion, "the lastest version '", tagSelected, "' will be added."), + c.logWriter, + ) + + dep.Tag = tagSelected + localPath = localPath + dep.Tag + } else { + tagSelected = dep.Tag + } + + reporter.ReportEventTo( + reporter.NewEvent( + reporter.DownloadingFromOCI, + fmt.Sprintf("downloading '%s:%s' from '%s/%s:%s'.", dep.Repo, tagSelected, dep.Reg, dep.Repo, tagSelected), + ), + c.logWriter, + ) + + // Pull the package with the tag. + err = ociClient.Pull(localPath, tagSelected) + if err != nil { + return "", err + } + + matches, finderr := filepath.Glob(filepath.Join(localPath, "*.tar")) + if finderr != nil || len(matches) != 1 { + if finderr == nil { + err = reporter.NewErrorEvent( + reporter.InvalidKclPkg, + err, + fmt.Sprintf("failed to find the kcl package tar from '%s'.", localPath), + ) + } + + return "", reporter.NewErrorEvent( + reporter.InvalidKclPkg, + err, + fmt.Sprintf("failed to find the kcl package tar from '%s'.", localPath), + ) + } + + tarPath := matches[0] + untarErr := utils.UnTarDir(tarPath, localPath) + if untarErr != nil { + return "", reporter.NewErrorEvent( + reporter.FailedUntarKclPkg, + untarErr, + fmt.Sprintf("failed to untar the kcl package tar from '%s' into '%s'.", tarPath, localPath), + ) + } + + // After untar the downloaded kcl package tar file, remove the tar file. + if utils.DirExists(tarPath) { + rmErr := os.Remove(tarPath) + if rmErr != nil { + return "", reporter.NewErrorEvent( + reporter.FailedUntarKclPkg, + err, + fmt.Sprintf("failed to untar the kcl package tar from '%s' into '%s'.", tarPath, localPath), + ) + } + } + + return localPath, nil +} + +// PullFromOci will pull a kcl package from oci registry and unpack it. +func (c *KpmClient) PullFromOci(localPath, source, tag string) error { + localPath, err := filepath.Abs(localPath) + if err != nil { + return reporter.NewErrorEvent(reporter.Bug, err) + } + if len(source) == 0 { + return reporter.NewErrorEvent( + reporter.UnKnownPullWhat, + errors.FailedPull, + "oci url or package name must be specified.", + ) + } + + if len(tag) == 0 { + reporter.ReportEventTo( + reporter.NewEvent( + reporter.PullingStarted, + fmt.Sprintf("start to pull '%s'.", source), + ), + c.logWriter, + ) + } else { + reporter.ReportEventTo( + reporter.NewEvent( + reporter.PullingStarted, + fmt.Sprintf("start to pull '%s' with tag '%s'.", source, tag), + ), + c.logWriter, + ) + } + + ociOpts, err := c.ParseOciOptionFromString(source, tag) + if err != nil { + return err + } + + tmpDir, err := os.MkdirTemp("", "") + if err != nil { + return reporter.NewErrorEvent(reporter.Bug, err, fmt.Sprintf("failed to create temp dir '%s'.", tmpDir)) + } + // clean the temp dir. + defer os.RemoveAll(tmpDir) + + storepath := ociOpts.AddStoragePathSuffix(tmpDir) + err = c.pullTarFromOci(storepath, ociOpts) + if err != nil { + return err + } + + // Get the (*.tar) file path. + tarPath := filepath.Join(storepath, constants.KCL_PKG_TAR) + matches, err := filepath.Glob(tarPath) + if err != nil || len(matches) != 1 { + if err == nil { + err = errors.InvalidPkg + } + + return reporter.NewErrorEvent( + reporter.InvalidKclPkg, + err, + fmt.Sprintf("failed to find the kcl package tar from '%s'.", tarPath), + ) + } + + // Untar the tar file. + storagePath := ociOpts.AddStoragePathSuffix(localPath) + err = utils.UnTarDir(matches[0], storagePath) + if err != nil { + return reporter.NewErrorEvent( + reporter.FailedUntarKclPkg, + err, + fmt.Sprintf("failed to untar the kcl package tar from '%s' into '%s'.", matches[0], storagePath), + ) + } + + reporter.ReportEventTo( + reporter.NewEvent(reporter.PullingFinished, fmt.Sprintf("pulled '%s' in '%s' successfully.", source, storagePath)), + c.logWriter, + ) + return nil +} + +// PushToOci will push a kcl package to oci registry. +func (c *KpmClient) PushToOci(localPath string, ociOpts *opt.OciOptions) error { + ociCli, err := oci.NewOciClient(ociOpts.Reg, ociOpts.Repo, &c.settings) + if err != nil { + return err + } + + ociCli.SetLogWriter(c.logWriter) + + exist, err := ociCli.ContainsTag(ociOpts.Tag) + if err != (*reporter.KpmEvent)(nil) { + return err + } + + if exist { + return reporter.NewErrorEvent( + reporter.PkgTagExists, + fmt.Errorf("package version '%s' already exists", ociOpts.Tag), + ) + } + + return ociCli.Push(localPath, ociOpts.Tag) +} + +// LoginOci will login to the oci registry. +func (c *KpmClient) LoginOci(hostname, username, password string) error { + return oci.Login(hostname, username, password, &c.settings) +} + +// LogoutOci will logout from the oci registry. +func (c *KpmClient) LogoutOci(hostname string) error { + return oci.Logout(hostname, &c.settings) +} + +// ParseOciRef will parser ':' into an 'OciOptions'. +func (c *KpmClient) ParseOciRef(ociRef string) (*opt.OciOptions, error) { + oci_address := strings.Split(ociRef, constants.OCI_SEPARATOR) + if len(oci_address) == 1 { + return &opt.OciOptions{ + Reg: c.GetSettings().DefaultOciRegistry(), + Repo: utils.JoinPath(c.GetSettings().DefaultOciRepo(), oci_address[0]), + }, nil + } else if len(oci_address) == 2 { + return &opt.OciOptions{ + Reg: c.GetSettings().DefaultOciRegistry(), + Repo: utils.JoinPath(c.GetSettings().DefaultOciRepo(), oci_address[0]), + Tag: oci_address[1], + }, nil + } else { + return nil, reporter.NewEvent(reporter.IsNotRef) + } +} + +// ParseOciOptionFromString will parser ':' into an 'OciOptions' with an OCI registry. +// the default OCI registry is 'docker.io'. +// if the 'ociUrl' is only '', ParseOciOptionFromString will take 'latest' as the default tag. +func (c *KpmClient) ParseOciOptionFromString(oci string, tag string) (*opt.OciOptions, error) { + ociOpt, event := opt.ParseOciUrl(oci) + if event != nil && (event.Type() == reporter.IsNotUrl || event.Type() == reporter.UrlSchemeNotOci) { + ociOpt, err := c.ParseOciRef(oci) + if err != nil { + return nil, err + } + if len(tag) != 0 { + reporter.ReportEventTo( + reporter.NewEvent( + reporter.InvalidFlag, + "kpm get version from oci reference ':'", + ), + c.logWriter, + ) + reporter.ReportEventTo( + reporter.NewEvent( + reporter.InvalidFlag, + "arg '--tag' is invalid for oci reference", + ), + c.logWriter, + ) + } + return ociOpt, nil + } + + ociOpt.Tag = tag + + return ociOpt, nil +} + +// downloadDeps will download all the dependencies of the current kcl package. +func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencies) (*pkg.Dependencies, error) { + newDeps := pkg.Dependencies{ + Deps: make(map[string]pkg.Dependency), + } + + // Traverse all dependencies in kcl.mod + for _, d := range deps.Deps { + if len(d.Name) == 0 { + return nil, errors.InvalidDependency + } + + lockDep, present := lockDeps.Deps[d.Name] + + // Check if the sum of this dependency in kcl.mod.lock has been chanaged. + if present { + // If the dependent package does not exist locally, then method 'check' will return false. + if check(lockDep, filepath.Join(c.homePath, d.FullName)) { + newDeps.Deps[d.Name] = lockDep + continue + } + } + expectedSum := lockDeps.Deps[d.Name].Sum + // Clean the cache + if len(c.homePath) == 0 || len(d.FullName) == 0 { + return nil, errors.InternalBug + } + dir := filepath.Join(c.homePath, d.FullName) + os.RemoveAll(dir) + + // download dependencies + + lockedDep, err := c.Download(&d, dir) + if err != nil { + return nil, err + } + + if !lockedDep.IsFromLocal() { + if expectedSum != "" && lockedDep.Sum != expectedSum && lockDep.FullName == d.FullName { + return nil, reporter.NewErrorEvent( + reporter.CheckSumMismatch, + errors.CheckSumMismatchError, + fmt.Sprintf("checksum for '%s' changed in lock file", lockedDep.Name), + ) + } + } + + // Update kcl.mod and kcl.mod.lock + newDeps.Deps[d.Name] = *lockedDep + lockDeps.Deps[d.Name] = *lockedDep + } + + // Recursively download the dependencies of the new dependencies. + for _, d := range newDeps.Deps { + // Load kcl.mod file of the new downloaded dependencies. + deppkg, err := pkg.LoadKclPkg(filepath.Join(c.homePath, d.FullName)) + if len(d.LocalFullPath) != 0 { + deppkg, err = pkg.LoadKclPkg(d.LocalFullPath) + } + + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, err + } + + // Download the dependencies. + nested, err := c.downloadDeps(deppkg.ModFile.Dependencies, lockDeps) + if err != nil { + return nil, err + } + + // Update kcl.mod. + for _, d := range nested.Deps { + if _, ok := newDeps.Deps[d.Name]; !ok { + newDeps.Deps[d.Name] = d + } + } + } + + return &newDeps, nil +} + +// pullTarFromOci will pull a kcl package tar file from oci registry. +func (c *KpmClient) pullTarFromOci(localPath string, ociOpts *opt.OciOptions) error { + absPullPath, err := filepath.Abs(localPath) + if err != nil { + return reporter.NewErrorEvent(reporter.Bug, err) + } + + ociCli, err := oci.NewOciClient(ociOpts.Reg, ociOpts.Repo, &c.settings) + if err != nil { + return err + } + + ociCli.SetLogWriter(c.logWriter) + + var tagSelected string + if len(ociOpts.Tag) == 0 { + tagSelected, err = ociCli.TheLatestTag() + if err != nil { + return err + } + reporter.ReportEventTo( + reporter.NewEvent(reporter.SelectLatestVersion, "the lastest version '", tagSelected, "' will be pulled."), + c.logWriter, + ) + } else { + tagSelected = ociOpts.Tag + } + + reporter.ReportEventTo( + reporter.NewEvent( + reporter.Pulling, + fmt.Sprintf("pulling '%s'.", ociCli.GetReference()), + ), + c.logWriter, + ) + + err = ociCli.Pull(absPullPath, tagSelected) + if err != nil { + return err + } + + return nil +} + +// createDefaultPkgIn will create the default kcl program in the local path. +func createDefaultPkgIn(localPath string) error { + mainProgPath := filepath.Join(localPath, constants.DEFAULT_KCL_FILE_NAME) + if !utils.DirExists(mainProgPath) { + err := os.WriteFile(mainProgPath, []byte(constants.DEFAULT_KCL_FILE_CONTENT), 0644) + if err != nil { + return err + } + } + return nil +} + +// check sum for a Dependency. +func check(dep pkg.Dependency, newDepPath string) bool { + if dep.Sum == "" { + return false + } + + sum, err := utils.HashDir(newDepPath) + + if err != nil { + return false + } + + return dep.Sum == sum +} diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go new file mode 100644 index 00000000..da8cc2b7 --- /dev/null +++ b/pkg/client/client_test.go @@ -0,0 +1,668 @@ +package client + +import ( + "archive/tar" + "encoding/json" + "fmt" + "io" + "log" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "kcl-lang.io/kpm/pkg/env" + "kcl-lang.io/kpm/pkg/opt" + pkg "kcl-lang.io/kpm/pkg/package" + "kcl-lang.io/kpm/pkg/runner" + "kcl-lang.io/kpm/pkg/utils" +) + +const testDataDir = "test_data" + +func getTestDir(subDir string) string { + pwd, _ := os.Getwd() + testDir := filepath.Join(pwd, testDataDir) + testDir = filepath.Join(testDir, subDir) + + return testDir +} + +func initTestDir(subDir string) string { + testDir := getTestDir(subDir) + // clean the test data + _ = os.RemoveAll(testDir) + _ = os.Mkdir(testDir, 0755) + + return testDir +} + +// TestDownloadGit test download from oci registry. +func TestDownloadOci(t *testing.T) { + testPath := filepath.Join(getTestDir("download"), "k8s_1.27") + err := os.MkdirAll(testPath, 0755) + assert.Equal(t, err, nil) + + depFromOci := pkg.Dependency{ + Name: "k8s", + Version: "1.27", + Source: pkg.Source{ + Oci: &pkg.Oci{ + Reg: "ghcr.io", + Repo: "kcl-lang/k8s", + Tag: "1.27", + }, + }, + } + kpmcli, err := NewKpmClient() + assert.Equal(t, err, nil) + dep, err := kpmcli.Download(&depFromOci, testPath) + assert.Equal(t, err, nil) + assert.Equal(t, dep.Name, "k8s") + assert.Equal(t, dep.FullName, "k8s_1.27") + assert.Equal(t, dep.Version, "1.27") + assert.Equal(t, dep.Sum, "xnYM1FWHAy3m+KcQMQb2rjZouTxumqYt6FGZpu2T4yM=") + assert.NotEqual(t, dep.Source.Oci, nil) + assert.Equal(t, dep.Source.Oci.Reg, "ghcr.io") + assert.Equal(t, dep.Source.Oci.Repo, "kcl-lang/k8s") + assert.Equal(t, dep.Source.Oci.Tag, "1.27") + assert.Equal(t, dep.LocalFullPath, testPath) + + // Check whether the tar downloaded by `kpm add` has been deleted. + assert.Equal(t, utils.DirExists(filepath.Join(testPath, "k8s_1.27.tar")), false) + + err = os.RemoveAll(getTestDir("download")) + assert.Equal(t, err, nil) +} + +// TestDownloadLatestOci tests the case that the version is empty. +func TestDownloadLatestOci(t *testing.T) { + testPath := filepath.Join(getTestDir("download"), "a_random_name") + err := os.MkdirAll(testPath, 0755) + assert.Equal(t, err, nil) + depFromOci := pkg.Dependency{ + Name: "k8s", + Version: "", + Source: pkg.Source{ + Oci: &pkg.Oci{ + Reg: "ghcr.io", + Repo: "kcl-lang/k8s", + Tag: "", + }, + }, + } + kpmcli, err := NewKpmClient() + assert.Equal(t, err, nil) + dep, err := kpmcli.Download(&depFromOci, testPath) + assert.Equal(t, err, nil) + assert.Equal(t, dep.Name, "k8s") + assert.Equal(t, dep.FullName, "k8s_1.27") + assert.Equal(t, dep.Version, "1.27") + assert.Equal(t, dep.Sum, "xnYM1FWHAy3m+KcQMQb2rjZouTxumqYt6FGZpu2T4yM=") + assert.NotEqual(t, dep.Source.Oci, nil) + assert.Equal(t, dep.Source.Oci.Reg, "ghcr.io") + assert.Equal(t, dep.Source.Oci.Repo, "kcl-lang/k8s") + assert.Equal(t, dep.Source.Oci.Tag, "1.27") + assert.Equal(t, dep.LocalFullPath, testPath+"1.27") + assert.Equal(t, err, nil) + + // Check whether the tar downloaded by `kpm add` has been deleted. + assert.Equal(t, utils.DirExists(filepath.Join(testPath, "k8s_1.27.tar")), false) + + err = os.RemoveAll(getTestDir("download")) + assert.Equal(t, err, nil) +} + +func TestInitEmptyPkg(t *testing.T) { + testDir := initTestDir("test_init_empty_mod") + kclPkg := pkg.NewKclPkg(&opt.InitOptions{Name: "test_name", InitPath: testDir}) + kpmcli, err := NewKpmClient() + assert.Equal(t, err, nil) + err = kpmcli.InitEmptyPkg(&kclPkg) + assert.Equal(t, err, nil) + + testKclPkg, err := pkg.LoadKclPkg(testDir) + if err != nil { + t.Errorf("Failed to 'LoadKclPkg'.") + } + + assert.Equal(t, testKclPkg.ModFile.Pkg.Name, "test_name") + assert.Equal(t, testKclPkg.ModFile.Pkg.Version, "0.0.1") + assert.Equal(t, testKclPkg.ModFile.Pkg.Edition, "0.0.1") +} + +func TestUpdateKclModAndLock(t *testing.T) { + testDir := initTestDir("test_data_add_deps") + // Init an empty package + kclPkg := pkg.NewKclPkg(&opt.InitOptions{ + Name: "test_add_deps", + InitPath: testDir, + }) + + kpmcli, err := NewKpmClient() + assert.Equal(t, err, nil) + err = kpmcli.InitEmptyPkg(&kclPkg) + assert.Equal(t, err, nil) + + dep := pkg.Dependency{ + Name: "name", + FullName: "test_version", + Version: "test_version", + Sum: "test_sum", + Source: pkg.Source{ + Git: &pkg.Git{ + Url: "test_url", + Tag: "test_tag", + }, + }, + } + + oci_dep := pkg.Dependency{ + Name: "oci_name", + FullName: "test_version", + Version: "test_version", + Sum: "test_sum", + Source: pkg.Source{ + Oci: &pkg.Oci{ + Reg: "test_reg", + Repo: "test_repo", + Tag: "test_tag", + }, + }, + } + + kclPkg.Dependencies.Deps["oci_test"] = oci_dep + kclPkg.ModFile.Dependencies.Deps["oci_test"] = oci_dep + + kclPkg.Dependencies.Deps["test"] = dep + kclPkg.ModFile.Dependencies.Deps["test"] = dep + + err = kclPkg.ModFile.StoreModFile() + + if err != nil { + t.Errorf("failed to LockDepsVersion.") + } + + err = kclPkg.LockDepsVersion() + + if err != nil { + t.Errorf("failed to LockDepsVersion.") + } + + expectDir := getTestDir("expected") + + if gotKclMod, err := os.ReadFile(filepath.Join(testDir, "kcl.mod")); os.IsNotExist(err) { + t.Errorf("failed to find kcl.mod.") + } else { + assert.Equal(t, len(kclPkg.Dependencies.Deps), 2) + assert.Equal(t, len(kclPkg.ModFile.Deps), 2) + expectKclMod, _ := os.ReadFile(filepath.Join(expectDir, "kcl.mod")) + expectKclModReverse, _ := os.ReadFile(filepath.Join(expectDir, "kcl.reverse.mod")) + + gotKclModStr := utils.RmNewline(string(gotKclMod)) + fmt.Printf("gotKclModStr: '%v'\n", gotKclModStr) + expectKclModStr := utils.RmNewline(string(expectKclMod)) + fmt.Printf("expectKclModStr: '%v'\n", expectKclModStr) + expectKclModReverseStr := utils.RmNewline(string(expectKclModReverse)) + fmt.Printf("expectKclModReverseStr: '%v'\n", expectKclModReverseStr) + + assert.Equal(t, + (gotKclModStr == expectKclModStr || gotKclModStr == expectKclModReverseStr), + true, + ) + } + + if gotKclModLock, err := os.ReadFile(filepath.Join(testDir, "kcl.mod.lock")); os.IsNotExist(err) { + t.Errorf("failed to find kcl.mod.lock.") + } else { + assert.Equal(t, len(kclPkg.Dependencies.Deps), 2) + assert.Equal(t, len(kclPkg.ModFile.Deps), 2) + expectKclModLock, _ := os.ReadFile(filepath.Join(expectDir, "kcl.mod.lock")) + expectKclModLockReverse, _ := os.ReadFile(filepath.Join(expectDir, "kcl.mod.reverse.lock")) + + gotKclModLockStr := utils.RmNewline(string(gotKclModLock)) + fmt.Printf("gotKclModLockStr: '%v'\n", gotKclModLockStr) + expectKclModLockStr := utils.RmNewline(string(expectKclModLock)) + fmt.Printf("expectKclModLockStr: '%v'\n", expectKclModLockStr) + expectKclModLockReverseStr := utils.RmNewline(string(expectKclModLockReverse)) + fmt.Printf("expectKclModLockReverseStr: '%v'\n", expectKclModLockReverseStr) + + assert.Equal(t, + (gotKclModLockStr == expectKclModLockStr) || (gotKclModLockStr == expectKclModLockReverseStr), + true, + ) + } +} + +func TestVendorDeps(t *testing.T) { + testDir := getTestDir("resolve_deps") + kpm_home := filepath.Join(testDir, "kpm_home") + os.RemoveAll(filepath.Join(testDir, "my_kcl")) + kcl1Sum, _ := utils.HashDir(filepath.Join(kpm_home, "kcl1")) + kcl2Sum, _ := utils.HashDir(filepath.Join(kpm_home, "kcl2")) + + depKcl1 := pkg.Dependency{ + Name: "kcl1", + FullName: "kcl1", + Sum: kcl1Sum, + } + + depKcl2 := pkg.Dependency{ + Name: "kcl2", + FullName: "kcl2", + Sum: kcl2Sum, + } + + kclPkg := pkg.KclPkg{ + ModFile: pkg.ModFile{ + HomePath: filepath.Join(testDir, "my_kcl"), + // Whether the current package uses the vendor mode + // In the vendor mode, kpm will look for the package in the vendor subdirectory + // in the current package directory. + VendorMode: false, + Dependencies: pkg.Dependencies{ + Deps: map[string]pkg.Dependency{ + "kcl1": depKcl1, + "kcl2": depKcl2, + }, + }, + }, + HomePath: filepath.Join(testDir, "my_kcl"), + // The dependencies in the current kcl package are the dependencies of kcl.mod.lock, + // not the dependencies in kcl.mod. + Dependencies: pkg.Dependencies{ + Deps: map[string]pkg.Dependency{ + "kcl1": depKcl1, + "kcl2": depKcl2, + }, + }, + } + + mykclVendorPath := filepath.Join(filepath.Join(testDir, "my_kcl"), "vendor") + assert.Equal(t, utils.DirExists(mykclVendorPath), false) + kpmcli, err := NewKpmClient() + kpmcli.homePath = kpm_home + assert.Equal(t, err, nil) + err = kpmcli.VendorDeps(&kclPkg) + assert.Equal(t, err, nil) + assert.Equal(t, utils.DirExists(mykclVendorPath), true) + assert.Equal(t, utils.DirExists(filepath.Join(mykclVendorPath, "kcl1")), true) + assert.Equal(t, utils.DirExists(filepath.Join(mykclVendorPath, "kcl2")), true) + + maps, err := kpmcli.ResolveDepsIntoMap(&kclPkg) + assert.Equal(t, err, nil) + assert.Equal(t, len(maps), 2) + + os.RemoveAll(filepath.Join(testDir, "my_kcl")) +} + +func TestResolveDepsVendorMode(t *testing.T) { + testDir := getTestDir("resolve_deps") + kpm_home := filepath.Join(testDir, "kpm_home") + home_path := filepath.Join(testDir, "my_kcl_resolve_deps_vendor_mode") + os.RemoveAll(home_path) + kcl1Sum, _ := utils.HashDir(filepath.Join(kpm_home, "kcl1")) + kcl2Sum, _ := utils.HashDir(filepath.Join(kpm_home, "kcl2")) + + depKcl1 := pkg.Dependency{ + Name: "kcl1", + FullName: "kcl1", + Sum: kcl1Sum, + } + + depKcl2 := pkg.Dependency{ + Name: "kcl2", + FullName: "kcl2", + Sum: kcl2Sum, + } + + kclPkg := pkg.KclPkg{ + ModFile: pkg.ModFile{ + HomePath: home_path, + // Whether the current package uses the vendor mode + // In the vendor mode, kpm will look for the package in the vendor subdirectory + // in the current package directory. + VendorMode: true, + Dependencies: pkg.Dependencies{ + Deps: map[string]pkg.Dependency{ + "kcl1": depKcl1, + "kcl2": depKcl2, + }, + }, + }, + HomePath: home_path, + // The dependencies in the current kcl package are the dependencies of kcl.mod.lock, + // not the dependencies in kcl.mod. + Dependencies: pkg.Dependencies{ + Deps: map[string]pkg.Dependency{ + "kcl1": depKcl1, + "kcl2": depKcl2, + }, + }, + } + mySearchPath := filepath.Join(home_path, "vendor") + assert.Equal(t, utils.DirExists(mySearchPath), false) + + kpmcli, err := NewKpmClient() + assert.Equal(t, err, nil) + kpmcli.homePath = kpm_home + + maps, err := kpmcli.ResolveDepsIntoMap(&kclPkg) + assert.Equal(t, err, nil) + assert.Equal(t, len(maps), 2) + checkDepsMapInSearchPath(t, depKcl1, mySearchPath, maps) + + kclPkg.SetVendorMode(false) + maps, err = kpmcli.ResolveDepsIntoMap(&kclPkg) + assert.Equal(t, err, nil) + assert.Equal(t, len(maps), 2) + checkDepsMapInSearchPath(t, depKcl1, kpm_home, maps) + + os.RemoveAll(home_path) +} + +func TestCompileWithEntryFile(t *testing.T) { + testDir := getTestDir("resolve_deps") + kpm_home := filepath.Join(testDir, "kpm_home") + home_path := filepath.Join(testDir, "my_kcl_compile") + vendor_path := filepath.Join(home_path, "vendor") + entry_file := filepath.Join(home_path, "main.k") + os.RemoveAll(vendor_path) + + kcl1Sum, _ := utils.HashDir(filepath.Join(kpm_home, "kcl1")) + depKcl1 := pkg.Dependency{ + Name: "kcl1", + FullName: "kcl1", + Sum: kcl1Sum, + } + kcl2Sum, _ := utils.HashDir(filepath.Join(kpm_home, "kcl2")) + depKcl2 := pkg.Dependency{ + Name: "kcl2", + FullName: "kcl2", + Sum: kcl2Sum, + } + + kclPkg := pkg.KclPkg{ + ModFile: pkg.ModFile{ + HomePath: home_path, + // Whether the current package uses the vendor mode + // In the vendor mode, kpm will look for the package in the vendor subdirectory + // in the current package directory. + VendorMode: true, + Dependencies: pkg.Dependencies{ + Deps: map[string]pkg.Dependency{ + "kcl1": depKcl1, + "kcl2": depKcl2, + }, + }, + }, + HomePath: home_path, + // The dependencies in the current kcl package are the dependencies of kcl.mod.lock, + // not the dependencies in kcl.mod. + Dependencies: pkg.Dependencies{ + Deps: map[string]pkg.Dependency{ + "kcl1": depKcl1, + "kcl2": depKcl2, + }, + }, + } + + assert.Equal(t, utils.DirExists(vendor_path), false) + + compiler := runner.DefaultCompiler() + compiler.AddKFile(entry_file) + kpmcli, err := NewKpmClient() + assert.Equal(t, err, nil) + kpmcli.homePath = kpm_home + result, err := kpmcli.Compile(&kclPkg, compiler) + assert.Equal(t, err, nil) + assert.Equal(t, utils.DirExists(filepath.Join(vendor_path, "kcl1")), true) + assert.Equal(t, utils.DirExists(filepath.Join(vendor_path, "kcl2")), true) + assert.Equal(t, result.GetRawYamlResult(), "c1: 1\nc2: 2") + os.RemoveAll(vendor_path) + + kclPkg.SetVendorMode(false) + assert.Equal(t, utils.DirExists(vendor_path), false) + + result, err = kpmcli.Compile(&kclPkg, compiler) + assert.Equal(t, err, nil) + assert.Equal(t, utils.DirExists(vendor_path), false) + assert.Equal(t, result.GetRawYamlResult(), "c1: 1\nc2: 2") + os.RemoveAll(vendor_path) +} + +func checkDepsMapInSearchPath(t *testing.T, dep pkg.Dependency, searchPath string, maps map[string]string) { + assert.Equal(t, maps[dep.Name], filepath.Join(searchPath, dep.FullName)) + assert.Equal(t, utils.DirExists(filepath.Join(searchPath, dep.FullName)), true) +} + +func TestPackageCurrentPkgPath(t *testing.T) { + testDir := getTestDir("tar_kcl_pkg") + + kclPkg, err := pkg.LoadKclPkg(testDir) + assert.Equal(t, err, nil) + assert.Equal(t, kclPkg.GetPkgTag(), "0.0.1") + assert.Equal(t, kclPkg.GetPkgName(), "test_tar") + assert.Equal(t, kclPkg.GetPkgFullName(), "test_tar-0.0.1") + assert.Equal(t, kclPkg.GetPkgTarName(), "test_tar-0.0.1.tar") + + assert.Equal(t, utils.DirExists(filepath.Join(testDir, kclPkg.GetPkgTarName())), false) + + kpmcli, err := NewKpmClient() + assert.Equal(t, err, nil) + path, err := kpmcli.PackagePkg(kclPkg, true) + assert.Equal(t, err, nil) + assert.Equal(t, path, filepath.Join(testDir, kclPkg.GetPkgTarName())) + assert.Equal(t, utils.DirExists(filepath.Join(testDir, kclPkg.GetPkgTarName())), true) + defer func() { + if r := os.RemoveAll(filepath.Join(testDir, kclPkg.GetPkgTarName())); r != nil { + err = fmt.Errorf("panic: %v", r) + } + }() +} + +func TestResolveMetadataInJsonStr(t *testing.T) { + originalValue := os.Getenv(env.PKG_PATH) + defer os.Setenv(env.PKG_PATH, originalValue) + + testDir := getTestDir("resolve_metadata") + + testHomePath := filepath.Join(filepath.Dir(testDir), "test_home_path") + prepareKpmHomeInPath(testHomePath) + defer os.RemoveAll(testHomePath) + + os.Setenv(env.PKG_PATH, testHomePath) + + kclpkg, err := pkg.LoadKclPkg(testDir) + assert.Equal(t, err, nil) + + globalPkgPath, _ := env.GetAbsPkgPath() + kpmcli, err := NewKpmClient() + assert.Equal(t, err, nil) + res, err := kpmcli.ResolveDepsMetadataInJsonStr(kclpkg, true) + assert.Equal(t, err, nil) + + expectedDep := pkg.Dependencies{ + Deps: make(map[string]pkg.Dependency), + } + + expectedDep.Deps["konfig"] = pkg.Dependency{ + Name: "konfig", + FullName: "konfig_v0.0.1", + LocalFullPath: filepath.Join(globalPkgPath, "konfig_v0.0.1"), + } + + expectedDepStr, err := json.Marshal(expectedDep) + assert.Equal(t, err, nil) + + assert.Equal(t, res, string(expectedDepStr)) + + vendorDir := filepath.Join(testDir, "vendor") + if utils.DirExists(vendorDir) { + err = os.RemoveAll(vendorDir) + assert.Equal(t, err, nil) + } + kclpkg.SetVendorMode(true) + res, err = kpmcli.ResolveDepsMetadataInJsonStr(kclpkg, true) + assert.Equal(t, err, nil) + assert.Equal(t, utils.DirExists(vendorDir), true) + assert.Equal(t, utils.DirExists(filepath.Join(vendorDir, "konfig_v0.0.1")), true) + + expectedDep.Deps["konfig"] = pkg.Dependency{ + Name: "konfig", + FullName: "konfig_v0.0.1", + LocalFullPath: filepath.Join(vendorDir, "konfig_v0.0.1"), + } + + expectedDepStr, err = json.Marshal(expectedDep) + assert.Equal(t, err, nil) + + assert.Equal(t, res, string(expectedDepStr)) + if utils.DirExists(vendorDir) { + err = os.RemoveAll(vendorDir) + assert.Equal(t, err, nil) + } + + kclpkg, err = pkg.LoadKclPkg(testDir) + assert.Equal(t, err, nil) + kpmcli.homePath = "not_exist" + res, err = kpmcli.ResolveDepsMetadataInJsonStr(kclpkg, true) + assert.Equal(t, err, nil) + assert.Equal(t, utils.DirExists(vendorDir), false) + assert.Equal(t, utils.DirExists(filepath.Join(vendorDir, "konfig_v0.0.1")), false) + expectedStr := "{\"packages\":{\"konfig\":{\"name\":\"konfig\",\"manifest_path\":\"not_exist/konfig_v0.0.1\"}}}" + assert.Equal(t, res, expectedStr) + defer func() { + if r := os.RemoveAll(filepath.Join("not_exist", "konfig_v0.0.1")); r != nil { + err = fmt.Errorf("panic: %v", r) + } + }() +} + +func prepareKpmHomeInPath(path string) { + dirPath := filepath.Join(filepath.Join(path, ".kpm"), "config") + _ = os.MkdirAll(dirPath, 0755) + + filePath := filepath.Join(dirPath, "kpm.json") + + _ = os.WriteFile(filePath, []byte("{\"DefaultOciRegistry\":\"ghcr.io\",\"DefaultOciRepo\":\"awesome-kusion\"}"), 0644) +} + +func TestPkgWithInVendorMode(t *testing.T) { + testDir := getTestDir("test_pkg_with_vendor") + kcl1Path := filepath.Join(testDir, "kcl1") + + createKclPkg1 := func() { + assert.Equal(t, utils.DirExists(kcl1Path), false) + err := os.MkdirAll(kcl1Path, 0755) + assert.Equal(t, err, nil) + } + + defer func() { + if err := os.RemoveAll(kcl1Path); err != nil { + log.Printf("failed to close file: %v", err) + } + }() + + createKclPkg1() + + initOpts := opt.InitOptions{ + Name: "kcl1", + InitPath: kcl1Path, + } + kclPkg1 := pkg.NewKclPkg(&initOpts) + kpmcli, err := NewKpmClient() + assert.Equal(t, err, nil) + _, err = kpmcli.AddDepWithOpts(&kclPkg1, &opt.AddOptions{ + LocalPath: "localPath", + RegistryOpts: opt.RegistryOptions{ + Local: &opt.LocalOptions{ + Path: filepath.Join(testDir, "kcl2"), + }, + }, + }) + + assert.Equal(t, err, nil) + + // package the kcl1 into tar in vendor mode. + tarPath, err := kpmcli.PackagePkg(&kclPkg1, true) + assert.Equal(t, err, nil) + hasSubDir, err := hasSubdirInTar(tarPath, "vendor") + assert.Equal(t, err, nil) + assert.Equal(t, hasSubDir, true) + + // clean the kcl1 + err = os.RemoveAll(kcl1Path) + assert.Equal(t, err, nil) + + createKclPkg1() + // package the kcl1 into tar in non-vendor mode. + tarPath, err = kpmcli.PackagePkg(&kclPkg1, false) + assert.Equal(t, err, nil) + hasSubDir, err = hasSubdirInTar(tarPath, "vendor") + assert.Equal(t, err, nil) + assert.Equal(t, hasSubDir, false) +} + +// check if the tar file contains the subdir +func hasSubdirInTar(tarPath, subdir string) (bool, error) { + f, err := os.Open(tarPath) + if err != nil { + return false, err + } + defer f.Close() + + tr := tar.NewReader(f) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if hdr.Typeflag == tar.TypeDir && filepath.Base(hdr.Name) == subdir { + return true, nil + } + } + + return false, nil +} + +func TestNewKpmClient(t *testing.T) { + kpmcli, err := NewKpmClient() + assert.Equal(t, err, nil) + kpmhome, err := env.GetAbsPkgPath() + assert.Equal(t, err, nil) + assert.Equal(t, kpmcli.homePath, kpmhome) + assert.Equal(t, kpmcli.GetSettings().KpmConfFile, filepath.Join(kpmhome, ".kpm", "config", "kpm.json")) + assert.Equal(t, kpmcli.GetSettings().CredentialsFile, filepath.Join(kpmhome, ".kpm", "config", "config.json")) + assert.Equal(t, kpmcli.GetSettings().Conf.DefaultOciRepo, "kcl-lang") + assert.Equal(t, kpmcli.GetSettings().Conf.DefaultOciRegistry, "ghcr.io") + assert.Equal(t, kpmcli.GetSettings().Conf.DefaultOciPlainHttp, false) +} + +func TestParseOciOptionFromString(t *testing.T) { + kpmcli, err := NewKpmClient() + assert.Equal(t, err, nil) + oci_ref_with_tag := "test_oci_repo:test_oci_tag" + ociOption, err := kpmcli.ParseOciOptionFromString(oci_ref_with_tag, "test_tag") + assert.Equal(t, err, nil) + assert.Equal(t, ociOption.PkgName, "") + assert.Equal(t, ociOption.Reg, "ghcr.io") + assert.Equal(t, ociOption.Repo, "kcl-lang/test_oci_repo") + assert.Equal(t, ociOption.Tag, "test_oci_tag") + + oci_ref_without_tag := "test_oci_repo:test_oci_tag" + ociOption, err = kpmcli.ParseOciOptionFromString(oci_ref_without_tag, "test_tag") + assert.Equal(t, err, nil) + assert.Equal(t, ociOption.PkgName, "") + assert.Equal(t, ociOption.Reg, "ghcr.io") + assert.Equal(t, ociOption.Repo, "kcl-lang/test_oci_repo") + assert.Equal(t, ociOption.Tag, "test_oci_tag") + + oci_url_with_tag := "oci://test_reg/test_oci_repo" + ociOption, err = kpmcli.ParseOciOptionFromString(oci_url_with_tag, "test_tag") + assert.Equal(t, err, nil) + assert.Equal(t, ociOption.PkgName, "") + assert.Equal(t, ociOption.Reg, "test_reg") + assert.Equal(t, ociOption.Repo, "/test_oci_repo") + assert.Equal(t, ociOption.Tag, "test_tag") +} diff --git a/pkg/client/test_data/expected/kcl.mod b/pkg/client/test_data/expected/kcl.mod new file mode 100644 index 00000000..72026f01 --- /dev/null +++ b/pkg/client/test_data/expected/kcl.mod @@ -0,0 +1,8 @@ +[package] +name = "test_add_deps" +edition = "0.0.1" +version = "0.0.1" + +[dependencies] +oci_name = "test_tag" +name = { git = "test_url", tag = "test_tag" } diff --git a/pkg/package/test_data/test_data_add_deps/kcl.mod.lock b/pkg/client/test_data/expected/kcl.mod.lock similarity index 100% rename from pkg/package/test_data/test_data_add_deps/kcl.mod.lock rename to pkg/client/test_data/expected/kcl.mod.lock diff --git a/pkg/client/test_data/expected/kcl.mod.reverse.lock b/pkg/client/test_data/expected/kcl.mod.reverse.lock new file mode 100644 index 00000000..cebf047c --- /dev/null +++ b/pkg/client/test_data/expected/kcl.mod.reverse.lock @@ -0,0 +1,16 @@ +[dependencies] + [dependencies.test] + name = "name" + full_name = "test_version" + version = "test_version" + sum = "test_sum" + url = "test_url" + git_tag = "test_tag" + [dependencies.oci_test] + name = "oci_name" + full_name = "test_version" + version = "test_version" + sum = "test_sum" + reg = "test_reg" + repo = "test_repo" + oci_tag = "test_tag" diff --git a/pkg/package/test_data/test_data_add_deps/kcl.mod b/pkg/client/test_data/expected/kcl.reverse.mod similarity index 100% rename from pkg/package/test_data/test_data_add_deps/kcl.mod rename to pkg/client/test_data/expected/kcl.reverse.mod diff --git a/pkg/package/test_data/resolve_deps/kpm_home/kcl1/kcl.mod b/pkg/client/test_data/resolve_deps/kpm_home/kcl1/kcl.mod similarity index 100% rename from pkg/package/test_data/resolve_deps/kpm_home/kcl1/kcl.mod rename to pkg/client/test_data/resolve_deps/kpm_home/kcl1/kcl.mod diff --git a/pkg/package/test_data/resolve_deps/kpm_home/kcl1/kcl.mod.lock b/pkg/client/test_data/resolve_deps/kpm_home/kcl1/kcl.mod.lock similarity index 100% rename from pkg/package/test_data/resolve_deps/kpm_home/kcl1/kcl.mod.lock rename to pkg/client/test_data/resolve_deps/kpm_home/kcl1/kcl.mod.lock diff --git a/pkg/package/test_data/resolve_deps/kpm_home/kcl1/main.k b/pkg/client/test_data/resolve_deps/kpm_home/kcl1/main.k similarity index 100% rename from pkg/package/test_data/resolve_deps/kpm_home/kcl1/main.k rename to pkg/client/test_data/resolve_deps/kpm_home/kcl1/main.k diff --git a/pkg/package/test_data/resolve_deps/kpm_home/kcl2/kcl.mod b/pkg/client/test_data/resolve_deps/kpm_home/kcl2/kcl.mod similarity index 100% rename from pkg/package/test_data/resolve_deps/kpm_home/kcl2/kcl.mod rename to pkg/client/test_data/resolve_deps/kpm_home/kcl2/kcl.mod diff --git a/pkg/package/test_data/resolve_deps/kpm_home/kcl2/kcl.mod.lock b/pkg/client/test_data/resolve_deps/kpm_home/kcl2/kcl.mod.lock similarity index 100% rename from pkg/package/test_data/resolve_deps/kpm_home/kcl2/kcl.mod.lock rename to pkg/client/test_data/resolve_deps/kpm_home/kcl2/kcl.mod.lock diff --git a/pkg/package/test_data/resolve_deps/kpm_home/kcl2/main.k b/pkg/client/test_data/resolve_deps/kpm_home/kcl2/main.k similarity index 100% rename from pkg/package/test_data/resolve_deps/kpm_home/kcl2/main.k rename to pkg/client/test_data/resolve_deps/kpm_home/kcl2/main.k diff --git a/pkg/package/test_data/resolve_deps/my_kcl_compile/main.k b/pkg/client/test_data/resolve_deps/my_kcl_compile/main.k similarity index 100% rename from pkg/package/test_data/resolve_deps/my_kcl_compile/main.k rename to pkg/client/test_data/resolve_deps/my_kcl_compile/main.k diff --git a/pkg/package/test_data/resolve_metadata/kcl.mod b/pkg/client/test_data/resolve_metadata/kcl.mod similarity index 100% rename from pkg/package/test_data/resolve_metadata/kcl.mod rename to pkg/client/test_data/resolve_metadata/kcl.mod diff --git a/pkg/package/test_data/resolve_metadata/kcl.mod.lock b/pkg/client/test_data/resolve_metadata/kcl.mod.lock similarity index 100% rename from pkg/package/test_data/resolve_metadata/kcl.mod.lock rename to pkg/client/test_data/resolve_metadata/kcl.mod.lock diff --git a/pkg/package/test_data/tar_kcl_pkg/kcl.mod b/pkg/client/test_data/tar_kcl_pkg/kcl.mod similarity index 100% rename from pkg/package/test_data/tar_kcl_pkg/kcl.mod rename to pkg/client/test_data/tar_kcl_pkg/kcl.mod diff --git a/pkg/package/test_data/tar_kcl_pkg/kcl.mod.lock b/pkg/client/test_data/tar_kcl_pkg/kcl.mod.lock similarity index 100% rename from pkg/package/test_data/tar_kcl_pkg/kcl.mod.lock rename to pkg/client/test_data/tar_kcl_pkg/kcl.mod.lock diff --git a/pkg/client/test_data/test_data_add_deps/kcl.mod b/pkg/client/test_data/test_data_add_deps/kcl.mod new file mode 100644 index 00000000..72026f01 --- /dev/null +++ b/pkg/client/test_data/test_data_add_deps/kcl.mod @@ -0,0 +1,8 @@ +[package] +name = "test_add_deps" +edition = "0.0.1" +version = "0.0.1" + +[dependencies] +oci_name = "test_tag" +name = { git = "test_url", tag = "test_tag" } diff --git a/pkg/client/test_data/test_data_add_deps/kcl.mod.lock b/pkg/client/test_data/test_data_add_deps/kcl.mod.lock new file mode 100644 index 00000000..05d1a2ce --- /dev/null +++ b/pkg/client/test_data/test_data_add_deps/kcl.mod.lock @@ -0,0 +1,16 @@ +[dependencies] + [dependencies.oci_test] + name = "oci_name" + full_name = "test_version" + version = "test_version" + sum = "test_sum" + reg = "test_reg" + repo = "test_repo" + oci_tag = "test_tag" + [dependencies.test] + name = "name" + full_name = "test_version" + version = "test_version" + sum = "test_sum" + url = "test_url" + git_tag = "test_tag" diff --git a/pkg/package/test_data/test_init_empty_mod/main.k b/pkg/client/test_data/test_data_add_deps/main.k similarity index 100% rename from pkg/package/test_data/test_init_empty_mod/main.k rename to pkg/client/test_data/test_data_add_deps/main.k diff --git a/pkg/client/test_data/test_init_empty_mod/kcl.mod b/pkg/client/test_data/test_init_empty_mod/kcl.mod new file mode 100644 index 00000000..202d01bc --- /dev/null +++ b/pkg/client/test_data/test_init_empty_mod/kcl.mod @@ -0,0 +1,5 @@ +[package] +name = "test_name" +edition = "0.0.1" +version = "0.0.1" + diff --git a/pkg/package/test_data/test_init_empty_mod/kcl.mod.lock b/pkg/client/test_data/test_init_empty_mod/kcl.mod.lock similarity index 100% rename from pkg/package/test_data/test_init_empty_mod/kcl.mod.lock rename to pkg/client/test_data/test_init_empty_mod/kcl.mod.lock diff --git a/pkg/package/test_data/test_pkg_with_vendor/kcl2/main.k b/pkg/client/test_data/test_init_empty_mod/main.k similarity index 100% rename from pkg/package/test_data/test_pkg_with_vendor/kcl2/main.k rename to pkg/client/test_data/test_init_empty_mod/main.k diff --git a/pkg/package/test_data/test_pkg_with_vendor/kcl2/kcl.mod b/pkg/client/test_data/test_pkg_with_vendor/kcl2/kcl.mod similarity index 100% rename from pkg/package/test_data/test_pkg_with_vendor/kcl2/kcl.mod rename to pkg/client/test_data/test_pkg_with_vendor/kcl2/kcl.mod diff --git a/pkg/package/test_data/test_pkg_with_vendor/kcl2/kcl.mod.lock b/pkg/client/test_data/test_pkg_with_vendor/kcl2/kcl.mod.lock similarity index 100% rename from pkg/package/test_data/test_pkg_with_vendor/kcl2/kcl.mod.lock rename to pkg/client/test_data/test_pkg_with_vendor/kcl2/kcl.mod.lock diff --git a/pkg/client/test_data/test_pkg_with_vendor/kcl2/main.k b/pkg/client/test_data/test_pkg_with_vendor/kcl2/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/test_pkg_with_vendor/kcl2/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/cmd/cmd_add.go b/pkg/cmd/cmd_add.go index 3fa7c770..e5e5aa82 100644 --- a/pkg/cmd/cmd_add.go +++ b/pkg/cmd/cmd_add.go @@ -9,16 +9,16 @@ import ( "strings" "github.com/urfave/cli/v2" + "kcl-lang.io/kpm/pkg/client" "kcl-lang.io/kpm/pkg/env" "kcl-lang.io/kpm/pkg/errors" "kcl-lang.io/kpm/pkg/opt" pkg "kcl-lang.io/kpm/pkg/package" "kcl-lang.io/kpm/pkg/reporter" - "kcl-lang.io/kpm/pkg/settings" ) // NewAddCmd new a Command for `kpm add`. -func NewAddCmd() *cli.Command { +func NewAddCmd(kpmcli *client.KpmClient) *cli.Command { return &cli.Command{ Hidden: false, Name: "add", @@ -35,27 +35,21 @@ func NewAddCmd() *cli.Command { }, Action: func(c *cli.Context) error { - return KpmAdd(c) + return KpmAdd(c, kpmcli) }, } } -func KpmAdd(c *cli.Context) error { - // 1. get settings from the global config file. - settings := settings.GetSettings() - if settings.ErrorEvent != (*reporter.KpmEvent)(nil) { - return settings.ErrorEvent - } - - // 2. acquire the lock of the package cache. - err := settings.AcquirePackageCacheLock() +func KpmAdd(c *cli.Context, kpmcli *client.KpmClient) error { + // acquire the lock of the package cache. + err := kpmcli.AcquirePackageCacheLock() if err != nil { return err } defer func() { - // 3. release the lock of the package cache after the function returns. - releaseErr := settings.ReleasePackageCacheLock() + // release the lock of the package cache after the function returns. + releaseErr := kpmcli.ReleasePackageCacheLock() if releaseErr != nil && err == nil { err = releaseErr } @@ -82,7 +76,7 @@ func KpmAdd(c *cli.Context) error { return err } - addOpts, err := parseAddOptions(c, globalPkgPath) + addOpts, err := parseAddOptions(c, kpmcli, globalPkgPath) if err != nil { return err } @@ -105,7 +99,7 @@ func KpmAdd(c *cli.Context) error { return err } - err = kclPkg.AddDeps(addOpts) + _, err = kpmcli.AddDepWithOpts(kclPkg, addOpts) if err != nil { return err } @@ -125,7 +119,7 @@ func onlyOnceOption(c *cli.Context, name string) (*string, *reporter.KpmEvent) { } // parseAddOptions will parse the user cli inputs. -func parseAddOptions(c *cli.Context, localPath string) (*opt.AddOptions, error) { +func parseAddOptions(c *cli.Context, kpmcli *client.KpmClient, localPath string) (*opt.AddOptions, error) { // parse from 'kpm add -git https://xxx/xxx.git -tag v0.0.1'. if c.NArg() == 0 { gitOpts, err := parseGitRegistryOptions(c) @@ -143,7 +137,7 @@ func parseAddOptions(c *cli.Context, localPath string) (*opt.AddOptions, error) localPkg, err := parseLocalPathOptions(c) if err != (*reporter.KpmEvent)(nil) { // parse from 'kpm add xxx:0.0.1'. - ociReg, err := parseOciRegistryOptions(c) + ociReg, err := parseOciRegistryOptions(c, kpmcli) if err != nil { return nil, err } @@ -191,22 +185,17 @@ func parseGitRegistryOptions(c *cli.Context) (*opt.RegistryOptions, *reporter.Kp } // parseOciRegistryOptions will parse the oci registry information from user cli inputs. -func parseOciRegistryOptions(c *cli.Context) (*opt.RegistryOptions, error) { +func parseOciRegistryOptions(c *cli.Context, kpmcli *client.KpmClient) (*opt.RegistryOptions, error) { ociPkgRef := c.Args().First() name, version, err := parseOciPkgNameAndVersion(ociPkgRef) if err != nil { return nil, err } - settings := settings.GetSettings() - if settings.ErrorEvent != nil { - return nil, settings.ErrorEvent - } - return &opt.RegistryOptions{ Oci: &opt.OciOptions{ - Reg: settings.DefaultOciRegistry(), - Repo: settings.DefaultOciRepo(), + Reg: kpmcli.GetSettings().DefaultOciRegistry(), + Repo: kpmcli.GetSettings().DefaultOciRepo(), PkgName: name, Tag: version, }, diff --git a/pkg/cmd/cmd_init.go b/pkg/cmd/cmd_init.go index b4c640e9..9e6ba340 100644 --- a/pkg/cmd/cmd_init.go +++ b/pkg/cmd/cmd_init.go @@ -7,6 +7,7 @@ import ( "path/filepath" "github.com/urfave/cli/v2" + "kcl-lang.io/kpm/pkg/client" "kcl-lang.io/kpm/pkg/env" "kcl-lang.io/kpm/pkg/opt" pkg "kcl-lang.io/kpm/pkg/package" @@ -14,7 +15,7 @@ import ( ) // NewInitCmd new a Command for `kpm init`. -func NewInitCmd() *cli.Command { +func NewInitCmd(kpmcli *client.KpmClient) *cli.Command { return &cli.Command{ Hidden: false, Name: "init", @@ -66,8 +67,7 @@ func NewInitCmd() *cli.Command { return err } - err = kclPkg.InitEmptyPkg() - + err = kpmcli.InitEmptyPkg(&kclPkg) if err == nil { reporter.Report("kpm: package '", pkgName, "' init finished") } else { diff --git a/pkg/cmd/cmd_login.go b/pkg/cmd/cmd_login.go index 3cfc4c4c..a43e0f35 100644 --- a/pkg/cmd/cmd_login.go +++ b/pkg/cmd/cmd_login.go @@ -4,14 +4,13 @@ package cmd import ( "github.com/urfave/cli/v2" - "kcl-lang.io/kpm/pkg/oci" + "kcl-lang.io/kpm/pkg/client" "kcl-lang.io/kpm/pkg/reporter" - "kcl-lang.io/kpm/pkg/settings" "kcl-lang.io/kpm/pkg/utils" ) // NewLoginCmd new a Command for `kpm login`. -func NewLoginCmd(settings *settings.Settings) *cli.Command { +func NewLoginCmd(kpmcli *client.KpmClient) *cli.Command { return &cli.Command{ Hidden: false, Name: "login", @@ -42,7 +41,7 @@ func NewLoginCmd(settings *settings.Settings) *cli.Command { return err } - err = oci.Login(registry, username, password, settings) + err = kpmcli.LoginOci(registry, username, password) if err != nil { return err } diff --git a/pkg/cmd/cmd_logout.go b/pkg/cmd/cmd_logout.go index 7823943d..09a0646b 100644 --- a/pkg/cmd/cmd_logout.go +++ b/pkg/cmd/cmd_logout.go @@ -4,13 +4,12 @@ package cmd import ( "github.com/urfave/cli/v2" - "kcl-lang.io/kpm/pkg/oci" + "kcl-lang.io/kpm/pkg/client" "kcl-lang.io/kpm/pkg/reporter" - "kcl-lang.io/kpm/pkg/settings" ) // NewLogoutCmd new a Command for `kpm logout`. -func NewLogoutCmd(settings *settings.Settings) *cli.Command { +func NewLogoutCmd(kpmcli *client.KpmClient) *cli.Command { return &cli.Command{ Hidden: false, Name: "logout", @@ -20,9 +19,7 @@ func NewLogoutCmd(settings *settings.Settings) *cli.Command { reporter.Report("kpm: registry must be specified.") reporter.ExitWithReport("kpm: run 'kpm registry help' for more information.") } - registry := c.Args().First() - - err := oci.Logout(registry, settings) + err := kpmcli.LogoutOci(c.Args().First()) if err != nil { return err } diff --git a/pkg/cmd/cmd_metadata.go b/pkg/cmd/cmd_metadata.go index 6af85100..e179cf6a 100644 --- a/pkg/cmd/cmd_metadata.go +++ b/pkg/cmd/cmd_metadata.go @@ -7,6 +7,7 @@ import ( "os" "github.com/urfave/cli/v2" + "kcl-lang.io/kpm/pkg/client" "kcl-lang.io/kpm/pkg/env" "kcl-lang.io/kpm/pkg/errors" pkg "kcl-lang.io/kpm/pkg/package" @@ -14,7 +15,7 @@ import ( ) // NewMetadataCmd new a Command for `kpm metadata`. -func NewMetadataCmd() *cli.Command { +func NewMetadataCmd(kpmcli *client.KpmClient) *cli.Command { return &cli.Command{ Hidden: false, Name: "metadata", @@ -59,7 +60,8 @@ func NewMetadataCmd() *cli.Command { } autoUpdate := c.Bool(FLAG_UPDATE) - jsonStr, err := kclPkg.ResolveDepsMetadataInJsonStr(globalPkgPath, autoUpdate) + + jsonStr, err := kpmcli.ResolveDepsMetadataInJsonStr(kclPkg, autoUpdate) if err != nil { return err } diff --git a/pkg/cmd/cmd_pkg.go b/pkg/cmd/cmd_pkg.go index 231842b0..644dfdf9 100644 --- a/pkg/cmd/cmd_pkg.go +++ b/pkg/cmd/cmd_pkg.go @@ -7,6 +7,7 @@ import ( "path/filepath" "github.com/urfave/cli/v2" + "kcl-lang.io/kpm/pkg/client" "kcl-lang.io/kpm/pkg/errors" pkg "kcl-lang.io/kpm/pkg/package" "kcl-lang.io/kpm/pkg/reporter" @@ -14,7 +15,7 @@ import ( ) // NewPkgCmd new a Command for `kpm pkg`. -func NewPkgCmd() *cli.Command { +func NewPkgCmd(kpmcli *client.KpmClient) *cli.Command { return &cli.Command{ Hidden: false, Name: "pkg", @@ -60,8 +61,8 @@ func NewPkgCmd() *cli.Command { return errors.InternalBug } } - // The method for packaging kcl package should be a member method of KclPkg. - return kclPkg.PackageToTarball(filepath.Join(tarPath, kclPkg.GetPkgTarName()), c.Bool(FLAG_VENDOR)) + + return kpmcli.Package(kclPkg, filepath.Join(tarPath, kclPkg.GetPkgTarName()), c.Bool(FLAG_VENDOR)) }, } } diff --git a/pkg/cmd/cmd_pull.go b/pkg/cmd/cmd_pull.go index db03bcf9..0bde3219 100644 --- a/pkg/cmd/cmd_pull.go +++ b/pkg/cmd/cmd_pull.go @@ -3,22 +3,12 @@ package cmd import ( - "fmt" - "os" - "path/filepath" - "github.com/urfave/cli/v2" - "kcl-lang.io/kpm/pkg/api" - "kcl-lang.io/kpm/pkg/errors" - "kcl-lang.io/kpm/pkg/oci" - "kcl-lang.io/kpm/pkg/opt" - "kcl-lang.io/kpm/pkg/reporter" - "kcl-lang.io/kpm/pkg/settings" - "kcl-lang.io/kpm/pkg/utils" + "kcl-lang.io/kpm/pkg/client" ) // NewPullCmd new a Command for `kpm pull`. -func NewPullCmd() *cli.Command { +func NewPullCmd(kpmcli *client.KpmClient) *cli.Command { return &cli.Command{ Hidden: false, Name: "pull", @@ -30,108 +20,11 @@ func NewPullCmd() *cli.Command { }, }, Action: func(c *cli.Context) error { - return KpmPull(c) + return KpmPull(c, kpmcli) }, } } -func KpmPull(c *cli.Context) error { - tag := c.String(FLAG_TAG) - ociUrlOrPkgName := c.Args().Get(0) - localPath := c.Args().Get(1) - - if len(ociUrlOrPkgName) == 0 { - return reporter.NewErrorEvent( - reporter.UnKnownPullWhat, - errors.FailedPull, - "oci url or package name must be specified.", - ) - } - - if len(tag) == 0 { - reporter.ReportEventToStdout( - reporter.NewEvent( - reporter.PullingStarted, - fmt.Sprintf("start to pull '%s'.", ociUrlOrPkgName), - ), - ) - } else { - reporter.ReportEventToStdout( - reporter.NewEvent( - reporter.PullingStarted, - fmt.Sprintf("start to pull '%s' with tag '%s'.", ociUrlOrPkgName, tag), - ), - ) - } - - ociOpt, event := opt.ParseOciOptionFromOciUrl(ociUrlOrPkgName, tag) - var err error - if event != nil && (event.Type() == reporter.IsNotUrl || event.Type() == reporter.UrlSchemeNotOci) { - settings := settings.GetSettings() - if settings.ErrorEvent != nil { - return settings.ErrorEvent - } - - urlpath := utils.JoinPath(settings.DefaultOciRepo(), ociUrlOrPkgName) - - ociOpt, err = opt.ParseOciRef(urlpath) - if err != nil { - return err - } - } else if event != nil { - return event - } - - absPullPath, err := filepath.Abs(localPath) - if err != nil { - return reporter.NewErrorEvent(reporter.Bug, err) - } - - tmpDir, err := os.MkdirTemp("", "") - if err != nil { - return reporter.NewErrorEvent(reporter.Bug, err, fmt.Sprintf("failed to create temp dir '%s'.", tmpDir)) - } - - // clean the temp dir. - defer os.RemoveAll(tmpDir) - - localPath = ociOpt.AddStoragePathSuffix(tmpDir) - - // 2. Pull the tar. - err = oci.Pull(localPath, ociOpt.Reg, ociOpt.Repo, ociOpt.Tag) - - if err != (*reporter.KpmEvent)(nil) { - return err - } - - // 3. Get the (*.tar) file path. - tarPath := filepath.Join(localPath, api.KCL_PKG_TAR) - matches, err := filepath.Glob(tarPath) - if err != nil || len(matches) != 1 { - if err == nil { - err = errors.InvalidPkg - } - - return reporter.NewErrorEvent( - reporter.InvalidKclPkg, - err, - fmt.Sprintf("failed to find the kcl package tar from '%s'.", tarPath), - ) - } - - // 4. Untar the tar file. - storagePath := ociOpt.AddStoragePathSuffix(absPullPath) - err = utils.UnTarDir(matches[0], storagePath) - if err != nil { - return reporter.NewErrorEvent( - reporter.FailedUntarKclPkg, - err, - fmt.Sprintf("failed to untar the kcl package tar from '%s' into '%s'.", matches[0], storagePath), - ) - } - - reporter.ReportEventToStdout( - reporter.NewEvent(reporter.PullingFinished, fmt.Sprintf("pulled '%s' in '%s' successfully.", ociUrlOrPkgName, storagePath)), - ) - return nil +func KpmPull(c *cli.Context, kpmcli *client.KpmClient) error { + return kpmcli.PullFromOci(c.Args().Get(1), c.Args().Get(0), c.String(FLAG_TAG)) } diff --git a/pkg/cmd/cmd_push.go b/pkg/cmd/cmd_push.go index 81594dd4..fa3dd117 100644 --- a/pkg/cmd/cmd_push.go +++ b/pkg/cmd/cmd_push.go @@ -8,17 +8,17 @@ import ( "os" "github.com/urfave/cli/v2" + "kcl-lang.io/kpm/pkg/client" "kcl-lang.io/kpm/pkg/errors" "kcl-lang.io/kpm/pkg/oci" "kcl-lang.io/kpm/pkg/opt" pkg "kcl-lang.io/kpm/pkg/package" "kcl-lang.io/kpm/pkg/reporter" - "kcl-lang.io/kpm/pkg/settings" "kcl-lang.io/kpm/pkg/utils" ) // NewPushCmd new a Command for `kpm push`. -func NewPushCmd(settings *settings.Settings) *cli.Command { +func NewPushCmd(kpmcli *client.KpmClient) *cli.Command { return &cli.Command{ Hidden: false, Name: "push", @@ -37,12 +37,12 @@ func NewPushCmd(settings *settings.Settings) *cli.Command { }, }, Action: func(c *cli.Context) error { - return KpmPush(c, settings) + return KpmPush(c, kpmcli) }, } } -func KpmPush(c *cli.Context, settings *settings.Settings) error { +func KpmPush(c *cli.Context, kpmcli *client.KpmClient) error { localTarPath := c.String(FLAG_TAR_PATH) ociUrl := c.Args().First() @@ -51,10 +51,10 @@ func KpmPush(c *cli.Context, settings *settings.Settings) error { if len(localTarPath) == 0 { // If the tar package to be pushed is not specified, // the current kcl package is packaged into tar and pushed. - err = pushCurrentPackage(ociUrl, c.Bool(FLAG_VENDOR), settings) + err = pushCurrentPackage(ociUrl, c.Bool(FLAG_VENDOR), kpmcli) } else { // Else push the tar package specified. - err = pushTarPackage(ociUrl, localTarPath, c.Bool(FLAG_VENDOR), settings) + err = pushTarPackage(ociUrl, localTarPath, c.Bool(FLAG_VENDOR), kpmcli) } if err != nil { @@ -65,17 +65,13 @@ func KpmPush(c *cli.Context, settings *settings.Settings) error { } // genDefaultOciUrlForKclPkg will generate the default oci url from the current package. -func genDefaultOciUrlForKclPkg(pkg *pkg.KclPkg) (string, error) { - settings := settings.GetSettings() - if settings.ErrorEvent != nil { - return "", settings.ErrorEvent - } +func genDefaultOciUrlForKclPkg(pkg *pkg.KclPkg, kpmcli *client.KpmClient) (string, error) { - urlPath := utils.JoinPath(settings.DefaultOciRepo(), pkg.GetPkgName()) + urlPath := utils.JoinPath(kpmcli.GetSettings().DefaultOciRepo(), pkg.GetPkgName()) u := &url.URL{ Scheme: oci.OCI_SCHEME, - Host: settings.DefaultOciRegistry(), + Host: kpmcli.GetSettings().DefaultOciRegistry(), Path: urlPath, } @@ -83,7 +79,7 @@ func genDefaultOciUrlForKclPkg(pkg *pkg.KclPkg) (string, error) { } // pushCurrentPackage will push the current package to the oci registry. -func pushCurrentPackage(ociUrl string, vendorMode bool, settings *settings.Settings) error { +func pushCurrentPackage(ociUrl string, vendorMode bool, kpmcli *client.KpmClient) error { pwd, err := os.Getwd() if err != nil { @@ -99,12 +95,12 @@ func pushCurrentPackage(ociUrl string, vendorMode bool, settings *settings.Setti } // 2. push the package - return pushPackage(ociUrl, kclPkg, vendorMode, settings) + return pushPackage(ociUrl, kclPkg, vendorMode, kpmcli) } // pushTarPackage will push the kcl package in tarPath to the oci registry. // If the tar in 'tarPath' is not a kcl package tar, pushTarPackage will return an error. -func pushTarPackage(ociUrl, localTarPath string, vendorMode bool, settings *settings.Settings) error { +func pushTarPackage(ociUrl, localTarPath string, vendorMode bool, kpmcli *client.KpmClient) error { var kclPkg *pkg.KclPkg var err error @@ -125,7 +121,7 @@ func pushTarPackage(ociUrl, localTarPath string, vendorMode bool, settings *sett } // 2. push the package - return pushPackage(ociUrl, kclPkg, vendorMode, settings) + return pushPackage(ociUrl, kclPkg, vendorMode, kpmcli) } // pushPackage will push the kcl package to the oci registry. @@ -133,9 +129,9 @@ func pushTarPackage(ociUrl, localTarPath string, vendorMode bool, settings *sett // 2. If the oci url is not specified, generate the default oci url from the current package. // 3. Generate the OCI options from oci url and the version of current kcl package. // 4. Push the package to the oci registry. -func pushPackage(ociUrl string, kclPkg *pkg.KclPkg, vendorMode bool, settings *settings.Settings) error { - // 1. Package the current kcl package into default tar path. - tarPath, err := kclPkg.PackageCurrentPkgPath(vendorMode) +func pushPackage(ociUrl string, kclPkg *pkg.KclPkg, vendorMode bool, kpmcli *client.KpmClient) error { + + tarPath, err := kpmcli.PackagePkg(kclPkg, vendorMode) if err != nil { return err } @@ -152,7 +148,7 @@ func pushPackage(ociUrl string, kclPkg *pkg.KclPkg, vendorMode bool, settings *s // 2. If the oci url is not specified, generate the default oci url from the current package. if len(ociUrl) == 0 { - ociUrl, err = genDefaultOciUrlForKclPkg(kclPkg) + ociUrl, err = genDefaultOciUrlForKclPkg(kclPkg, kpmcli) if err != nil || len(ociUrl) == 0 { reporter.Report("kpm: failed to generate default oci url for current package.") reporter.Report("kpm: run 'kpm push help' for more information.") @@ -172,7 +168,7 @@ func pushPackage(ociUrl string, kclPkg *pkg.KclPkg, vendorMode bool, settings *s reporter.Report("kpm: package '" + kclPkg.GetPkgName() + "' will be pushed.") // 4. Push it. - err = oci.Push(tarPath, ociOpts.Reg, ociOpts.Repo, ociOpts.Tag, settings) + err = kpmcli.PushToOci(tarPath, ociOpts) if err != (*reporter.KpmEvent)(nil) { return err } diff --git a/pkg/cmd/cmd_push_test.go b/pkg/cmd/cmd_push_test.go index 521c5313..40e50082 100644 --- a/pkg/cmd/cmd_push_test.go +++ b/pkg/cmd/cmd_push_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "kcl-lang.io/kpm/pkg/client" pkg "kcl-lang.io/kpm/pkg/package" ) @@ -25,7 +26,9 @@ func TestGenDefaultOciUrlForKclPkg(t *testing.T) { pkgPath := getTestDir("test_gen_oci_url") kclPkg, err := pkg.LoadKclPkg(pkgPath) assert.Equal(t, err, nil) - url, err := genDefaultOciUrlForKclPkg(kclPkg) + kpmcli, err := client.NewKpmClient() + assert.Equal(t, err, nil) + url, err := genDefaultOciUrlForKclPkg(kclPkg, kpmcli) assert.Equal(t, err, nil) assert.Equal(t, url, "oci://ghcr.io/kcl-lang/test_gen_oci_url") } diff --git a/pkg/cmd/cmd_run.go b/pkg/cmd/cmd_run.go index ebd84c38..ac95ee3c 100644 --- a/pkg/cmd/cmd_run.go +++ b/pkg/cmd/cmd_run.go @@ -8,12 +8,13 @@ import ( "github.com/urfave/cli/v2" "kcl-lang.io/kcl-go/pkg/kcl" "kcl-lang.io/kpm/pkg/api" + "kcl-lang.io/kpm/pkg/client" "kcl-lang.io/kpm/pkg/opt" "kcl-lang.io/kpm/pkg/runner" ) // NewRunCmd new a Command for `kpm run`. -func NewRunCmd() *cli.Command { +func NewRunCmd(kpmcli *client.KpmClient) *cli.Command { return &cli.Command{ Hidden: false, Name: "run", @@ -72,12 +73,12 @@ func NewRunCmd() *cli.Command { }, }, Action: func(c *cli.Context) error { - return KpmRun(c) + return KpmRun(c, kpmcli) }, } } -func KpmRun(c *cli.Context) error { +func KpmRun(c *cli.Context, kpmcli *client.KpmClient) error { kclOpts := CompileOptionFromCli(c) runEntry, errEvent := runner.FindRunEntryFrom(c.Args().Slice()) if errEvent != nil { @@ -103,14 +104,14 @@ func KpmRun(c *cli.Context) error { compileResult, err = api.RunWithOpt(kclOpts) } else { // Else compile the kcl pacakge. - compileResult, err = api.RunPkgWithOpt(kclOpts) + compileResult, err = kpmcli.CompileWithOpts(kclOpts) } } else if runEntry.IsTar() { // 'kpm run' compile the package from the kcl pakcage tar. - compileResult, err = api.RunTarPkg(runEntry.PackageSource(), kclOpts) + compileResult, err = kpmcli.CompileTarPkg(runEntry.PackageSource(), kclOpts) } else { // 'kpm run' compile the package from the OCI reference or url. - compileResult, err = api.RunOciPkg(runEntry.PackageSource(), c.String(FLAG_TAG), kclOpts) + compileResult, err = kpmcli.CompileOciPkg(runEntry.PackageSource(), c.String(FLAG_TAG), kclOpts) } if err != nil { diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 42395d7f..8dc7e6c7 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -1,13 +1,17 @@ package constants const ( - KFilePathSuffix = ".k" - TarPathSuffix = ".tar" - OciScheme = "oci" - FileEntry = "file" - FileWithKclModEntry = "file_with_kcl_mod" - UrlEntry = "url" - RefEntry = "ref" - TarEntry = "tar" - KCL_MOD = "kcl.mod" + KFilePathSuffix = ".k" + TarPathSuffix = ".tar" + OciScheme = "oci" + FileEntry = "file" + FileWithKclModEntry = "file_with_kcl_mod" + UrlEntry = "url" + RefEntry = "ref" + TarEntry = "tar" + KCL_MOD = "kcl.mod" + OCI_SEPARATOR = ":" + KCL_PKG_TAR = "*.tar" + DEFAULT_KCL_FILE_NAME = "main.k" + DEFAULT_KCL_FILE_CONTENT = "The_first_kcl_program = 'Hello World!'" ) diff --git a/pkg/git/git.go b/pkg/git/git.go index fb1d1114..385a65ce 100644 --- a/pkg/git/git.go +++ b/pkg/git/git.go @@ -2,17 +2,17 @@ package git import ( "fmt" - "os" + "io" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" ) -/// Clone will clone from `repoURL` to `localPath` via git. -func Clone(repoURL string, tagName string, localPath string) (*git.Repository, error) { +// Clone will clone from `repoURL` to `localPath` via git. +func Clone(repoURL string, tagName string, localPath string, writer io.Writer) (*git.Repository, error) { repo, err := git.PlainClone(localPath, false, &git.CloneOptions{ URL: repoURL, - Progress: os.Stdout, + Progress: writer, ReferenceName: plumbing.ReferenceName(CreateTagRef(tagName)), }) return repo, err diff --git a/pkg/oci/oci.go b/pkg/oci/oci.go index 7a3fe421..6c5cb817 100644 --- a/pkg/oci/oci.go +++ b/pkg/oci/oci.go @@ -3,6 +3,7 @@ package oci import ( "context" "fmt" + "io" "path/filepath" v1 "github.com/opencontainers/image-spec/specs-go/v1" @@ -72,14 +73,23 @@ func Logout(hostname string, setting *settings.Settings) error { // OciClient is mainly responsible for interacting with OCI registry type OciClient struct { - repo *remote.Repository - ctx *context.Context + repo *remote.Repository + ctx *context.Context + logWriter io.Writer +} + +func (ociClient *OciClient) SetLogWriter(writer io.Writer) { + ociClient.logWriter = writer +} + +func (ociClient *OciClient) GetReference() string { + return ociClient.repo.Reference.String() } // NewOciClient will new an OciClient. // regName is the registry. e.g. ghcr.io or docker.io. // repoName is the repo name on registry. -func NewOciClient(regName, repoName string) (*OciClient, *reporter.KpmEvent) { +func NewOciClient(regName, repoName string, settings *settings.Settings) (*OciClient, error) { repoPath := utils.JoinPath(regName, repoName) repo, err := remote.NewRepository(repoPath) @@ -91,11 +101,6 @@ func NewOciClient(regName, repoName string) (*OciClient, *reporter.KpmEvent) { ) } ctx := context.Background() - - settings := settings.GetSettings() - if err != nil { - return nil, settings.ErrorEvent - } repo.PlainHTTP = settings.DefaultOciPlainHttp() // Login @@ -120,7 +125,7 @@ func NewOciClient(regName, repoName string) (*OciClient, *reporter.KpmEvent) { } // Pull will pull the oci artifacts from oci registry to local path. -func (ociClient *OciClient) Pull(localPath, tag string) *reporter.KpmEvent { +func (ociClient *OciClient) Pull(localPath, tag string) error { // Create a file store fs, err := file.New(localPath) if err != nil { @@ -142,7 +147,7 @@ func (ociClient *OciClient) Pull(localPath, tag string) *reporter.KpmEvent { } // TheLatestTag will return the latest tag of the kcl packages. -func (ociClient *OciClient) TheLatestTag() (string, *reporter.KpmEvent) { +func (ociClient *OciClient) TheLatestTag() (string, error) { var tagSelected string err := ociClient.repo.Tags(*ociClient.ctx, "", func(tags []string) error { @@ -224,18 +229,18 @@ func (ociClient *OciClient) Push(localPath, tag string) *reporter.KpmEvent { PackImageManifest: true, }) if err != nil { - return reporter.NewErrorEvent(reporter.FailedPush, err, fmt.Sprintf("Failed to pack package in '%s'", localPath)) + return reporter.NewErrorEvent(reporter.FailedPush, err, fmt.Sprintf("failed to pack package in '%s'", localPath)) } if err = fs.Tag(*ociClient.ctx, manifestDescriptor, tag); err != nil { - return reporter.NewErrorEvent(reporter.FailedPush, err, fmt.Sprintf("Failed to tag package with tag '%s'", tag)) + return reporter.NewErrorEvent(reporter.FailedPush, err, fmt.Sprintf("failed to tag package with tag '%s'", tag)) } // 3. Copy from the file store to the remote repository desc, err := oras.Copy(*ociClient.ctx, fs, tag, ociClient.repo, tag, oras.DefaultCopyOptions) if err != nil { - return reporter.NewErrorEvent(reporter.FailedPush, err, fmt.Sprintf("Failed to push '%s'", ociClient.repo.Reference)) + return reporter.NewErrorEvent(reporter.FailedPush, err, fmt.Sprintf("failed to push '%s'", ociClient.repo.Reference)) } reporter.Report("kpm: pushed [registry]", ociClient.repo.Reference) @@ -261,8 +266,8 @@ func loadCredential(hostName string, settings *settings.Settings) (*remoteauth.C } // Pull will pull the oci artifacts from oci registry to local path. -func Pull(localPath, hostName, repoName, tag string) *reporter.KpmEvent { - ociClient, err := NewOciClient(hostName, repoName) +func Pull(localPath, hostName, repoName, tag string, settings *settings.Settings) error { + ociClient, err := NewOciClient(hostName, repoName, settings) if err != nil { return err } @@ -290,9 +295,9 @@ func Pull(localPath, hostName, repoName, tag string) *reporter.KpmEvent { } // Push will push the oci artifacts to oci registry from local path -func Push(localPath, hostName, repoName, tag string, settings *settings.Settings) *reporter.KpmEvent { +func Push(localPath, hostName, repoName, tag string, settings *settings.Settings) error { // Create an oci client. - ociClient, err := NewOciClient(hostName, repoName) + ociClient, err := NewOciClient(hostName, repoName, settings) if err != nil { return err } diff --git a/pkg/opt/opt.go b/pkg/opt/opt.go index ebdf907b..8cd45731 100644 --- a/pkg/opt/opt.go +++ b/pkg/opt/opt.go @@ -3,15 +3,14 @@ package opt import ( + "io" "net/url" "os" "path/filepath" - "strings" "kcl-lang.io/kcl-go/pkg/kcl" "kcl-lang.io/kpm/pkg/errors" "kcl-lang.io/kpm/pkg/reporter" - "kcl-lang.io/kpm/pkg/settings" ) // CompileOptions is the input options of 'kpm run'. @@ -19,12 +18,15 @@ type CompileOptions struct { isVendor bool hasSettingsYaml bool entries []string + // Add a writer to control the output of the compiler. + writer io.Writer *kcl.Option } // DefaultCompileOptions returns a default CompileOptions. func DefaultCompileOptions() *CompileOptions { return &CompileOptions{ + writer: os.Stdout, Option: kcl.NewOption(), } } @@ -159,30 +161,6 @@ func (opts *LocalOptions) Validate() error { return nil } -const OCI_SEPARATOR = ":" - -// ParseOciOptionFromString will parser ':' into an 'OciOptions' with an OCI registry. -// the default OCI registry is 'docker.io'. -// if the 'ociUrl' is only '', ParseOciOptionFromString will take 'latest' as the default tag. -func ParseOciOptionFromString(oci string, tag string) (*OciOptions, error) { - ociOpt, event := ParseOciUrl(oci) - if event != nil && (event.Type() == reporter.IsNotUrl || event.Type() == reporter.UrlSchemeNotOci) { - ociOpt, err := ParseOciRef(oci) - if err != nil { - return nil, err - } - if len(tag) != 0 { - reporter.Report("kpm: kpm get version from oci reference ':'") - reporter.Report("kpm: arg '--tag' is invalid for oci reference") - } - return ociOpt, nil - } - - ociOpt.Tag = tag - - return ociOpt, nil -} - // ParseOciOptionFromOciUrl will parse oci url into an 'OciOptions'. // If the 'tag' is empty, ParseOciOptionFromOciUrl will take 'latest' as the default tag. func ParseOciOptionFromOciUrl(url, tag string) (*OciOptions, *reporter.KpmEvent) { @@ -194,30 +172,6 @@ func ParseOciOptionFromOciUrl(url, tag string) (*OciOptions, *reporter.KpmEvent) return ociOpt, nil } -// ParseOciRef will parse 'repoName:repoTag' into OciOptions, -// with default registry host 'docker.io'. -func ParseOciRef(ociRef string) (*OciOptions, error) { - oci_address := strings.Split(ociRef, OCI_SEPARATOR) - settings := settings.GetSettings() - if settings.ErrorEvent != nil { - return nil, settings.ErrorEvent - } - if len(oci_address) == 1 { - return &OciOptions{ - Reg: settings.DefaultOciRegistry(), - Repo: oci_address[0], - }, nil - } else if len(oci_address) == 2 { - return &OciOptions{ - Reg: settings.DefaultOciRegistry(), - Repo: oci_address[0], - Tag: oci_address[1], - }, nil - } else { - return nil, reporter.NewEvent(reporter.IsNotRef) - } -} - // ParseOciUrl will parse 'oci://hostName/repoName:repoTag' into OciOptions without tag. func ParseOciUrl(ociUrl string) (*OciOptions, *reporter.KpmEvent) { u, err := url.Parse(ociUrl) diff --git a/pkg/opt/opt_test.go b/pkg/opt/opt_test.go index 98d4a79e..a8571d76 100644 --- a/pkg/opt/opt_test.go +++ b/pkg/opt/opt_test.go @@ -9,32 +9,6 @@ import ( "kcl-lang.io/kcl-go/pkg/kcl" ) -func TestParseOciOptionFromString(t *testing.T) { - oci_ref_with_tag := "test_oci_repo:test_oci_tag" - ociOption, err := ParseOciOptionFromString(oci_ref_with_tag, "test_tag") - assert.Equal(t, err, nil) - assert.Equal(t, ociOption.PkgName, "") - assert.Equal(t, ociOption.Reg, "ghcr.io") - assert.Equal(t, ociOption.Repo, "test_oci_repo") - assert.Equal(t, ociOption.Tag, "test_oci_tag") - - oci_ref_without_tag := "test_oci_repo:test_oci_tag" - ociOption, err = ParseOciOptionFromString(oci_ref_without_tag, "test_tag") - assert.Equal(t, err, nil) - assert.Equal(t, ociOption.PkgName, "") - assert.Equal(t, ociOption.Reg, "ghcr.io") - assert.Equal(t, ociOption.Repo, "test_oci_repo") - assert.Equal(t, ociOption.Tag, "test_oci_tag") - - oci_url_with_tag := "oci://test_reg/test_oci_repo" - ociOption, err = ParseOciOptionFromString(oci_url_with_tag, "test_tag") - assert.Equal(t, err, nil) - assert.Equal(t, ociOption.PkgName, "") - assert.Equal(t, ociOption.Reg, "test_reg") - assert.Equal(t, ociOption.Repo, "/test_oci_repo") - assert.Equal(t, ociOption.Tag, "test_tag") -} - func TestWorkDirAsPkgPath(t *testing.T) { opts := DefaultCompileOptions() assert.Equal(t, opts.PkgPath(), "") diff --git a/pkg/package/modfile.go b/pkg/package/modfile.go index 4b2aeab9..3f65e427 100644 --- a/pkg/package/modfile.go +++ b/pkg/package/modfile.go @@ -8,8 +8,6 @@ import ( "github.com/BurntSushi/toml" "kcl-lang.io/kcl-go/pkg/kcl" - "kcl-lang.io/kpm/pkg/git" - "kcl-lang.io/kpm/pkg/oci" "kcl-lang.io/kpm/pkg/opt" "kcl-lang.io/kpm/pkg/reporter" "kcl-lang.io/kpm/pkg/settings" @@ -101,7 +99,7 @@ type Dependency struct { // GetLocalFullPath will get the local path of a dependency. func (dep *Dependency) GetLocalFullPath(rootpath string) string { - if dep.isFromLocal() { + if dep.IsFromLocal() { if filepath.IsAbs(dep.Source.Local.Path) { return dep.Source.Local.Path } @@ -110,7 +108,7 @@ func (dep *Dependency) GetLocalFullPath(rootpath string) string { return dep.LocalFullPath } -func (dep *Dependency) isFromLocal() bool { +func (dep *Dependency) IsFromLocal() bool { return dep.Source.Oci == nil && dep.Source.Git == nil && dep.Source.Local != nil } @@ -135,147 +133,6 @@ func (dep *Dependency) GenDepFullName() string { return dep.FullName } -// Download will download the kcl package to localPath from registory. -func (dep *Dependency) Download(localPath string) (*Dependency, error) { - if dep.Source.Git != nil { - _, err := dep.Source.Git.Download(localPath) - if err != nil { - return nil, err - } - dep.Version = dep.Source.Git.Tag - dep.LocalFullPath = localPath - dep.FullName = dep.GenDepFullName() - } - - if dep.Source.Oci != nil { - localPath, err := dep.Source.Oci.Download(localPath) - if err != nil { - return nil, err - } - dep.Version = dep.Source.Oci.Tag - dep.LocalFullPath = localPath - dep.FullName = dep.GenDepFullName() - } - - if dep.Source.Local != nil { - dep.LocalFullPath = dep.Source.Local.Path - } - - var err error - dep.Sum, err = utils.HashDir(dep.LocalFullPath) - if err != nil { - return nil, reporter.NewErrorEvent( - reporter.FailedHashPkg, - err, - fmt.Sprintf("failed to hash the kcl package '%s' in '%s'.", dep.Name, dep.LocalFullPath), - ) - } - - return dep, nil -} - -// Download will download the kcl package to localPath from git url. -func (dep *Git) Download(localPath string) (string, error) { - - reporter.ReportEventToStdout( - reporter.NewEvent( - reporter.DownloadingFromGit, - fmt.Sprintf("downloading '%s' with tag '%s'.", dep.Url, dep.Tag), - ), - ) - - _, err := git.Clone(dep.Url, dep.Tag, localPath) - - if err != nil { - return localPath, reporter.NewErrorEvent( - reporter.FailedCloneFromGit, - err, - fmt.Sprintf("failed to clone from '%s' into '%s'.", dep.Url, localPath), - ) - } - - return localPath, err -} - -func (dep *Oci) Download(localPath string) (string, error) { - - ociClient, err := oci.NewOciClient(dep.Reg, dep.Repo) - if err != nil { - return "", err - } - // Select the latest tag, if the tag, the user inputed, is empty. - var tagSelected string - if len(dep.Tag) == 0 { - tagSelected, err = ociClient.TheLatestTag() - if err != nil { - return "", err - } - - reporter.ReportEventToStdout( - reporter.NewEvent(reporter.SelectLatestVersion, "the lastest version '", tagSelected, "' will be added."), - ) - - dep.Tag = tagSelected - localPath = localPath + dep.Tag - } else { - tagSelected = dep.Tag - } - - reporter.ReportEventToStdout( - reporter.NewEvent( - reporter.DownloadingFromOCI, - fmt.Sprintf("downloading '%s:%s' from '%s/%s:%s'.", dep.Repo, tagSelected, dep.Reg, dep.Repo, tagSelected), - ), - ) - - // Pull the package with the tag. - err = ociClient.Pull(localPath, tagSelected) - if err != nil { - return "", err - } - - matches, finderr := filepath.Glob(filepath.Join(localPath, "*.tar")) - if finderr != nil || len(matches) != 1 { - if finderr == nil { - err = reporter.NewErrorEvent( - reporter.InvalidKclPkg, - err, - fmt.Sprintf("failed to find the kcl package tar from '%s'.", localPath), - ) - } - - return "", reporter.NewErrorEvent( - reporter.InvalidKclPkg, - err, - fmt.Sprintf("failed to find the kcl package tar from '%s'.", localPath), - ) - } - - tarPath := matches[0] - untarErr := utils.UnTarDir(tarPath, localPath) - if untarErr != nil { - return "", reporter.NewErrorEvent( - reporter.FailedUntarKclPkg, - untarErr, - fmt.Sprintf("failed to untar the kcl package tar from '%s' into '%s'.", tarPath, localPath), - ) - } - - // After untar the downloaded kcl package tar file, remove the tar file. - if utils.DirExists(tarPath) { - rmErr := os.Remove(tarPath) - if rmErr != nil { - return "", reporter.NewErrorEvent( - reporter.FailedUntarKclPkg, - err, - fmt.Sprintf("failed to untar the kcl package tar from '%s' into '%s'.", tarPath, localPath), - ) - } - } - - return localPath, nil -} - // Source is the package source from registry. type Source struct { *Git @@ -311,27 +168,6 @@ func ModLockFileExists(path string) (bool, error) { return utils.Exists(filepath.Join(path, MOD_LOCK_FILE)) } -// LoadModFile load the contents of the 'kcl.mod' file in the path. -func LoadModFile(homePath string) (*ModFile, error) { - modFile := new(ModFile) - err := modFile.loadModFile(filepath.Join(homePath, MOD_FILE)) - if err != nil { - return nil, err - } - - modFile.HomePath = homePath - - if modFile.Dependencies.Deps == nil { - modFile.Dependencies.Deps = make(map[string]Dependency) - } - err = modFile.FillDependenciesInfo() - if err != nil { - return nil, err - } - - return modFile, nil -} - // LoadLockDeps will load all dependencies from 'kcl.mod.lock'. func LoadLockDeps(homePath string) (*Dependencies, error) { deps := new(Dependencies) @@ -399,6 +235,27 @@ func (mod *ModFile) loadModFile(filepath string) error { return nil } +// LoadModFile load the contents of the 'kcl.mod' file in the path. +func LoadModFile(homePath string) (*ModFile, error) { + modFile := new(ModFile) + err := modFile.loadModFile(filepath.Join(homePath, MOD_FILE)) + if err != nil { + return nil, err + } + + modFile.HomePath = homePath + + if modFile.Dependencies.Deps == nil { + modFile.Dependencies.Deps = make(map[string]Dependency) + } + err = modFile.FillDependenciesInfo() + if err != nil { + return nil, err + } + + return modFile, nil +} + // Load the kcl.mod.lock file. func (deps *Dependencies) loadLockFile(filepath string) error { data, err := os.ReadFile(filepath) @@ -463,15 +320,15 @@ func ParseOpt(opt *opt.RegistryOptions) (*Dependency, error) { return nil, err } return &Dependency{ - Name: depPkg.modFile.Pkg.Name, - FullName: depPkg.modFile.Pkg.Name + "_" + depPkg.modFile.Pkg.Version, + Name: depPkg.ModFile.Pkg.Name, + FullName: depPkg.ModFile.Pkg.Name + "_" + depPkg.ModFile.Pkg.Version, LocalFullPath: opt.Local.Path, Source: Source{ Local: &Local{ Path: opt.Local.Path, }, }, - Version: depPkg.modFile.Pkg.Version, + Version: depPkg.ModFile.Pkg.Version, }, nil } diff --git a/pkg/package/modfile_test.go b/pkg/package/modfile_test.go index cd43f054..cf495f75 100644 --- a/pkg/package/modfile_test.go +++ b/pkg/package/modfile_test.go @@ -7,7 +7,6 @@ import ( "github.com/stretchr/testify/assert" "kcl-lang.io/kpm/pkg/opt" - "kcl-lang.io/kpm/pkg/settings" "kcl-lang.io/kpm/pkg/utils" ) @@ -154,81 +153,3 @@ func TestGetFilePath(t *testing.T) { assert.Equal(t, mfile.GetModFilePath(), filepath.Join(testPath, MOD_FILE)) assert.Equal(t, mfile.GetModLockFilePath(), filepath.Join(testPath, MOD_LOCK_FILE)) } - -// TestDownloadGit test download from oci registry. -func TestDownloadOci(t *testing.T) { - testPath := filepath.Join(getTestDir("download"), "k8s_1.27") - err := os.MkdirAll(testPath, 0755) - assert.Equal(t, err, nil) - - urlpath := utils.JoinPath(settings.DEFAULT_REPO, "k8s") - - depFromOci := Dependency{ - Name: "k8s", - Version: "1.27", - Source: Source{ - Oci: &Oci{ - Reg: settings.DEFAULT_REGISTRY, - Repo: urlpath, - Tag: "1.27", - }, - }, - } - - dep, err := depFromOci.Download(testPath) - assert.Equal(t, err, nil) - assert.Equal(t, dep.Name, "k8s") - assert.Equal(t, dep.FullName, "k8s_1.27") - assert.Equal(t, dep.Version, "1.27") - assert.Equal(t, dep.Sum, "xnYM1FWHAy3m+KcQMQb2rjZouTxumqYt6FGZpu2T4yM=") - assert.NotEqual(t, dep.Source.Oci, nil) - assert.Equal(t, dep.Source.Oci.Reg, settings.DEFAULT_REGISTRY) - assert.Equal(t, dep.Source.Oci.Repo, urlpath) - assert.Equal(t, dep.Source.Oci.Tag, "1.27") - assert.Equal(t, dep.LocalFullPath, testPath) - - // Check whether the tar downloaded by `kpm add` has been deleted. - assert.Equal(t, utils.DirExists(filepath.Join(testPath, "k8s_1.27.tar")), false) - - err = os.RemoveAll(getTestDir("download")) - assert.Equal(t, err, nil) -} - -// TestDownloadLatestOci tests the case that the version is empty. -func TestDownloadLatestOci(t *testing.T) { - testPath := filepath.Join(getTestDir("download"), "a_random_name") - err := os.MkdirAll(testPath, 0755) - assert.Equal(t, err, nil) - - urlpath := utils.JoinPath(settings.DEFAULT_REPO, "k8s") - - depFromOci := Dependency{ - Name: "k8s", - Version: "", - Source: Source{ - Oci: &Oci{ - Reg: settings.DEFAULT_REGISTRY, - Repo: urlpath, - Tag: "", - }, - }, - } - - dep, err := depFromOci.Download(testPath) - assert.Equal(t, dep.Name, "k8s") - assert.Equal(t, dep.FullName, "k8s_1.27") - assert.Equal(t, dep.Version, "1.27") - assert.Equal(t, dep.Sum, "xnYM1FWHAy3m+KcQMQb2rjZouTxumqYt6FGZpu2T4yM=") - assert.NotEqual(t, dep.Source.Oci, nil) - assert.Equal(t, dep.Source.Oci.Reg, settings.DEFAULT_REGISTRY) - assert.Equal(t, dep.Source.Oci.Repo, urlpath) - assert.Equal(t, dep.Source.Oci.Tag, "1.27") - assert.Equal(t, dep.LocalFullPath, testPath+"1.27") - assert.Equal(t, err, nil) - - // Check whether the tar downloaded by `kpm add` has been deleted. - assert.Equal(t, utils.DirExists(filepath.Join(testPath, "k8s_1.27.tar")), false) - - err = os.RemoveAll(getTestDir("download")) - assert.Equal(t, err, nil) -} diff --git a/pkg/package/package.go b/pkg/package/package.go index bb256eca..a37d288e 100644 --- a/pkg/package/package.go +++ b/pkg/package/package.go @@ -1,25 +1,19 @@ package pkg import ( - "encoding/json" "fmt" - "os" "path/filepath" - "reflect" "strings" - "github.com/otiai10/copy" "kcl-lang.io/kcl-go/pkg/kcl" - "kcl-lang.io/kpm/pkg/env" errors "kcl-lang.io/kpm/pkg/errors" "kcl-lang.io/kpm/pkg/opt" "kcl-lang.io/kpm/pkg/reporter" - "kcl-lang.io/kpm/pkg/runner" "kcl-lang.io/kpm/pkg/utils" ) type KclPkg struct { - modFile ModFile + ModFile ModFile HomePath string // The dependencies in the current kcl package are the dependencies of kcl.mod.lock, // not the dependencies in kcl.mod. @@ -28,23 +22,12 @@ type KclPkg struct { func NewKclPkg(opts *opt.InitOptions) KclPkg { return KclPkg{ - modFile: *NewModFile(opts), + ModFile: *NewModFile(opts), HomePath: opts.InitPath, Dependencies: Dependencies{Deps: make(map[string]Dependency)}, } } -// GetKclOpts will return the kcl options from kcl.mod. -func (kclPkg *KclPkg) GetKclOpts() *kcl.Option { - return kclPkg.modFile.Profiles.IntoKclOptions() -} - -// GetEntryKclFilesFromModFile will return the entry kcl files from kcl.mod. -func (kclPkg *KclPkg) GetEntryKclFilesFromModFile() []string { - return kclPkg.modFile.Profiles.Entries -} - -// Load the kcl package from directory containing kcl.mod and kcl.mod.lock file. func LoadKclPkg(pkgPath string) (*KclPkg, error) { modFile, err := LoadModFile(pkgPath) if err != nil { @@ -59,13 +42,12 @@ func LoadKclPkg(pkgPath string) (*KclPkg, error) { } return &KclPkg{ - modFile: *modFile, + ModFile: *modFile, HomePath: pkgPath, Dependencies: *deps, }, nil } -// LoadKclPkgFromTar will load a kcl package from a tar path. func LoadKclPkgFromTar(pkgTarPath string) (*KclPkg, error) { destDir := strings.TrimSuffix(pkgTarPath, filepath.Ext(pkgTarPath)) err := utils.UnTarDir(pkgTarPath, destDir) @@ -75,12 +57,22 @@ func LoadKclPkgFromTar(pkgTarPath string) (*KclPkg, error) { return LoadKclPkg(destDir) } +// GetKclOpts will return the kcl options from kcl.mod. +func (kclPkg *KclPkg) GetKclOpts() *kcl.Option { + return kclPkg.ModFile.Profiles.IntoKclOptions() +} + +// GetEntryKclFilesFromModFile will return the entry kcl files from kcl.mod. +func (kclPkg *KclPkg) GetEntryKclFilesFromModFile() []string { + return kclPkg.ModFile.Profiles.Entries +} + func (kclPkg *KclPkg) IsVendorMode() bool { - return kclPkg.modFile.VendorMode + return kclPkg.ModFile.VendorMode } func (kclPkg *KclPkg) SetVendorMode(vendorMode bool) { - kclPkg.modFile.VendorMode = vendorMode + kclPkg.ModFile.VendorMode = vendorMode } // Return the full vendor path. @@ -88,237 +80,10 @@ func (kclPkg *KclPkg) LocalVendorPath() string { return filepath.Join(kclPkg.HomePath, "vendor") } -// CompileWithEntryFile will call kcl compiler to compile the current kcl package and its dependent packages. -func (kclPkg *KclPkg) Compile(kpmHome string, kclvmCompiler *runner.Compiler) (*kcl.KCLResultList, error) { - - pkgMap, err := kclPkg.ResolveDeps(kpmHome) - if err != nil { - return nil, err - } - - // Fill the dependency path. - for dName, dPath := range pkgMap { - if !filepath.IsAbs(dPath) { - dPath = filepath.Join(kclPkg.HomePath, dPath) - } - kclvmCompiler.AddDepPath(dName, dPath) - } - - return kclvmCompiler.Run() -} - -// ResolveDeps will return a map between dependency name and its local path, -// and analyze the dependencies of the current kcl package, -// look for the package in the $KCL_PKG_PATH or kcl package vendor subdirectory, -// if find it, ResolveDeps will remember the local path of the dependency, -// if can’t find it, re-download the dependency and remember the local path. -func (kclPkg *KclPkg) ResolveDeps(kpmHome string) (map[string]string, error) { - - err := kclPkg.ResolveDepsMetadata(kpmHome, true) - if err != nil { - return nil, err - } - - var pkgMap map[string]string = make(map[string]string) - for name, d := range kclPkg.Dependencies.Deps { - pkgMap[name] = d.GetLocalFullPath(kclPkg.HomePath) - } - - return pkgMap, nil -} - -// ResolveDepsMetadata will calculate the local storage path of the external package, -// and check whether the package exists locally. -// If the package does not exist, it will re-download to the local. -func (kclPkg *KclPkg) ResolveDepsMetadata(kpmHome string, update bool) error { - var searchPath string - if kclPkg.IsVendorMode() { - // In the vendor mode, the search path is the vendor subdirectory of the current package. - err := kclPkg.VendorDeps(kpmHome) - if err != nil { - return err - } - searchPath = kclPkg.LocalVendorPath() - } else { - // Otherwise, the search path is the $KCL_PKG_PATH. - searchPath = kpmHome - } - - for name, d := range kclPkg.Dependencies.Deps { - searchFullPath := filepath.Join(searchPath, d.FullName) - if !update { - if utils.DirExists(searchFullPath) { - // Find it and update the local path of the dependency. - d.LocalFullPath = searchFullPath - kclPkg.Dependencies.Deps[name] = d - } - } else { - if utils.DirExists(searchFullPath) && check(d, searchFullPath) { - // Find it and update the local path of the dependency. - d.LocalFullPath = searchFullPath - kclPkg.Dependencies.Deps[name] = d - } else if d.isFromLocal() && !utils.DirExists(d.GetLocalFullPath(kclPkg.HomePath)) { - return reporter.NewErrorEvent(reporter.DependencyNotFound, fmt.Errorf("dependency '%s' not found in '%s'", d.Name, searchFullPath)) - } else if d.isFromLocal() && utils.DirExists(d.GetLocalFullPath(kclPkg.HomePath)) { - sum, err := utils.HashDir(d.GetLocalFullPath(kclPkg.HomePath)) - if err != nil { - return reporter.NewErrorEvent(reporter.CalSumFailed, err, fmt.Sprintf("failed to calculate checksum for '%s' in '%s'", d.Name, searchFullPath)) - } - d.Sum = sum - kclPkg.Dependencies.Deps[name] = d - } else { - // Otherwise, re-vendor it. - if kclPkg.IsVendorMode() { - err := kclPkg.VendorDeps(kpmHome) - if err != nil { - return err - } - } else { - // Or, re-download it. - err := kclPkg.DownloadDep(&d, kpmHome) - if err != nil { - return err - } - } - // After re-downloading or re-vendoring, - // re-resolving is required to update the dependent paths. - err := kclPkg.ResolveDepsMetadata(kpmHome, update) - if err != nil { - return err - } - return nil - } - } - } - - return nil -} - -// ResolveDepsMetadataInJsonStr will calculate the local storage path of the external package, -// and check whether the package exists locally. If the package does not exist, it will re-download to the local. -// Finally, the calculated metadata of the dependent packages is serialized into a json string and returned. -func (kclPkg *KclPkg) ResolveDepsMetadataInJsonStr(kpmHome string, update bool) (string, error) { - // 1. Calculate the dependency path, check whether the dependency exists - // and re-download the dependency that does not exist. - err := kclPkg.ResolveDepsMetadata(kpmHome, update) - if err != nil { - return "", err - } - - // 2. Serialize to JSON - jsonData, err := json.Marshal(kclPkg.Dependencies) - if err != nil { - return "", errors.InternalBug - } - - return string(jsonData), nil -} - -// InitEmptyModule inits an empty kcl module and create a default kcl.modfile. -func (kclPkg *KclPkg) InitEmptyPkg() error { - err := utils.CreateFileIfNotExist( - kclPkg.modFile.GetModFilePath(), - kclPkg.modFile.StoreModFile, - ) - if err != nil { - return err - } - - err = utils.CreateFileIfNotExist( - kclPkg.modFile.GetModLockFilePath(), - kclPkg.LockDepsVersion, - ) - if err != nil { - return err - } - - // create the default kcl program. - err = kclPkg.CreateDefaultKclProgram() - if err != nil { - return err - } - - return nil -} - -const DEFAULT_KCL_FILE_NAME = "main.k" -const DEFAULT_KCL_FILE_CONTENT = "The_first_kcl_program = 'Hello World!'" - -// CreateDefaultKclProgram will create a default kcl program "The_first_kcl_program = 'Hello World!'" in 'main.k'. -func (kclPkg *KclPkg) CreateDefaultKclProgram() error { - mainProgPath := filepath.Join(kclPkg.HomePath, DEFAULT_KCL_FILE_NAME) - if !utils.DirExists(mainProgPath) { - err := os.WriteFile(mainProgPath, []byte(DEFAULT_KCL_FILE_CONTENT), 0644) - if err != nil { - return err - } - } - return nil -} - -// AddDeps will add the dependencies to current kcl package and update kcl.mod and kcl.mod.lock. -func (kclPkg *KclPkg) AddDeps(opt *opt.AddOptions) error { - // 1. get the name and version of the repository from the input arguments. - d, err := ParseOpt(&opt.RegistryOpts) - if err != nil { - return err - } - - reporter.ReportEventToStdout(reporter.NewEvent(reporter.Adding, fmt.Sprintf("adding dependency '%s'.", d.Name))) - // 2. download the dependency to the local path. - err = kclPkg.DownloadDep(d, opt.LocalPath) - if err != nil { - return err - } - - // 3. update the kcl.mod and kcl.mod.lock. - err = kclPkg.UpdateModAndLockFile() - if err != nil { - return err - } - - succeedMsgInfo := d.Name - if len(d.Version) != 0 { - succeedMsgInfo = fmt.Sprintf("%s:%s", d.Name, d.Version) - } - - reporter.ReportEventToStdout( - reporter.NewEvent( - reporter.Adding, - fmt.Sprintf("add dependency '%s' successfully.", succeedMsgInfo), - ), - ) - return nil -} - -// DownloadDep will download the corresponding dependency. -func (kclPkg *KclPkg) DownloadDep(d *Dependency, localPath string) error { - - if !reflect.DeepEqual(kclPkg.modFile.Dependencies.Deps[d.Name], *d) { - // the dep passed on the cli is different from the kcl.mod. - kclPkg.modFile.Dependencies.Deps[d.Name] = *d - } - - // download all the dependencies. - changedDeps, err := getDeps(kclPkg.modFile.Dependencies, kclPkg.Dependencies, localPath) - - if err != nil { - return err - } - - // Update kcl.mod and kcl.mod.lock - for k, v := range changedDeps.Deps { - kclPkg.modFile.Dependencies.Deps[k] = v - kclPkg.Dependencies.Deps[k] = v - } - - return err -} - // updateModAndLockFile will update kcl.mod and kcl.mod.lock func (kclPkg *KclPkg) UpdateModAndLockFile() error { // Generate file kcl.mod. - err := kclPkg.modFile.StoreModFile() + err := kclPkg.ModFile.StoreModFile() if err != nil { return err } @@ -343,90 +108,6 @@ func (kclPkg *KclPkg) LockDepsVersion() error { return utils.StoreToFile(fullPath, lockToml) } -// getDeps will recursively download all dependencies to the 'localPath' directory, -// and return the dependencies that need to be updated to kcl.mod and kcl.mod.lock. -func getDeps(deps Dependencies, lockDeps Dependencies, localPath string) (*Dependencies, error) { - newDeps := Dependencies{ - Deps: make(map[string]Dependency), - } - - // Traverse all dependencies in kcl.mod - for _, d := range deps.Deps { - if len(d.Name) == 0 { - return nil, errors.InvalidDependency - } - - lockDep, present := lockDeps.Deps[d.Name] - - // Check if the sum of this dependency in kcl.mod.lock has been chanaged. - if present { - // If the dependent package does not exist locally, then method 'check' will return false. - if check(lockDep, filepath.Join(localPath, d.FullName)) { - newDeps.Deps[d.Name] = lockDep - continue - } - } - expectedSum := lockDeps.Deps[d.Name].Sum - // Clean the cache - if len(localPath) == 0 || len(d.FullName) == 0 { - return nil, errors.InternalBug - } - dir := filepath.Join(localPath, d.FullName) - os.RemoveAll(dir) - - // download dependencies - lockedDep, err := d.Download(dir) - if err != nil { - return nil, err - } - - if !lockedDep.isFromLocal() { - if expectedSum != "" && lockedDep.Sum != expectedSum && lockDep.FullName == d.FullName { - return nil, reporter.NewErrorEvent( - reporter.CheckSumMismatch, - errors.CheckSumMismatchError, - fmt.Sprintf("checksum for '%s' changed in lock file", lockedDep.Name), - ) - } - } - - // Update kcl.mod and kcl.mod.lock - newDeps.Deps[d.Name] = *lockedDep - lockDeps.Deps[d.Name] = *lockedDep - } - - // Recursively download the dependencies of the new dependencies. - for _, d := range newDeps.Deps { - // Load kcl.mod file of the new downloaded dependencies. - modfile, err := LoadModFile(filepath.Join(localPath, d.FullName)) - if len(d.LocalFullPath) != 0 { - modfile, err = LoadModFile(d.LocalFullPath) - } - - if err != nil { - if os.IsNotExist(err) { - continue - } - return nil, err - } - - // Download the dependencies. - nested, err := getDeps(modfile.Dependencies, lockDeps, localPath) - if err != nil { - return nil, err - } - - // Update kcl.mod. - for _, d := range nested.Deps { - if _, ok := newDeps.Deps[d.Name]; !ok { - newDeps.Deps[d.Name] = d - } - } - } - - return &newDeps, nil -} - // check sum for a Dependency. func check(dep Dependency, newDepPath string) bool { if dep.Sum == "" { @@ -442,30 +123,6 @@ func check(dep Dependency, newDepPath string) bool { return dep.Sum == sum } -// PackageCurrentPkg will package the current kcl package into the current path and return the tar path. -// And the tar will be named "-.tar" -// is the package name specified in kcl.mod. -// is the package version specified in kcl.mod. -func (kclPkg *KclPkg) PackageCurrentPkgPath(vendorMode bool) (string, error) { - globalPkgPath, err := env.GetAbsPkgPath() - if err != nil { - return "", err - } - - err = kclPkg.ValidateKpmHome(globalPkgPath) - if err != (*reporter.KpmEvent)(nil) { - return "", err - } - - err = kclPkg.PackageKclPkg(globalPkgPath, kclPkg.DefaultTarPath(), vendorMode) - - if err != nil { - reporter.ExitWithReport("kpm: failed to package pkg " + kclPkg.GetPkgName() + ".") - return "", err - } - return kclPkg.DefaultTarPath(), nil -} - const TAR_SUFFIX = ".tar" // DefaultTarPath will return "/-.tar" @@ -473,106 +130,6 @@ func (kclPkg *KclPkg) DefaultTarPath() string { return filepath.Join(kclPkg.HomePath, kclPkg.GetPkgTarName()) } -// PkgCurrentPackageIntoTarPath will package the current kcl package into 'tarPath'. -func (kclPkg *KclPkg) PackageToTarball(tarPath string, vendorMode bool) error { - - globalPkgPath, err := env.GetAbsPkgPath() - if err != nil { - return err - } - - err = kclPkg.ValidateKpmHome(globalPkgPath) - if err != (*reporter.KpmEvent)(nil) { - return err - } - - err = kclPkg.PackageKclPkg(globalPkgPath, tarPath, vendorMode) - - if err != nil { - reporter.ExitWithReport("kpm: failed to package pkg " + kclPkg.GetPkgName() + ".") - return err - } - return nil -} - -// PackageKclPkg will save all dependencies to the 'vendor' in current pacakge -// and package the current package into tar -func (kclPkg *KclPkg) PackageKclPkg(kpmHome string, tarPath string, vendorMode bool) error { - // Vendor all the dependencies into the current kcl package. - if vendorMode { - err := kclPkg.VendorDeps(kpmHome) - if err != nil { - return errors.FailedToVendorDependency - } - } - - // Tar the current kcl package into a "*.tar" file. - err := utils.TarDir(kclPkg.HomePath, tarPath) - if err != nil { - return errors.FailedToPackage - } - return nil -} - -// Vendor all dependencies to the 'vendor' in current pacakge. -func (kclPkg *KclPkg) VendorDeps(cachePath string) error { - // Mkdir the dir "vendor". - vendorPath := kclPkg.LocalVendorPath() - err := os.MkdirAll(vendorPath, 0755) - if err != nil { - return errors.InternalBug - } - - lockDeps := make([]Dependency, 0, len(kclPkg.Dependencies.Deps)) - - for _, d := range kclPkg.Dependencies.Deps { - lockDeps = append(lockDeps, d) - } - - // Traverse all dependencies in kcl.mod.lock. - for i := 0; i < len(lockDeps); i++ { - d := lockDeps[i] - if len(d.Name) == 0 { - return errors.InvalidDependency - } - vendorFullPath := filepath.Join(vendorPath, d.FullName) - // If the package already exists in the 'vendor', do nothing. - if utils.DirExists(vendorFullPath) && check(d, vendorFullPath) { - continue - } else { - // If not in the 'vendor', check the global cache. - cacheFullPath := filepath.Join(cachePath, d.FullName) - if utils.DirExists(cacheFullPath) && check(d, cacheFullPath) { - // If there is, copy it into the 'vendor' directory. - err := copy.Copy(cacheFullPath, vendorFullPath) - if err != nil { - return errors.FailedToVendorDependency - } - } else if utils.DirExists(d.GetLocalFullPath(kclPkg.HomePath)) && check(d, d.GetLocalFullPath(kclPkg.HomePath)) { - // If there is, copy it into the 'vendor' directory. - err := copy.Copy(d.GetLocalFullPath(kclPkg.HomePath), vendorFullPath) - if err != nil { - return errors.FailedToVendorDependency - } - } else { - // re-download if not. - err = kclPkg.DownloadDep(&d, cachePath) - if err != nil { - return errors.FailedToVendorDependency - } - // re-vendor again with new kcl.mod and kcl.mod.lock - err = kclPkg.VendorDeps(cachePath) - if err != nil { - return errors.FailedToVendorDependency - } - return nil - } - } - } - - return nil -} - // Verify that the environment variable KPM HOME is set correctly func (kclPkg *KclPkg) ValidateKpmHome(kpmHome string) *reporter.KpmEvent { if kclPkg.HomePath == kpmHome { @@ -586,27 +143,27 @@ func (kclPkg *KclPkg) ValidateKpmHome(kpmHome string) *reporter.KpmEvent { // is the name of package. // is the version of package func (kclPkg *KclPkg) GetPkgFullName() string { - return kclPkg.modFile.Pkg.Name + "-" + kclPkg.modFile.Pkg.Version + return kclPkg.ModFile.Pkg.Name + "-" + kclPkg.ModFile.Pkg.Version } // GetPkgName returns name of package. func (kclPkg *KclPkg) GetPkgName() string { - return kclPkg.modFile.Pkg.Name + return kclPkg.ModFile.Pkg.Name } // GetPkgTag returns version of package. func (kclPkg *KclPkg) GetPkgTag() string { - return kclPkg.modFile.Pkg.Version + return kclPkg.ModFile.Pkg.Version } // GetPkgEdition returns compile edition of package. func (kclPkg *KclPkg) GetPkgEdition() string { - return kclPkg.modFile.Pkg.Edition + return kclPkg.ModFile.Pkg.Edition } // GetPkgProfile returns the profile of package. func (kclPkg *KclPkg) GetPkgProfile() Profile { - return kclPkg.modFile.Profiles + return kclPkg.ModFile.Profiles } // GetPkgTarName returns the kcl package tar name "-v.tar" diff --git a/pkg/package/package_test.go b/pkg/package/package_test.go index 9ab51d9c..91fcbd46 100644 --- a/pkg/package/package_test.go +++ b/pkg/package/package_test.go @@ -1,11 +1,6 @@ package pkg import ( - "archive/tar" - "encoding/json" - "fmt" - "io" - "log" "os" "path/filepath" "testing" @@ -14,7 +9,6 @@ import ( "kcl-lang.io/kpm/pkg/env" "kcl-lang.io/kpm/pkg/opt" "kcl-lang.io/kpm/pkg/reporter" - "kcl-lang.io/kpm/pkg/runner" "kcl-lang.io/kpm/pkg/utils" ) @@ -51,131 +45,13 @@ func TestLoadKclPkg(t *testing.T) { if err != nil { t.Errorf("Failed to 'LoadKclPkg'.") } - assert.Equal(t, kclPkg.modFile.Pkg.Name, "test_name") - assert.Equal(t, kclPkg.modFile.Pkg.Version, "0.0.1") - assert.Equal(t, kclPkg.modFile.Pkg.Edition, "0.0.1") - assert.Equal(t, len(kclPkg.modFile.Dependencies.Deps), 0) + assert.Equal(t, kclPkg.ModFile.Pkg.Name, "test_name") + assert.Equal(t, kclPkg.ModFile.Pkg.Version, "0.0.1") + assert.Equal(t, kclPkg.ModFile.Pkg.Edition, "0.0.1") + assert.Equal(t, len(kclPkg.ModFile.Dependencies.Deps), 0) assert.Equal(t, len(kclPkg.Dependencies.Deps), 0) } -func TestInitEmptyPkg(t *testing.T) { - testDir := initTestDir("test_init_empty_mod") - kclPkg := NewKclPkg(&opt.InitOptions{Name: "test_name", InitPath: testDir}) - err := kclPkg.InitEmptyPkg() - if err != nil { - t.Errorf("Failed to 'InitEmptyPkg'.") - } - - testKclPkg, err := LoadKclPkg(testDir) - if err != nil { - t.Errorf("Failed to 'LoadKclPkg'.") - } - - assert.Equal(t, testKclPkg.modFile.Pkg.Name, "test_name") - assert.Equal(t, testKclPkg.modFile.Pkg.Version, "0.0.1") - assert.Equal(t, testKclPkg.modFile.Pkg.Edition, "0.0.1") -} - -func TestUpdateKclModAndLock(t *testing.T) { - testDir := initTestDir("test_data_add_deps") - // Init an empty package - kclPkg := NewKclPkg(&opt.InitOptions{ - Name: "test_add_deps", - InitPath: testDir, - }) - - _ = kclPkg.InitEmptyPkg() - - dep := Dependency{ - Name: "name", - FullName: "test_version", - Version: "test_version", - Sum: "test_sum", - Source: Source{ - Git: &Git{ - Url: "test_url", - Tag: "test_tag", - }, - }, - } - - oci_dep := Dependency{ - Name: "oci_name", - FullName: "test_version", - Version: "test_version", - Sum: "test_sum", - Source: Source{ - Oci: &Oci{ - Reg: "test_reg", - Repo: "test_repo", - Tag: "test_tag", - }, - }, - } - - kclPkg.Dependencies.Deps["oci_test"] = oci_dep - kclPkg.modFile.Dependencies.Deps["oci_test"] = oci_dep - - kclPkg.Dependencies.Deps["test"] = dep - kclPkg.modFile.Dependencies.Deps["test"] = dep - - err := kclPkg.modFile.StoreModFile() - - if err != nil { - t.Errorf("failed to LockDepsVersion.") - } - - err = kclPkg.LockDepsVersion() - - if err != nil { - t.Errorf("failed to LockDepsVersion.") - } - - expectDir := getTestDir("expected") - - if gotKclMod, err := os.ReadFile(filepath.Join(testDir, "kcl.mod")); os.IsNotExist(err) { - t.Errorf("failed to find kcl.mod.") - } else { - assert.Equal(t, len(kclPkg.Dependencies.Deps), 2) - assert.Equal(t, len(kclPkg.modFile.Deps), 2) - expectKclMod, _ := os.ReadFile(filepath.Join(expectDir, "kcl.mod")) - expectKclModReverse, _ := os.ReadFile(filepath.Join(expectDir, "kcl.reverse.mod")) - - gotKclModStr := utils.RmNewline(string(gotKclMod)) - fmt.Printf("gotKclModStr: '%v'\n", gotKclModStr) - expectKclModStr := utils.RmNewline(string(expectKclMod)) - fmt.Printf("expectKclModStr: '%v'\n", expectKclModStr) - expectKclModReverseStr := utils.RmNewline(string(expectKclModReverse)) - fmt.Printf("expectKclModReverseStr: '%v'\n", expectKclModReverseStr) - - assert.Equal(t, - (gotKclModStr == expectKclModStr || gotKclModStr == expectKclModReverseStr), - true, - ) - } - - if gotKclModLock, err := os.ReadFile(filepath.Join(testDir, "kcl.mod.lock")); os.IsNotExist(err) { - t.Errorf("failed to find kcl.mod.lock.") - } else { - assert.Equal(t, len(kclPkg.Dependencies.Deps), 2) - assert.Equal(t, len(kclPkg.modFile.Deps), 2) - expectKclModLock, _ := os.ReadFile(filepath.Join(expectDir, "kcl.mod.lock")) - expectKclModLockReverse, _ := os.ReadFile(filepath.Join(expectDir, "kcl.mod.reverse.lock")) - - gotKclModLockStr := utils.RmNewline(string(gotKclModLock)) - fmt.Printf("gotKclModLockStr: '%v'\n", gotKclModLockStr) - expectKclModLockStr := utils.RmNewline(string(expectKclModLock)) - fmt.Printf("expectKclModLockStr: '%v'\n", expectKclModLockStr) - expectKclModLockReverseStr := utils.RmNewline(string(expectKclModLockReverse)) - fmt.Printf("expectKclModLockReverseStr: '%v'\n", expectKclModLockReverseStr) - - assert.Equal(t, - (gotKclModLockStr == expectKclModLockStr) || (gotKclModLockStr == expectKclModLockReverseStr), - true, - ) - } -} - func TestCheck(t *testing.T) { testDir := getTestDir("test_check") dep := Dependency{ @@ -194,7 +70,7 @@ func TestCheck(t *testing.T) { func TestGetPkgName(t *testing.T) { kclPkg := KclPkg{ - modFile: ModFile{ + ModFile: ModFile{ Pkg: Package{ Name: "test", }, @@ -203,198 +79,6 @@ func TestGetPkgName(t *testing.T) { assert.Equal(t, kclPkg.GetPkgName(), "test") } -func TestVendorDeps(t *testing.T) { - testDir := getTestDir("resolve_deps") - kpm_home := filepath.Join(testDir, "kpm_home") - os.RemoveAll(filepath.Join(testDir, "my_kcl")) - kcl1Sum, _ := utils.HashDir(filepath.Join(kpm_home, "kcl1")) - kcl2Sum, _ := utils.HashDir(filepath.Join(kpm_home, "kcl2")) - - depKcl1 := Dependency{ - Name: "kcl1", - FullName: "kcl1", - Sum: kcl1Sum, - } - - depKcl2 := Dependency{ - Name: "kcl2", - FullName: "kcl2", - Sum: kcl2Sum, - } - - kclPkg := KclPkg{ - modFile: ModFile{ - HomePath: filepath.Join(testDir, "my_kcl"), - // Whether the current package uses the vendor mode - // In the vendor mode, kpm will look for the package in the vendor subdirectory - // in the current package directory. - VendorMode: false, - Dependencies: Dependencies{ - Deps: map[string]Dependency{ - "kcl1": depKcl1, - "kcl2": depKcl2, - }, - }, - }, - HomePath: filepath.Join(testDir, "my_kcl"), - // The dependencies in the current kcl package are the dependencies of kcl.mod.lock, - // not the dependencies in kcl.mod. - Dependencies: Dependencies{ - Deps: map[string]Dependency{ - "kcl1": depKcl1, - "kcl2": depKcl2, - }, - }, - } - - mykclVendorPath := filepath.Join(filepath.Join(testDir, "my_kcl"), "vendor") - assert.Equal(t, utils.DirExists(mykclVendorPath), false) - err := kclPkg.VendorDeps(kpm_home) - assert.Equal(t, err, nil) - assert.Equal(t, utils.DirExists(mykclVendorPath), true) - assert.Equal(t, utils.DirExists(filepath.Join(mykclVendorPath, "kcl1")), true) - assert.Equal(t, utils.DirExists(filepath.Join(mykclVendorPath, "kcl2")), true) - - maps, err := kclPkg.ResolveDeps(kpm_home) - assert.Equal(t, err, nil) - assert.Equal(t, len(maps), 2) - - os.RemoveAll(filepath.Join(testDir, "my_kcl")) -} - -func TestResolveDepsVendorMode(t *testing.T) { - testDir := getTestDir("resolve_deps") - kpm_home := filepath.Join(testDir, "kpm_home") - home_path := filepath.Join(testDir, "my_kcl_resolve_deps_vendor_mode") - os.RemoveAll(home_path) - kcl1Sum, _ := utils.HashDir(filepath.Join(kpm_home, "kcl1")) - kcl2Sum, _ := utils.HashDir(filepath.Join(kpm_home, "kcl2")) - - depKcl1 := Dependency{ - Name: "kcl1", - FullName: "kcl1", - Sum: kcl1Sum, - } - - depKcl2 := Dependency{ - Name: "kcl2", - FullName: "kcl2", - Sum: kcl2Sum, - } - - kclPkg := KclPkg{ - modFile: ModFile{ - HomePath: home_path, - // Whether the current package uses the vendor mode - // In the vendor mode, kpm will look for the package in the vendor subdirectory - // in the current package directory. - VendorMode: true, - Dependencies: Dependencies{ - Deps: map[string]Dependency{ - "kcl1": depKcl1, - "kcl2": depKcl2, - }, - }, - }, - HomePath: home_path, - // The dependencies in the current kcl package are the dependencies of kcl.mod.lock, - // not the dependencies in kcl.mod. - Dependencies: Dependencies{ - Deps: map[string]Dependency{ - "kcl1": depKcl1, - "kcl2": depKcl2, - }, - }, - } - mySearchPath := filepath.Join(home_path, "vendor") - assert.Equal(t, utils.DirExists(mySearchPath), false) - - maps, err := kclPkg.ResolveDeps(kpm_home) - assert.Equal(t, err, nil) - assert.Equal(t, len(maps), 2) - checkDepsMapInSearchPath(t, depKcl1, mySearchPath, maps) - - kclPkg.SetVendorMode(false) - maps, err = kclPkg.ResolveDeps(kpm_home) - assert.Equal(t, err, nil) - assert.Equal(t, len(maps), 2) - checkDepsMapInSearchPath(t, depKcl1, kpm_home, maps) - - os.RemoveAll(home_path) -} - -func checkDepsMapInSearchPath(t *testing.T, dep Dependency, searchPath string, maps map[string]string) { - assert.Equal(t, maps[dep.Name], filepath.Join(searchPath, dep.FullName)) - assert.Equal(t, utils.DirExists(filepath.Join(searchPath, dep.FullName)), true) -} - -func TestCompileWithEntryFile(t *testing.T) { - testDir := getTestDir("resolve_deps") - kpm_home := filepath.Join(testDir, "kpm_home") - home_path := filepath.Join(testDir, "my_kcl_compile") - vendor_path := filepath.Join(home_path, "vendor") - entry_file := filepath.Join(home_path, "main.k") - os.RemoveAll(vendor_path) - - kcl1Sum, _ := utils.HashDir(filepath.Join(kpm_home, "kcl1")) - depKcl1 := Dependency{ - Name: "kcl1", - FullName: "kcl1", - Sum: kcl1Sum, - } - kcl2Sum, _ := utils.HashDir(filepath.Join(kpm_home, "kcl2")) - depKcl2 := Dependency{ - Name: "kcl2", - FullName: "kcl2", - Sum: kcl2Sum, - } - - kclPkg := KclPkg{ - modFile: ModFile{ - HomePath: home_path, - // Whether the current package uses the vendor mode - // In the vendor mode, kpm will look for the package in the vendor subdirectory - // in the current package directory. - VendorMode: true, - Dependencies: Dependencies{ - Deps: map[string]Dependency{ - "kcl1": depKcl1, - "kcl2": depKcl2, - }, - }, - }, - HomePath: home_path, - // The dependencies in the current kcl package are the dependencies of kcl.mod.lock, - // not the dependencies in kcl.mod. - Dependencies: Dependencies{ - Deps: map[string]Dependency{ - "kcl1": depKcl1, - "kcl2": depKcl2, - }, - }, - } - - assert.Equal(t, utils.DirExists(vendor_path), false) - - compiler := runner.DefaultCompiler() - compiler.AddKFile(entry_file) - result, err := kclPkg.Compile(kpm_home, compiler) - assert.Equal(t, utils.DirExists(filepath.Join(vendor_path, "kcl1")), true) - assert.Equal(t, utils.DirExists(filepath.Join(vendor_path, "kcl2")), true) - assert.Equal(t, err, nil) - assert.Equal(t, result.GetRawYamlResult(), "c1: 1\nc2: 2") - os.RemoveAll(vendor_path) - - kclPkg.SetVendorMode(false) - assert.Equal(t, utils.DirExists(vendor_path), false) - - result, err = kclPkg.Compile(kpm_home, compiler) - assert.Equal(t, utils.DirExists(vendor_path), false) - assert.Equal(t, err, nil) - assert.Equal(t, result.GetRawYamlResult(), "c1: 1\nc2: 2") - os.RemoveAll(vendor_path) -} - func TestValidateKpmHome(t *testing.T) { kclPkg := NewKclPkg(&opt.InitOptions{ Name: "test_name", @@ -408,26 +92,6 @@ func TestValidateKpmHome(t *testing.T) { os.Setenv(env.PKG_PATH, oldValue) } -func TestPackageCurrentPkgPath(t *testing.T) { - testDir := getTestDir("tar_kcl_pkg") - - kclPkg, err := LoadKclPkg(testDir) - assert.Equal(t, err, nil) - assert.Equal(t, kclPkg.GetPkgTag(), "0.0.1") - assert.Equal(t, kclPkg.GetPkgName(), "test_tar") - assert.Equal(t, kclPkg.GetPkgFullName(), "test_tar-0.0.1") - assert.Equal(t, kclPkg.GetPkgTarName(), "test_tar-0.0.1.tar") - - assert.Equal(t, utils.DirExists(filepath.Join(testDir, kclPkg.GetPkgTarName())), false) - - path, err := kclPkg.PackageCurrentPkgPath(true) - assert.Equal(t, err, nil) - assert.Equal(t, path, filepath.Join(testDir, kclPkg.GetPkgTarName())) - assert.Equal(t, utils.DirExists(filepath.Join(testDir, kclPkg.GetPkgTarName())), true) - err = os.RemoveAll(filepath.Join(testDir, kclPkg.GetPkgTarName())) - assert.Equal(t, err, nil) -} - func TestLoadKclPkgFromTar(t *testing.T) { testDir := getTestDir("load_kcl_tar") assert.Equal(t, utils.DirExists(filepath.Join(testDir, "kcl1-v0.0.3")), false) @@ -435,19 +99,19 @@ func TestLoadKclPkgFromTar(t *testing.T) { kclPkg, err := LoadKclPkgFromTar(filepath.Join(testDir, "kcl1-v0.0.3.tar")) assert.Equal(t, err, nil) assert.Equal(t, kclPkg.HomePath, filepath.Join(testDir, "kcl1-v0.0.3")) - assert.Equal(t, kclPkg.modFile.Pkg.Name, "kcl1") - assert.Equal(t, kclPkg.modFile.Pkg.Edition, "0.0.1") - assert.Equal(t, kclPkg.modFile.Pkg.Version, "0.0.3") + assert.Equal(t, kclPkg.ModFile.Pkg.Name, "kcl1") + assert.Equal(t, kclPkg.ModFile.Pkg.Edition, "0.0.1") + assert.Equal(t, kclPkg.ModFile.Pkg.Version, "0.0.3") - assert.Equal(t, len(kclPkg.modFile.Deps), 2) - assert.Equal(t, kclPkg.modFile.Deps["konfig"].Name, "konfig") - assert.Equal(t, kclPkg.modFile.Deps["konfig"].FullName, "konfig_v0.0.1") - assert.Equal(t, kclPkg.modFile.Deps["konfig"].Git.Url, "https://github.com/awesome-kusion/konfig.git") - assert.Equal(t, kclPkg.modFile.Deps["konfig"].Git.Tag, "v0.0.1") + assert.Equal(t, len(kclPkg.ModFile.Deps), 2) + assert.Equal(t, kclPkg.ModFile.Deps["konfig"].Name, "konfig") + assert.Equal(t, kclPkg.ModFile.Deps["konfig"].FullName, "konfig_v0.0.1") + assert.Equal(t, kclPkg.ModFile.Deps["konfig"].Git.Url, "https://github.com/awesome-kusion/konfig.git") + assert.Equal(t, kclPkg.ModFile.Deps["konfig"].Git.Tag, "v0.0.1") - assert.Equal(t, kclPkg.modFile.Deps["oci_konfig"].Name, "oci_konfig") - assert.Equal(t, kclPkg.modFile.Deps["oci_konfig"].FullName, "oci_konfig_0.0.1") - assert.Equal(t, kclPkg.modFile.Deps["oci_konfig"].Oci.Tag, "0.0.1") + assert.Equal(t, kclPkg.ModFile.Deps["oci_konfig"].Name, "oci_konfig") + assert.Equal(t, kclPkg.ModFile.Deps["oci_konfig"].FullName, "oci_konfig_0.0.1") + assert.Equal(t, kclPkg.ModFile.Deps["oci_konfig"].Oci.Tag, "0.0.1") assert.Equal(t, len(kclPkg.Deps), 2) assert.Equal(t, kclPkg.Deps["konfig"].Name, "konfig") @@ -472,159 +136,3 @@ func TestLoadKclPkgFromTar(t *testing.T) { err = os.RemoveAll(filepath.Join(testDir, "kcl1-v0.0.3")) assert.Equal(t, err, nil) } - -func prepareKpmHomeInPath(path string) { - dirPath := filepath.Join(filepath.Join(path, ".kpm"), "config") - _ = os.MkdirAll(dirPath, 0755) - - filePath := filepath.Join(dirPath, "kpm.json") - - _ = os.WriteFile(filePath, []byte("{\"DefaultOciRegistry\":\"ghcr.io\",\"DefaultOciRepo\":\"awesome-kusion\"}"), 0644) -} - -func TestResolveMetadataInJsonStr(t *testing.T) { - originalValue := os.Getenv(env.PKG_PATH) - defer os.Setenv(env.PKG_PATH, originalValue) - - testDir := getTestDir("resolve_metadata") - - testHomePath := filepath.Join(filepath.Dir(testDir), "test_home_path") - prepareKpmHomeInPath(testHomePath) - defer os.RemoveAll(testHomePath) - - os.Setenv(env.PKG_PATH, testHomePath) - - pkg, err := LoadKclPkg(testDir) - assert.Equal(t, err, nil) - - globalPkgPath, _ := env.GetAbsPkgPath() - res, err := pkg.ResolveDepsMetadataInJsonStr(globalPkgPath, true) - assert.Equal(t, err, nil) - - expectedDep := Dependencies{ - Deps: make(map[string]Dependency), - } - - expectedDep.Deps["konfig"] = Dependency{ - Name: "konfig", - FullName: "konfig_v0.0.1", - LocalFullPath: filepath.Join(globalPkgPath, "konfig_v0.0.1"), - } - - expectedDepStr, err := json.Marshal(expectedDep) - assert.Equal(t, err, nil) - - assert.Equal(t, res, string(expectedDepStr)) - - vendorDir := filepath.Join(testDir, "vendor") - if utils.DirExists(vendorDir) { - err = os.RemoveAll(vendorDir) - assert.Equal(t, err, nil) - } - pkg.SetVendorMode(true) - res, err = pkg.ResolveDepsMetadataInJsonStr(globalPkgPath, true) - assert.Equal(t, err, nil) - assert.Equal(t, utils.DirExists(vendorDir), true) - assert.Equal(t, utils.DirExists(filepath.Join(vendorDir, "konfig_v0.0.1")), true) - - expectedDep.Deps["konfig"] = Dependency{ - Name: "konfig", - FullName: "konfig_v0.0.1", - LocalFullPath: filepath.Join(vendorDir, "konfig_v0.0.1"), - } - - expectedDepStr, err = json.Marshal(expectedDep) - assert.Equal(t, err, nil) - - assert.Equal(t, res, string(expectedDepStr)) - if utils.DirExists(vendorDir) { - err = os.RemoveAll(vendorDir) - assert.Equal(t, err, nil) - } - - pkg, err = LoadKclPkg(testDir) - assert.Equal(t, err, nil) - res, err = pkg.ResolveDepsMetadataInJsonStr("not_exist", false) - assert.Equal(t, err, nil) - assert.Equal(t, utils.DirExists(vendorDir), false) - assert.Equal(t, utils.DirExists(filepath.Join(vendorDir, "konfig_v0.0.1")), false) - expectedStr := "{\"packages\":{\"konfig\":{\"name\":\"konfig\",\"manifest_path\":\"\"}}}" - assert.Equal(t, res, expectedStr) -} - -func TestPkgWithInVendorMode(t *testing.T) { - testDir := getTestDir("test_pkg_with_vendor") - kcl1Path := filepath.Join(testDir, "kcl1") - - createKclPkg1 := func() { - assert.Equal(t, utils.DirExists(kcl1Path), false) - err := os.MkdirAll(kcl1Path, 0755) - assert.Equal(t, err, nil) - } - - defer func() { - if err := os.RemoveAll(kcl1Path); err != nil { - log.Printf("failed to close file: %v", err) - } - }() - - createKclPkg1() - - initOpts := opt.InitOptions{ - Name: "kcl1", - InitPath: kcl1Path, - } - kclPkg1 := NewKclPkg(&initOpts) - - err := kclPkg1.AddDeps(&opt.AddOptions{ - LocalPath: "localPath", - RegistryOpts: opt.RegistryOptions{ - Local: &opt.LocalOptions{ - Path: filepath.Join(testDir, "kcl2"), - }, - }, - }) - - assert.Equal(t, err, nil) - - // package the kcl1 into tar in vendor mode. - tarPath, err := kclPkg1.PackageCurrentPkgPath(true) - assert.Equal(t, err, nil) - hasSubDir, err := hasSubdirInTar(tarPath, "vendor") - assert.Equal(t, err, nil) - assert.Equal(t, hasSubDir, true) - - // clean the kcl1 - err = os.RemoveAll(kcl1Path) - assert.Equal(t, err, nil) - - createKclPkg1() - // package the kcl1 into tar in non-vendor mode. - tarPath, err = kclPkg1.PackageCurrentPkgPath(false) - assert.Equal(t, err, nil) - hasSubDir, err = hasSubdirInTar(tarPath, "vendor") - assert.Equal(t, err, nil) - assert.Equal(t, hasSubDir, false) -} - -// check if the tar file contains the subdir -func hasSubdirInTar(tarPath, subdir string) (bool, error) { - f, err := os.Open(tarPath) - if err != nil { - return false, err - } - defer f.Close() - - tr := tar.NewReader(f) - for { - hdr, err := tr.Next() - if err == io.EOF { - break - } - if hdr.Typeflag == tar.TypeDir && filepath.Base(hdr.Name) == subdir { - return true, nil - } - } - - return false, nil -} diff --git a/pkg/reporter/reporter.go b/pkg/reporter/reporter.go index 8df2c680..27d017fd 100644 --- a/pkg/reporter/reporter.go +++ b/pkg/reporter/reporter.go @@ -80,7 +80,9 @@ const ( PullingStarted PullingFinished Pulling + InvalidFlag Adding + WaitingLock IsNotUrl IsNotRef UrlSchemeNotOci diff --git a/pkg/settings/settings.go b/pkg/settings/settings.go index 12e52f3f..8b370cac 100644 --- a/pkg/settings/settings.go +++ b/pkg/settings/settings.go @@ -3,6 +3,7 @@ package settings import ( "encoding/json" "fmt" + "io" "os" "path/filepath" "strings" @@ -69,7 +70,7 @@ type Settings struct { } // AcquirePackageCacheLock will try to lock the 'package-cache' file. -func (settings *Settings) AcquirePackageCacheLock() error { +func (settings *Settings) AcquirePackageCacheLock(logWriter io.Writer) error { // if the 'package-cache' file is not initialized, this is an internal bug. if settings.PackageCacheLock == nil { return errors.InternalBug @@ -83,7 +84,7 @@ func (settings *Settings) AcquirePackageCacheLock() error { // if failed to lock the 'package-cache' file, wait until it is unlocked. if !locked { - reporter.Report("kpm: waiting for package-cache lock...") + reporter.ReportEventTo(reporter.NewEvent(reporter.WaitingLock, "waiting for package-cache lock..."), logWriter) for { // try to lock the 'package-cache' file locked, err = settings.PackageCacheLock.TryLock() diff --git a/pkg/settings/settings_test.go b/pkg/settings/settings_test.go index 3991e601..b787ab4a 100644 --- a/pkg/settings/settings_test.go +++ b/pkg/settings/settings_test.go @@ -124,7 +124,7 @@ func TestPackageCacheLock(t *testing.T) { // goroutine 1: append "goroutine 1: %d" to the list go func() { defer wg.Done() - err := settings.AcquirePackageCacheLock() + err := settings.AcquirePackageCacheLock(os.Stdout) fmt.Printf("1: locked.") fmt.Printf("err: %v\n", err) for i := 0; i < 10; i++ { @@ -138,7 +138,7 @@ func TestPackageCacheLock(t *testing.T) { // goroutine 2: append "goroutine 2: %d" to the list go func() { defer wg.Done() - err := settings.AcquirePackageCacheLock() + err := settings.AcquirePackageCacheLock(os.Stdout) fmt.Printf("2: locked.") fmt.Printf("err: %v\n", err) for i := 0; i < 10; i++ { diff --git a/pkg/api/test_data/test_check_tar_path/test.tar b/pkg/utils/test_data/test_check_tar_path/test.tar similarity index 100% rename from pkg/api/test_data/test_check_tar_path/test.tar rename to pkg/utils/test_data/test_check_tar_path/test.tar diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index aca6c492..9593070f 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -394,3 +394,36 @@ func IsTar(str string) bool { func IsKfile(str string) bool { return strings.HasSuffix(str, constants.KFilePathSuffix) } + +// CheckPackageSum will check whether the 'checkedSum' is equal +// to the hash of the package under 'localPath'. +func CheckPackageSum(checkedSum, localPath string) bool { + if checkedSum == "" { + return false + } + + sum, err := HashDir(localPath) + + if err != nil { + return false + } + + return checkedSum == sum +} + +// AbsTarPath checks whether path 'tarPath' exists and whether path 'tarPath' ends with '.tar' +// And after checking, absTarPath return the abs path for 'tarPath'. +func AbsTarPath(tarPath string) (string, error) { + absTarPath, err := filepath.Abs(tarPath) + if err != nil { + return "", errors.InternalBug + } + + if filepath.Ext(absTarPath) != ".tar" { + return "", errors.InvalidKclPacakgeTar + } else if !DirExists(absTarPath) { + return "", errors.KclPacakgeTarNotFound + } + + return absTarPath, nil +} diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go index 138edc3b..d88d63cb 100644 --- a/pkg/utils/utils_test.go +++ b/pkg/utils/utils_test.go @@ -155,3 +155,20 @@ func TestIsKfile(t *testing.T) { assert.Equal(t, IsKfile("invalid kfile"), false) assert.Equal(t, IsKfile("xxx.k"), true) } + +func TestAbsTarPath(t *testing.T) { + pkgPath := getTestDir("test_check_tar_path") + expectAbsTarPath, _ := filepath.Abs(filepath.Join(pkgPath, "test.tar")) + + abs, err := AbsTarPath(filepath.Join(pkgPath, "test.tar")) + assert.Equal(t, err, nil) + assert.Equal(t, abs, expectAbsTarPath) + + abs, err = AbsTarPath(filepath.Join(pkgPath, "no_exist.tar")) + assert.NotEqual(t, err, nil) + assert.Equal(t, abs, "") + + abs, err = AbsTarPath(filepath.Join(pkgPath, "invalid_tar")) + assert.NotEqual(t, err, nil) + assert.Equal(t, abs, "") +} diff --git a/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url/test_suite.stdout b/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url/test_suite.stdout index d1b76114..a441e30a 100644 --- a/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url/test_suite.stdout +++ b/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url/test_suite.stdout @@ -1,4 +1,4 @@ kpm: start to pull 'oci://localhost:5001/test/k8s'. kpm: the lastest version '1.27' will be pulled. -kpm: pulling '/test/k8s:1.27' from 'localhost:5001/test/k8s'. -kpm: pulled 'oci://localhost:5001/test/k8s' in '/localhost:5001/test/k8s' successfully. +kpm: pulling 'localhost:5001/test/k8s'. +kpm: pulled 'oci://localhost:5001/test/k8s' in '/localhost:5001/test/k8s' successfully. \ No newline at end of file diff --git a/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url_tag/test_suite.stdout b/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url_tag/test_suite.stdout index 9e363eb1..e1dadb9d 100644 --- a/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url_tag/test_suite.stdout +++ b/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url_tag/test_suite.stdout @@ -1,3 +1,3 @@ kpm: start to pull 'oci://localhost:5001/test/k8s' with tag '1.14'. -kpm: pulling '/test/k8s:1.14' from 'localhost:5001/test/k8s'. -kpm: pulled 'oci://localhost:5001/test/k8s' in '/localhost:5001/test/k8s/1.14' successfully. +kpm: pulling 'localhost:5001/test/k8s'. +kpm: pulled 'oci://localhost:5001/test/k8s' in '/localhost:5001/test/k8s/1.14' successfully. \ No newline at end of file diff --git a/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_pkg_name/test_suite.stdout b/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_pkg_name/test_suite.stdout index 87ddcde3..2415ce8d 100644 --- a/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_pkg_name/test_suite.stdout +++ b/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_pkg_name/test_suite.stdout @@ -1,4 +1,4 @@ kpm: start to pull 'k8s'. kpm: the lastest version '1.27' will be pulled. -kpm: pulling 'test/k8s:1.27' from 'localhost:5001/test/k8s'. -kpm: pulled 'k8s' in '/localhost:5001/test/k8s' successfully. +kpm: pulling 'localhost:5001/test/k8s'. +kpm: pulled 'k8s' in '/localhost:5001/test/k8s' successfully. \ No newline at end of file diff --git a/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_pkg_name_tag/test_suite.stdout b/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_pkg_name_tag/test_suite.stdout index 1d740935..da84370f 100644 --- a/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_pkg_name_tag/test_suite.stdout +++ b/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_pkg_name_tag/test_suite.stdout @@ -1,3 +1,3 @@ kpm: start to pull 'k8s:1.27'. -kpm: pulling 'test/k8s:1.27' from 'localhost:5001/test/k8s'. -kpm: pulled 'k8s:1.27' in '/localhost:5001/test/k8s/1.27' successfully. +kpm: pulling 'localhost:5001/test/k8s'. +kpm: pulled 'k8s:1.27' in '/localhost:5001/test/k8s/1.27' successfully. \ No newline at end of file diff --git a/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_ref/test_suite.stderr b/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_ref/test_suite.stderr index 320cc82b..3509ad56 100644 --- a/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_ref/test_suite.stderr +++ b/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_ref/test_suite.stderr @@ -1 +1,2 @@ -kpm: failed to get package with 'invalid_tag' from 'localhost:5001/invalid_oci_repo'. +kpm: failed to get package with 'invalid_tag' from 'localhost:5001/test/invalid_oci_repo'. +kpm: failed to resolve invalid_tag: localhost:5001/test/invalid_oci_repo:invalid_tag: not found \ No newline at end of file diff --git a/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_ref/test_suite.stdout b/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_ref/test_suite.stdout index 30ea0c5c..76ca84a3 100644 --- a/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_ref/test_suite.stdout +++ b/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_ref/test_suite.stdout @@ -1 +1 @@ -kpm: pulling 'invalid_oci_repo:invalid_tag' from 'localhost:5001/invalid_oci_repo'. +kpm: pulling 'test/invalid_oci_repo:invalid_tag' from 'localhost:5001/test/invalid_oci_repo'. \ No newline at end of file From 242a2e3ad903b4be43223b4a4132a23ecc5c75e2 Mon Sep 17 00:00:00 2001 From: zongz Date: Mon, 25 Sep 2023 14:30:55 +0800 Subject: [PATCH 2/4] fix: fix CR comment --- pkg/client/client_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index da8cc2b7..8cfa34cf 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -530,7 +530,7 @@ func TestResolveMetadataInJsonStr(t *testing.T) { assert.Equal(t, err, nil) assert.Equal(t, utils.DirExists(vendorDir), false) assert.Equal(t, utils.DirExists(filepath.Join(vendorDir, "konfig_v0.0.1")), false) - expectedStr := "{\"packages\":{\"konfig\":{\"name\":\"konfig\",\"manifest_path\":\"not_exist/konfig_v0.0.1\"}}}" + expectedStr := fmt.Sprintf("{\"packages\":{\"konfig\":{\"name\":\"konfig\",\"manifest_path\":\"%s\"}}}", filepath.Join("not_exist", "konfig_v0.0.1")) assert.Equal(t, res, expectedStr) defer func() { if r := os.RemoveAll(filepath.Join("not_exist", "konfig_v0.0.1")); r != nil { From 4fc9641935ce91ae7549dd190d087332cd251776 Mon Sep 17 00:00:00 2001 From: zongz Date: Mon, 25 Sep 2023 15:50:10 +0800 Subject: [PATCH 3/4] fix: fix windows test cases --- pkg/client/client_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 8cfa34cf..da39b986 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -530,7 +530,9 @@ func TestResolveMetadataInJsonStr(t *testing.T) { assert.Equal(t, err, nil) assert.Equal(t, utils.DirExists(vendorDir), false) assert.Equal(t, utils.DirExists(filepath.Join(vendorDir, "konfig_v0.0.1")), false) - expectedStr := fmt.Sprintf("{\"packages\":{\"konfig\":{\"name\":\"konfig\",\"manifest_path\":\"%s\"}}}", filepath.Join("not_exist", "konfig_v0.0.1")) + jsonPath, err := json.Marshal(filepath.Join("not_exist", "konfig_v0.0.1")) + assert.Equal(t, err, nil) + expectedStr := fmt.Sprintf("{\"packages\":{\"konfig\":{\"name\":\"konfig\",\"manifest_path\":\"%s\"}}}", string(jsonPath)) assert.Equal(t, res, expectedStr) defer func() { if r := os.RemoveAll(filepath.Join("not_exist", "konfig_v0.0.1")); r != nil { From b3ea4ef426b1127a8079058a31b3940a95c70629 Mon Sep 17 00:00:00 2001 From: zongz Date: Mon, 25 Sep 2023 15:54:08 +0800 Subject: [PATCH 4/4] fix: fix windows test cases --- pkg/client/client_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index da39b986..29c50c18 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -532,7 +532,7 @@ func TestResolveMetadataInJsonStr(t *testing.T) { assert.Equal(t, utils.DirExists(filepath.Join(vendorDir, "konfig_v0.0.1")), false) jsonPath, err := json.Marshal(filepath.Join("not_exist", "konfig_v0.0.1")) assert.Equal(t, err, nil) - expectedStr := fmt.Sprintf("{\"packages\":{\"konfig\":{\"name\":\"konfig\",\"manifest_path\":\"%s\"}}}", string(jsonPath)) + expectedStr := fmt.Sprintf("{\"packages\":{\"konfig\":{\"name\":\"konfig\",\"manifest_path\":%s}}}", string(jsonPath)) assert.Equal(t, res, expectedStr) defer func() { if r := os.RemoveAll(filepath.Join("not_exist", "konfig_v0.0.1")); r != nil {