From 2ff2593f8373954e4cea123de4693d218124aa40 Mon Sep 17 00:00:00 2001 From: zongz Date: Wed, 4 Dec 2024 14:56:09 +0800 Subject: [PATCH 01/10] fix: fix missing tag in command push Signed-off-by: zongz --- pkg/checker/checker_test.go | 207 +++++++++--------- pkg/client/client_test.go | 4 +- pkg/client/issues_test.go | 122 +++++++++++ pkg/client/push.go | 122 ++++++++++- pkg/client/push_test.go | 86 ++++++++ .../kcl-lang/kcl/issues/1768/README.md | 22 ++ .../1768/depends_on_pushed_mod/kcl.mod.bk | 4 + .../1768/depends_on_pushed_mod/kcl.mod.expect | 7 + .../depends_on_pushed_mod/kcl.mod.lock.bk | 0 .../depends_on_pushed_mod/kcl.mod.lock.expect | 9 + .../issues/1768/depends_on_pushed_mod/main.k | 1 + .../kcl/issues/1768/pushed_mod/kcl.mod | 4 + .../kcl/issues/1768/pushed_mod/kcl.mod.lock | 0 .../kcl/issues/1768/pushed_mod/main.k | 1 + pkg/client/test_data/test_push/push_0/kcl.mod | 4 + .../test_data/test_push/push_0/kcl.mod.lock | 0 pkg/client/test_data/test_push/push_0/main.k | 1 + .../test_push/test_pushed_mod/kcl.mod | 7 + .../test_push/test_pushed_mod/kcl.mod.lock | 9 + .../test_push/test_pushed_mod/main.k | 1 + 20 files changed, 507 insertions(+), 104 deletions(-) create mode 100644 pkg/client/push_test.go create mode 100644 pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/README.md create mode 100644 pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.bk create mode 100644 pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.expect create mode 100644 pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.lock.bk create mode 100644 pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.lock.expect create mode 100644 pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/main.k create mode 100644 pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/pushed_mod/kcl.mod create mode 100644 pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/pushed_mod/kcl.mod.lock create mode 100644 pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/pushed_mod/main.k create mode 100644 pkg/client/test_data/test_push/push_0/kcl.mod create mode 100644 pkg/client/test_data/test_push/push_0/kcl.mod.lock create mode 100644 pkg/client/test_data/test_push/push_0/main.k create mode 100644 pkg/client/test_data/test_push/test_pushed_mod/kcl.mod create mode 100644 pkg/client/test_data/test_push/test_pushed_mod/kcl.mod.lock create mode 100644 pkg/client/test_data/test_push/test_pushed_mod/main.k diff --git a/pkg/checker/checker_test.go b/pkg/checker/checker_test.go index 56e51281..6a1cd203 100644 --- a/pkg/checker/checker_test.go +++ b/pkg/checker/checker_test.go @@ -12,6 +12,7 @@ import ( pkg "kcl-lang.io/kpm/pkg/package" "kcl-lang.io/kpm/pkg/reporter" "kcl-lang.io/kpm/pkg/settings" + "kcl-lang.io/kpm/pkg/test" ) func TestModCheckerCheck(t *testing.T) { @@ -145,126 +146,130 @@ func getTestSettings() (*settings.Settings, error) { } func TestModCheckerCheck_WithTrustedSum(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("Skipping TestModCheckerCheck_WithTrustedSum test on Windows") - } + testFunc := func(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Skipping TestModCheckerCheck_WithTrustedSum test on Windows") + } - // Start the local Docker registry required for testing - err := mock.StartDockerRegistry() - assert.Equal(t, err, nil) + // Start the local Docker registry required for testing + err := mock.StartDockerRegistry() + assert.Equal(t, err, nil) - // Push the test package to the local OCI registry - err = mock.PushTestPkgToRegistry() - assert.Equal(t, err, nil) + // Push the test package to the local OCI registry + err = mock.PushTestPkgToRegistry() + assert.Equal(t, err, nil) - // Initialize settings for use with the ModChecker - settings, err := getTestSettings() - assert.Equal(t, err, nil) + // Initialize settings for use with the ModChecker + settings, err := getTestSettings() + assert.Equal(t, err, nil) - // Initialize the ModChecker with required checkers - ModChecker := NewModChecker(WithCheckers(NewIdentChecker(), NewVersionChecker(), NewSumChecker(WithSettings(*settings)))) + // Initialize the ModChecker with required checkers + ModChecker := NewModChecker(WithCheckers(NewIdentChecker(), NewVersionChecker(), NewSumChecker(WithSettings(*settings)))) - deps1 := orderedmap.NewOrderedMap[string, pkg.Dependency]() - deps1.Set("kcl1", pkg.Dependency{ - Name: "test_data", - FullName: "test_data", - Version: "0.0.1", - Sum: "RpZZIvrXwfn5dpt6LqBR8+FlPE9Y+BEou47L3qaCCqk=", - Source: downloader.Source{ - Oci: &downloader.Oci{ - Reg: "localhost:5001", - Repo: "test", - Tag: "0.0.1", + deps1 := orderedmap.NewOrderedMap[string, pkg.Dependency]() + deps1.Set("kcl1", pkg.Dependency{ + Name: "test_data", + FullName: "test_data", + Version: "0.0.1", + Sum: "RpZZIvrXwfn5dpt6LqBR8+FlPE9Y+BEou47L3qaCCqk=", + Source: downloader.Source{ + Oci: &downloader.Oci{ + Reg: "localhost:5001", + Repo: "test", + Tag: "0.0.1", + }, }, - }, - }) + }) - deps2 := orderedmap.NewOrderedMap[string, pkg.Dependency]() - deps2.Set("kcl1", pkg.Dependency{ - Name: "test_data", - FullName: "test_data", - Version: "0.0.1", - Sum: "Invalid-sum", - Source: downloader.Source{ - Oci: &downloader.Oci{ - Reg: "localhost:5001", - Repo: "test", - Tag: "0.0.1", + deps2 := orderedmap.NewOrderedMap[string, pkg.Dependency]() + deps2.Set("kcl1", pkg.Dependency{ + Name: "test_data", + FullName: "test_data", + Version: "0.0.1", + Sum: "Invalid-sum", + Source: downloader.Source{ + Oci: &downloader.Oci{ + Reg: "localhost:5001", + Repo: "test", + Tag: "0.0.1", + }, }, - }, - }) + }) - tests := []struct { - name string - KclPkg pkg.KclPkg - wantErr bool - }{ - { - name: "valid kcl package - with sum check", - KclPkg: pkg.KclPkg{ - ModFile: pkg.ModFile{ - Pkg: pkg.Package{ - Name: "testmod", - Version: "0.0.1", + tests := []struct { + name string + KclPkg pkg.KclPkg + wantErr bool + }{ + { + name: "valid kcl package - with sum check", + KclPkg: pkg.KclPkg{ + ModFile: pkg.ModFile{ + Pkg: pkg.Package{ + Name: "testmod", + Version: "0.0.1", + }, + HomePath: "path/to/modfile", }, - HomePath: "path/to/modfile", - }, - HomePath: "path/to/kcl/pkg", - Dependencies: pkg.Dependencies{ - Deps: deps1, + HomePath: "path/to/kcl/pkg", + Dependencies: pkg.Dependencies{ + Deps: deps1, + }, + NoSumCheck: false, }, - NoSumCheck: false, + wantErr: false, }, - wantErr: false, - }, - { - name: "valid kcl package - with no sum check enabled", - KclPkg: pkg.KclPkg{ - ModFile: pkg.ModFile{ - Pkg: pkg.Package{ - Name: "testmod", - Version: "0.0.1", + { + name: "valid kcl package - with no sum check enabled", + KclPkg: pkg.KclPkg{ + ModFile: pkg.ModFile{ + Pkg: pkg.Package{ + Name: "testmod", + Version: "0.0.1", + }, + HomePath: "path/to/modfile", }, - HomePath: "path/to/modfile", - }, - HomePath: "path/to/kcl/pkg", - Dependencies: pkg.Dependencies{ - Deps: deps2, + HomePath: "path/to/kcl/pkg", + Dependencies: pkg.Dependencies{ + Deps: deps2, + }, + NoSumCheck: true, }, - NoSumCheck: true, + wantErr: false, }, - wantErr: false, - }, - { - name: "Invalid kcl package - with no sum check disabled - checksum mismatches", - KclPkg: pkg.KclPkg{ - ModFile: pkg.ModFile{ - Pkg: pkg.Package{ - Name: "testmod", - Version: "0.0.1", + { + name: "Invalid kcl package - with no sum check disabled - checksum mismatches", + KclPkg: pkg.KclPkg{ + ModFile: pkg.ModFile{ + Pkg: pkg.Package{ + Name: "testmod", + Version: "0.0.1", + }, + HomePath: "path/to/modfile", }, - HomePath: "path/to/modfile", - }, - HomePath: "path/to/kcl/pkg", - Dependencies: pkg.Dependencies{ - Deps: deps2, + HomePath: "path/to/kcl/pkg", + Dependencies: pkg.Dependencies{ + Deps: deps2, + }, + NoSumCheck: false, }, - NoSumCheck: false, + wantErr: true, }, - wantErr: true, - }, - } + } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotErr := ModChecker.Check(tt.KclPkg) - if (gotErr != nil) != tt.wantErr { - t.Errorf("ModChecker.Check(%v) = %v, want error %v", tt.KclPkg, gotErr, tt.wantErr) - } - }) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotErr := ModChecker.Check(tt.KclPkg) + if (gotErr != nil) != tt.wantErr { + t.Errorf("ModChecker.Check(%v) = %v, want error %v", tt.KclPkg, gotErr, tt.wantErr) + } + }) + } + + // Clean the environment after all tests have been run + err = mock.CleanTestEnv() + assert.Equal(t, err, nil) } - // Clean the environment after all tests have been run - err = mock.CleanTestEnv() - assert.Equal(t, err, nil) + test.RunTestWithGlobalLock(t, "TestModCheckerCheck_WithTrustedSum", testFunc) } diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 2f2944cc..247c6de2 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -2255,12 +2255,12 @@ func testPushWithInsecureSkipTLSverify(t *testing.T) { kpmcli, err := NewKpmClient() assert.Equal(t, err, nil) - _ = kpmcli.PushToOci("test", ociOpts) + _ = kpmcli.pushToOci("test", ociOpts) assert.Equal(t, buf.String(), "") kpmcli.SetInsecureSkipTLSverify(true) - _ = kpmcli.PushToOci("test", ociOpts) + _ = kpmcli.pushToOci("test", ociOpts) assert.Equal(t, buf.String(), "Called Success\n") } diff --git a/pkg/client/issues_test.go b/pkg/client/issues_test.go index 247bdef5..bde94956 100644 --- a/pkg/client/issues_test.go +++ b/pkg/client/issues_test.go @@ -13,7 +13,10 @@ import ( "github.com/stretchr/testify/assert" "kcl-lang.io/kpm/pkg/downloader" "kcl-lang.io/kpm/pkg/features" + "kcl-lang.io/kpm/pkg/mock" + "kcl-lang.io/kpm/pkg/opt" pkg "kcl-lang.io/kpm/pkg/package" + "kcl-lang.io/kpm/pkg/reporter" "kcl-lang.io/kpm/pkg/utils" ) @@ -430,3 +433,122 @@ func TestKpmIssue226(t *testing.T) { RunTestWithGlobalLockAndKpmCli(t, []TestSuite{{Name: "update_with_git_commit", TestFunc: test_update_with_git_commit}}) RunTestWithGlobalLockAndKpmCli(t, []TestSuite{{Name: "update_with_git_commit_invalid", TestFunc: test_update_with_git_commit_invalid}}) } + +func TestKclIssue1768(t *testing.T) { + testPath := "github.com/kcl-lang/kcl/issues/1768" + test_push_with_tag := func(t *testing.T, kpmcli *KpmClient) { + + err := mock.StartDockerRegistry() + if err != nil { + t.Errorf("Error starting docker registry: %v", err) + } + + defer func() { + err = mock.CleanTestEnv() + if err != nil { + t.Errorf("Error stopping docker registry: %v", err) + } + }() + + kpmcli.SetInsecureSkipTLSverify(true) + err = kpmcli.LoginOci("localhost:5001", "test", "1234") + if err != nil { + t.Errorf("Error logging in to docker registry: %v", err) + } + + rootPath := getTestDir("issues") + pushedModPath := filepath.Join(rootPath, testPath, "pushed_mod") + + modPath := filepath.Join(rootPath, testPath, "depends_on_pushed_mod") + modFileBk := filepath.Join(modPath, "kcl.mod.bk") + LockFileBk := filepath.Join(modPath, "kcl.mod.lock.bk") + modFile := filepath.Join(modPath, "kcl.mod") + LockFile := filepath.Join(modPath, "kcl.mod.lock") + modFileExpect := filepath.Join(modPath, "kcl.mod.expect") + LockFileExpect := filepath.Join(modPath, "kcl.mod.lock.expect") + + defer func() { + _ = os.RemoveAll(modFile) + _ = os.RemoveAll(LockFile) + }() + + err = copy.Copy(modFileBk, modFile) + if err != nil { + t.Fatal(err) + } + err = copy.Copy(LockFileBk, LockFile) + if err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + kpmcli.SetLogWriter(&buf) + + err = kpmcli.Push( + WithPushModPath(pushedModPath), + WithPushOciOptions(&opt.OciOptions{ + Reg: "localhost:5001", + Repo: "test", + Ref: "oci_pushed_mod", + Tag: "v9.9.9", + }), + ) + + if err != (*reporter.KpmEvent)(nil) { + t.Errorf("Error pushing kcl package: %v", err) + } + + assert.Contains(t, buf.String(), "package 'pushed_mod' will be pushed") + assert.Contains(t, buf.String(), "pushed [registry] localhost:5001/test/oci_pushed_mod") + assert.Contains(t, buf.String(), "digest: sha256:") + + kmod, err := pkg.LoadKclPkgWithOpts( + pkg.WithPath(modPath), + ) + + err = kpmcli.Add( + WithAddKclPkg(kmod), + WithAddSource( + &downloader.Source{ + Oci: &downloader.Oci{ + Reg: "localhost:5001", + Repo: "test/oci_pushed_mod", + Tag: "v9.9.9", + }}, + ), + WithAddModSpec( + &downloader.ModSpec{ + Name: "pushed_mod", + Version: "0.0.1", + }, + ), + ) + + if err != nil { + t.Errorf("Error adding dependency: %v", err) + } + + modFileContent, err := os.ReadFile(modFile) + if err != nil { + t.Fatal(err) + } + lockFileContent, err := os.ReadFile(LockFile) + if err != nil { + t.Fatal(err) + } + + modFileExpectContent, err := os.ReadFile(modFileExpect) + if err != nil { + t.Fatal(err) + } + + lockFileExpectContent, err := os.ReadFile(LockFileExpect) + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, utils.RmNewline(string(modFileContent)), utils.RmNewline(string(modFileExpectContent))) + assert.Equal(t, utils.RmNewline(string(lockFileContent)), utils.RmNewline(string(lockFileExpectContent))) + } + RunTestWithGlobalLockAndKpmCli(t, []TestSuite{{Name: "test_push_with_tag", TestFunc: test_push_with_tag}}) +} diff --git a/pkg/client/push.go b/pkg/client/push.go index aa540ff7..08559a4a 100644 --- a/pkg/client/push.go +++ b/pkg/client/push.go @@ -2,16 +2,136 @@ package client import ( "fmt" + "os" + "kcl-lang.io/kpm/pkg/errors" "kcl-lang.io/kpm/pkg/oci" "kcl-lang.io/kpm/pkg/opt" + pkg "kcl-lang.io/kpm/pkg/package" "kcl-lang.io/kpm/pkg/reporter" "kcl-lang.io/kpm/pkg/utils" ) +// PushOptions contains the options for the Push method. +type PushOptions struct { + // By default, the registry, repository, reference and tag are set to the default values + // registry is set to the default registry `ghcr.io` in the settings + // repository is set to the default repository `kcl-lang` in the settings + // reference is set to the package name in `kcl.mod` + // tag is set to the package version in `kcl.mod` + OciOpts *opt.OciOptions + ModPath string + VendorMode bool +} + +type PushOption func(*PushOptions) error + +// WithPushVendorMode sets the vendor mode for the Push method. +func WithPushVendorMode(vendorMode bool) PushOption { + return func(opts *PushOptions) error { + opts.VendorMode = vendorMode + return nil + } +} + +// WithPushOciOptions sets the oci options for the Push method. +func WithPushOciOptions(ociOpts *opt.OciOptions) PushOption { + return func(opts *PushOptions) error { + if ociOpts == nil { + return fmt.Errorf("ociOpts cannot be nil") + } + opts.OciOpts = ociOpts + return nil + } +} + +// WithPushModPath sets the modPath for the Push method. +func WithPushModPath(modPath string) PushOption { + return func(opts *PushOptions) error { + if modPath == "" { + return fmt.Errorf("modPath cannot be empty") + } + opts.ModPath = modPath + return nil + } +} + +// fillDefaultPushOptions will fill the default values for the PushOptions. +func (c *KpmClient) fillDefaultPushOptions(opts *PushOptions, kMod *pkg.KclPkg) { + if opts.OciOpts == nil { + opts.OciOpts = &opt.OciOptions{ + Reg: c.GetSettings().DefaultOciRegistry(), + Repo: c.GetSettings().DefaultOciRepo(), + Ref: kMod.ModFile.Pkg.Name, + Tag: kMod.ModFile.Pkg.Version, + } + } else { + if opts.OciOpts.Reg == "" { + opts.OciOpts.Reg = c.GetSettings().DefaultOciRegistry() + } + + if opts.OciOpts.Repo == "" { + opts.OciOpts.Repo = c.GetSettings().DefaultOciRepo() + } + + if opts.OciOpts.Ref == "" { + opts.OciOpts.Ref = kMod.ModFile.Pkg.Name + } + + if opts.OciOpts.Tag == "" { + opts.OciOpts.Tag = kMod.ModFile.Pkg.Version + } + } +} + +// Push will push a kcl package to a registry. +func (c *KpmClient) Push(opts ...PushOption) error { + pushOpts := &PushOptions{} + for _, opt := range opts { + if err := opt(pushOpts); err != nil { + return err + } + } + + kMod, err := pkg.LoadKclPkgWithOpts( + pkg.WithPath(pushOpts.ModPath), + pkg.WithSettings(c.GetSettings()), + ) + + if err != nil { + return err + } + + c.fillDefaultPushOptions(pushOpts, kMod) + ociOpts := pushOpts.OciOpts + ociOpts.Annotations, err = kMod.GenOciManifestFromPkg() + if err != nil { + return err + } + + tarPath, err := c.PackagePkg(kMod, pushOpts.VendorMode) + if err != nil { + return err + } + + // clean the tar path. + defer func() { + if kMod != nil && utils.DirExists(tarPath) { + err = os.RemoveAll(tarPath) + if err != nil { + err = errors.InternalBug + } + } + }() + + reporter.ReportMsgTo(fmt.Sprintf("package '%s' will be pushed", kMod.GetPkgName()), c.GetLogWriter()) + return c.pushToOci(tarPath, ociOpts) +} + // PushToOci will push a kcl package to oci registry. -func (c *KpmClient) PushToOci(localPath string, ociOpts *opt.OciOptions) error { +func (c *KpmClient) pushToOci(localPath string, ociOpts *opt.OciOptions) error { repoPath := utils.JoinPath(ociOpts.Reg, ociOpts.Repo) + repoPath = utils.JoinPath(repoPath, ociOpts.Ref) cred, err := c.GetCredentials(ociOpts.Reg) if err != nil { return err diff --git a/pkg/client/push_test.go b/pkg/client/push_test.go new file mode 100644 index 00000000..d360ffec --- /dev/null +++ b/pkg/client/push_test.go @@ -0,0 +1,86 @@ +package client + +import ( + "bytes" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "kcl-lang.io/kpm/pkg/downloader" + "kcl-lang.io/kpm/pkg/mock" + "kcl-lang.io/kpm/pkg/opt" + pkg "kcl-lang.io/kpm/pkg/package" + "kcl-lang.io/kpm/pkg/reporter" +) + +func TestPush(t *testing.T) { + testFunc := func(t *testing.T, kpmcli *KpmClient) { + err := mock.StartDockerRegistry() + if err != nil { + t.Errorf("Error starting docker registry: %v", err) + } + + defer func() { + err = mock.CleanTestEnv() + if err != nil { + t.Errorf("Error stopping docker registry: %v", err) + } + }() + + kpmcli.SetInsecureSkipTLSverify(true) + err = kpmcli.LoginOci("localhost:5001", "test", "1234") + if err != nil { + t.Errorf("Error logging in to docker registry: %v", err) + } + + var buf bytes.Buffer + kpmcli.SetLogWriter(&buf) + + testDir := getTestDir("test_push") + pushedModPath := filepath.Join(testDir, "push_0") + + err = kpmcli.Push( + WithPushModPath(pushedModPath), + WithPushOciOptions(&opt.OciOptions{ + Reg: "localhost:5001", + Repo: "test", + }), + ) + + if err != (*reporter.KpmEvent)(nil) { + t.Errorf("Error pushing kcl package: %v", err) + } + + assert.Contains(t, buf.String(), "package 'push_0' will be pushed") + assert.Contains(t, buf.String(), "pushed [registry] localhost:5001/test/push_0") + assert.Contains(t, buf.String(), "digest: sha256:") + + testPushModPath := filepath.Join(testDir, "test_pushed_mod") + + err = kpmcli.Init( + WithInitModPath(testPushModPath), + ) + if err != nil { + t.Errorf("Error initializing kcl package: %v", err) + } + + testMod, err := pkg.LoadKclPkgWithOpts( + pkg.WithPath(testPushModPath), + ) + + err = kpmcli.Add( + WithAddKclPkg(testMod), + WithAddSourceUrl("oci://localhost:5001/test/push_0"), + WithAddModSpec(&downloader.ModSpec{ + Name: "push_0", + Version: "0.0.1", + }), + ) + + if err != nil { + t.Errorf("Error adding kcl package: %v", err) + } + + } + RunTestWithGlobalLockAndKpmCli(t, []TestSuite{{Name: "TestPush", TestFunc: testFunc}}) +} diff --git a/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/README.md b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/README.md new file mode 100644 index 00000000..05756ead --- /dev/null +++ b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/README.md @@ -0,0 +1,22 @@ +## Bug Report + +https://github.com/kcl-lang/kcl/issues/1768 + +The [documentation](https://www.kcl-lang.io/docs/user_docs/guides/package-management/how-to/kpm_oci#kcl-mod-push-to-upload-a-kcl-package) mentions a `--tag` flag for `kcl mod push` to overwrite the version written in `kcl.mod`. + +However, it appears this flag doesn't exist. + +### 1. Minimal reproduce step (Required) + +Run `kcl mod push --tag 0.0.42` + +### 2. What did you expect to see? (Required) + +### 3. What did you see instead (Required) + +``` +unknown flag: --tag +``` +### 4. What is your KCL components version? (Required) + +kcl version: 0.10.10-linux-amd64 diff --git a/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.bk b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.bk new file mode 100644 index 00000000..f2e4fb50 --- /dev/null +++ b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.bk @@ -0,0 +1,4 @@ +[package] +name = "depends_on_pushed_mod" +edition = "v0.10.0" +version = "0.0.1" diff --git a/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.expect b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.expect new file mode 100644 index 00000000..8dd01d70 --- /dev/null +++ b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.expect @@ -0,0 +1,7 @@ +[package] +name = "depends_on_pushed_mod" +edition = "v0.10.0" +version = "0.0.1" + +[dependencies] +pushed_mod = { oci = "oci://localhost:5001/test/oci_pushed_mod", tag = "v9.9.9", version = "0.0.1" } diff --git a/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.lock.bk b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.lock.bk new file mode 100644 index 00000000..e69de29b diff --git a/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.lock.expect b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.lock.expect new file mode 100644 index 00000000..0478255a --- /dev/null +++ b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.lock.expect @@ -0,0 +1,9 @@ +[dependencies] + [dependencies.pushed_mod] + name = "pushed_mod" + full_name = "pushed_mod_0.0.1" + version = "0.0.1" + sum = "7reGjELgXlmS2kU3VeRzvlLv2gauyGmeUNyJmV4R2Kw=" + reg = "localhost:5001" + repo = "test/oci_pushed_mod" + oci_tag = "v9.9.9" diff --git a/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/main.k b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/pushed_mod/kcl.mod b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/pushed_mod/kcl.mod new file mode 100644 index 00000000..e562f6ba --- /dev/null +++ b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/pushed_mod/kcl.mod @@ -0,0 +1,4 @@ +[package] +name = "pushed_mod" +edition = "v0.10.0" +version = "0.0.1" diff --git a/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/pushed_mod/kcl.mod.lock b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/pushed_mod/kcl.mod.lock new file mode 100644 index 00000000..e69de29b diff --git a/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/pushed_mod/main.k b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/pushed_mod/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/pushed_mod/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/client/test_data/test_push/push_0/kcl.mod b/pkg/client/test_data/test_push/push_0/kcl.mod new file mode 100644 index 00000000..0dbc0c9f --- /dev/null +++ b/pkg/client/test_data/test_push/push_0/kcl.mod @@ -0,0 +1,4 @@ +[package] +name = "push_0" +edition = "v0.10.0" +version = "0.0.1" diff --git a/pkg/client/test_data/test_push/push_0/kcl.mod.lock b/pkg/client/test_data/test_push/push_0/kcl.mod.lock new file mode 100644 index 00000000..e69de29b diff --git a/pkg/client/test_data/test_push/push_0/main.k b/pkg/client/test_data/test_push/push_0/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/test_push/push_0/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/client/test_data/test_push/test_pushed_mod/kcl.mod b/pkg/client/test_data/test_push/test_pushed_mod/kcl.mod new file mode 100644 index 00000000..9c637545 --- /dev/null +++ b/pkg/client/test_data/test_push/test_pushed_mod/kcl.mod @@ -0,0 +1,7 @@ +[package] +name = "test_pushed_mod" +edition = "v0.10.0" +version = "0.0.1" + +[dependencies] +push_0 = { oci = "oci://localhost:5001/test/push_0", tag = "0.0.1", version = "0.0.1" } diff --git a/pkg/client/test_data/test_push/test_pushed_mod/kcl.mod.lock b/pkg/client/test_data/test_push/test_pushed_mod/kcl.mod.lock new file mode 100644 index 00000000..524ea52b --- /dev/null +++ b/pkg/client/test_data/test_push/test_pushed_mod/kcl.mod.lock @@ -0,0 +1,9 @@ +[dependencies] + [dependencies.push_0] + name = "push_0" + full_name = "push_0_0.0.1" + version = "0.0.1" + sum = "m9g+EU6NLk5sbcV2C26/ikgrkcNF7Myk8751xkfUwIk=" + reg = "localhost:5001" + repo = "test/push_0" + oci_tag = "0.0.1" diff --git a/pkg/client/test_data/test_push/test_pushed_mod/main.k b/pkg/client/test_data/test_push/test_pushed_mod/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/test_push/test_pushed_mod/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file From 512c483cc23c9afd07464bcbeacb0be994668408 Mon Sep 17 00:00:00 2001 From: zongz Date: Wed, 4 Dec 2024 15:28:05 +0800 Subject: [PATCH 02/10] fix: fix test ci Signed-off-by: zongz --- pkg/cmd/cmd_push.go | 40 ++++++++++++++-------------------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/pkg/cmd/cmd_push.go b/pkg/cmd/cmd_push.go index 3e6dca8b..8833277f 100644 --- a/pkg/cmd/cmd_push.go +++ b/pkg/cmd/cmd_push.go @@ -142,25 +142,9 @@ func pushTarPackage(ociUrl, localTarPath string, vendorMode bool, kpmcli *client // 3. Generate the OCI options from oci url and the version of current kcl package. // 4. Push the package to the oci registry. func pushPackage(ociUrl string, kclPkg *pkg.KclPkg, vendorMode bool, kpmcli *client.KpmClient) error { - - tarPath, err := kpmcli.PackagePkg(kclPkg, vendorMode) - if err != nil { - return err - } - - // clean the tar path. - defer func() { - if kclPkg != nil && utils.DirExists(tarPath) { - err = os.RemoveAll(tarPath) - if err != nil { - err = reporter.NewErrorEvent(reporter.Bug, err, "internal bugs, failed to clean the temp dir.") - } - } - }() - - // 2. If the oci url is not specified, generate the default oci url from the current package. + // If the oci url is not specified, generate the default oci url from the current package. if len(ociUrl) == 0 { - ociUrl, err = genDefaultOciUrlForKclPkg(kclPkg, kpmcli) + ociUrl, err := genDefaultOciUrlForKclPkg(kclPkg, kpmcli) if err != nil || len(ociUrl) == 0 { return reporter.NewErrorEvent( reporter.InvalidCmd, @@ -170,9 +154,9 @@ func pushPackage(ociUrl string, kclPkg *pkg.KclPkg, vendorMode bool, kpmcli *cli } } - // 3. Generate the OCI options from oci url and the version of current kcl package. + // Generate the OCI options from oci url and the version of current kcl package. ociOpts, err := opt.ParseOciOptionFromOciUrl(ociUrl, kclPkg.GetPkgTag()) - if err != (*reporter.KpmEvent)(nil) { + if err != nil { return reporter.NewErrorEvent( reporter.UnsupportOciUrlScheme, errors.InvalidOciUrl, @@ -180,16 +164,20 @@ func pushPackage(ociUrl string, kclPkg *pkg.KclPkg, vendorMode bool, kpmcli *cli ) } - ociOpts.Annotations, err = kclPkg.GenOciManifestFromPkg() - if err != nil { + var gerr error + ociOpts.Annotations, gerr = kclPkg.GenOciManifestFromPkg() + if gerr != nil { return err } reporter.ReportMsgTo(fmt.Sprintf("package '%s' will be pushed", kclPkg.GetPkgName()), kpmcli.GetLogWriter()) - // 4. Push it. - err = kpmcli.PushToOci(tarPath, ociOpts) - if err != (*reporter.KpmEvent)(nil) { - return err + // Push it. + gerr = kpmcli.Push( + client.WithPushModPath(kclPkg.HomePath), + client.WithPushOciOptions(ociOpts), + ) + if gerr != nil { + return gerr } return nil } From fa2315d01dc2749dd9e902825d0c65a01b5ed4da Mon Sep 17 00:00:00 2001 From: zongz Date: Wed, 4 Dec 2024 15:40:41 +0800 Subject: [PATCH 03/10] fix: fix test ci Signed-off-by: zongz --- pkg/cmd/cmd_push.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/cmd/cmd_push.go b/pkg/cmd/cmd_push.go index 8833277f..f3d2613c 100644 --- a/pkg/cmd/cmd_push.go +++ b/pkg/cmd/cmd_push.go @@ -143,8 +143,9 @@ func pushTarPackage(ociUrl, localTarPath string, vendorMode bool, kpmcli *client // 4. Push the package to the oci registry. func pushPackage(ociUrl string, kclPkg *pkg.KclPkg, vendorMode bool, kpmcli *client.KpmClient) error { // If the oci url is not specified, generate the default oci url from the current package. + var err error if len(ociUrl) == 0 { - ociUrl, err := genDefaultOciUrlForKclPkg(kclPkg, kpmcli) + ociUrl, err = genDefaultOciUrlForKclPkg(kclPkg, kpmcli) if err != nil || len(ociUrl) == 0 { return reporter.NewErrorEvent( reporter.InvalidCmd, @@ -156,7 +157,7 @@ func pushPackage(ociUrl string, kclPkg *pkg.KclPkg, vendorMode bool, kpmcli *cli // Generate the OCI options from oci url and the version of current kcl package. ociOpts, err := opt.ParseOciOptionFromOciUrl(ociUrl, kclPkg.GetPkgTag()) - if err != nil { + if err != (*reporter.KpmEvent)(nil) { return reporter.NewErrorEvent( reporter.UnsupportOciUrlScheme, errors.InvalidOciUrl, @@ -175,6 +176,7 @@ func pushPackage(ociUrl string, kclPkg *pkg.KclPkg, vendorMode bool, kpmcli *cli gerr = kpmcli.Push( client.WithPushModPath(kclPkg.HomePath), client.WithPushOciOptions(ociOpts), + client.WithPushVendorMode(vendorMode), ) if gerr != nil { return gerr From 412c2f83e0c43459a55bb6e6f5a724cc4200491a Mon Sep 17 00:00:00 2001 From: zongz Date: Wed, 4 Dec 2024 15:45:25 +0800 Subject: [PATCH 04/10] fix: fix test ci Signed-off-by: zongz --- pkg/cmd/cmd_push.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/cmd/cmd_push.go b/pkg/cmd/cmd_push.go index f3d2613c..fff6661d 100644 --- a/pkg/cmd/cmd_push.go +++ b/pkg/cmd/cmd_push.go @@ -165,21 +165,20 @@ func pushPackage(ociUrl string, kclPkg *pkg.KclPkg, vendorMode bool, kpmcli *cli ) } - var gerr error - ociOpts.Annotations, gerr = kclPkg.GenOciManifestFromPkg() - if gerr != nil { + ociOpts.Annotations, err = kclPkg.GenOciManifestFromPkg() + if err != nil { return err } reporter.ReportMsgTo(fmt.Sprintf("package '%s' will be pushed", kclPkg.GetPkgName()), kpmcli.GetLogWriter()) // Push it. - gerr = kpmcli.Push( + err = kpmcli.Push( client.WithPushModPath(kclPkg.HomePath), client.WithPushOciOptions(ociOpts), client.WithPushVendorMode(vendorMode), ) - if gerr != nil { - return gerr + if err != (*reporter.KpmEvent)(nil) { + return err } return nil } From fb2cebaf327be97068e8f24a378ed1ea935a6b39 Mon Sep 17 00:00:00 2001 From: zongz Date: Wed, 4 Dec 2024 15:47:33 +0800 Subject: [PATCH 05/10] fix: fix test TestModCheckerCheck_WithTrustedSum Signed-off-by: zongz --- pkg/checker/checker_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/checker/checker_test.go b/pkg/checker/checker_test.go index 6a1cd203..0ead1534 100644 --- a/pkg/checker/checker_test.go +++ b/pkg/checker/checker_test.go @@ -175,7 +175,7 @@ func TestModCheckerCheck_WithTrustedSum(t *testing.T) { Source: downloader.Source{ Oci: &downloader.Oci{ Reg: "localhost:5001", - Repo: "test", + Repo: "test/test_data", Tag: "0.0.1", }, }, @@ -190,7 +190,7 @@ func TestModCheckerCheck_WithTrustedSum(t *testing.T) { Source: downloader.Source{ Oci: &downloader.Oci{ Reg: "localhost:5001", - Repo: "test", + Repo: "test/test_data", Tag: "0.0.1", }, }, From 202e2b6cba803d4c7ec11ba30663f4a635beb175 Mon Sep 17 00:00:00 2001 From: zongz Date: Wed, 4 Dec 2024 17:26:21 +0800 Subject: [PATCH 06/10] fix: fix test ci Signed-off-by: zongz --- pkg/client/deperated.go | 9 ++- pkg/client/issues_test.go | 12 +-- pkg/client/push.go | 76 ++++++++++--------- pkg/client/push_test.go | 13 ++-- pkg/cmd/cmd_push.go | 17 +++-- pkg/opt/opt.go | 14 +++- pkg/utils/utils.go | 19 ++++- .../kpm_pull_with_oci_url/test_suite.stdout | 2 +- .../test_suite.stdout | 2 +- .../pull_private_pkg/test_suite.stdout | 2 +- .../test_suite.stderr | 2 +- .../test_suite.stderr | 2 +- 12 files changed, 101 insertions(+), 69 deletions(-) diff --git a/pkg/client/deperated.go b/pkg/client/deperated.go index 9cc72974..8a6d6385 100644 --- a/pkg/client/deperated.go +++ b/pkg/client/deperated.go @@ -198,7 +198,7 @@ func (c *KpmClient) CompileOciPkg(ociSource, version string, opts *opt.CompileOp source := downloader.Source{ Oci: &downloader.Oci{ Reg: ociOpts.Reg, - Repo: ociOpts.Repo, + Repo: utils.JoinPath(ociOpts.Repo, ociOpts.Ref), Tag: ociOpts.Tag, }, } @@ -1338,7 +1338,7 @@ func (c *KpmClient) pullTarFromOci(localPath string, ociOpts *opt.OciOptions) er return reporter.NewErrorEvent(reporter.Bug, err) } - repoPath := utils.JoinPath(ociOpts.Reg, ociOpts.Repo) + repoPath := utils.JoinPath(ociOpts.Reg, ociOpts.Repo, ociOpts.Ref) cred, err := c.GetCredentials(ociOpts.Reg) if err != nil { return err @@ -1373,9 +1373,10 @@ func (c *KpmClient) pullTarFromOci(localPath string, ociOpts *opt.OciOptions) er ociOpts.Tag = tagSelected - full_repo := utils.JoinPath(ociOpts.Reg, ociOpts.Repo) + full_repo := utils.JoinPath(ociOpts.Reg, ociOpts.Repo, ociOpts.Ref) + full_ref := utils.JoinPath(ociOpts.Repo, ociOpts.Ref) reporter.ReportMsgTo( - fmt.Sprintf("pulling '%s:%s' from '%s'", ociOpts.Repo, tagSelected, full_repo), + fmt.Sprintf("pulling '%s:%s' from '%s'", full_ref, tagSelected, full_repo), c.logWriter, ) diff --git a/pkg/client/issues_test.go b/pkg/client/issues_test.go index bde94956..a6babb5e 100644 --- a/pkg/client/issues_test.go +++ b/pkg/client/issues_test.go @@ -14,7 +14,6 @@ import ( "kcl-lang.io/kpm/pkg/downloader" "kcl-lang.io/kpm/pkg/features" "kcl-lang.io/kpm/pkg/mock" - "kcl-lang.io/kpm/pkg/opt" pkg "kcl-lang.io/kpm/pkg/package" "kcl-lang.io/kpm/pkg/reporter" "kcl-lang.io/kpm/pkg/utils" @@ -486,11 +485,12 @@ func TestKclIssue1768(t *testing.T) { err = kpmcli.Push( WithPushModPath(pushedModPath), - WithPushOciOptions(&opt.OciOptions{ - Reg: "localhost:5001", - Repo: "test", - Ref: "oci_pushed_mod", - Tag: "v9.9.9", + WithPushSource(downloader.Source{ + Oci: &downloader.Oci{ + Reg: "localhost:5001", + Repo: "test/oci_pushed_mod", + Tag: "v9.9.9", + }, }), ) diff --git a/pkg/client/push.go b/pkg/client/push.go index 08559a4a..a48da46e 100644 --- a/pkg/client/push.go +++ b/pkg/client/push.go @@ -4,6 +4,7 @@ import ( "fmt" "os" + "kcl-lang.io/kpm/pkg/downloader" "kcl-lang.io/kpm/pkg/errors" "kcl-lang.io/kpm/pkg/oci" "kcl-lang.io/kpm/pkg/opt" @@ -19,28 +20,25 @@ type PushOptions struct { // repository is set to the default repository `kcl-lang` in the settings // reference is set to the package name in `kcl.mod` // tag is set to the package version in `kcl.mod` - OciOpts *opt.OciOptions + Source downloader.Source ModPath string VendorMode bool } type PushOption func(*PushOptions) error -// WithPushVendorMode sets the vendor mode for the Push method. -func WithPushVendorMode(vendorMode bool) PushOption { +// WithPushSource sets the source for the Push method. +func WithPushSource(source downloader.Source) PushOption { return func(opts *PushOptions) error { - opts.VendorMode = vendorMode + opts.Source = source return nil } } -// WithPushOciOptions sets the oci options for the Push method. -func WithPushOciOptions(ociOpts *opt.OciOptions) PushOption { +// WithPushVendorMode sets the vendor mode for the Push method. +func WithPushVendorMode(vendorMode bool) PushOption { return func(opts *PushOptions) error { - if ociOpts == nil { - return fmt.Errorf("ociOpts cannot be nil") - } - opts.OciOpts = ociOpts + opts.VendorMode = vendorMode return nil } } @@ -57,30 +55,17 @@ func WithPushModPath(modPath string) PushOption { } // fillDefaultPushOptions will fill the default values for the PushOptions. -func (c *KpmClient) fillDefaultPushOptions(opts *PushOptions, kMod *pkg.KclPkg) { - if opts.OciOpts == nil { - opts.OciOpts = &opt.OciOptions{ - Reg: c.GetSettings().DefaultOciRegistry(), - Repo: c.GetSettings().DefaultOciRepo(), - Ref: kMod.ModFile.Pkg.Name, - Tag: kMod.ModFile.Pkg.Version, - } - } else { - if opts.OciOpts.Reg == "" { - opts.OciOpts.Reg = c.GetSettings().DefaultOciRegistry() - } - - if opts.OciOpts.Repo == "" { - opts.OciOpts.Repo = c.GetSettings().DefaultOciRepo() - } +func (c *KpmClient) fillDefaultPushOptions(ociOpt *opt.OciOptions, kMod *pkg.KclPkg) { + if ociOpt.Reg == "" { + ociOpt.Reg = c.GetSettings().DefaultOciRegistry() + } - if opts.OciOpts.Ref == "" { - opts.OciOpts.Ref = kMod.ModFile.Pkg.Name - } + if ociOpt.Repo == "" { + ociOpt.Repo = c.GetSettings().DefaultOciRepo() + } - if opts.OciOpts.Tag == "" { - opts.OciOpts.Tag = kMod.ModFile.Pkg.Version - } + if ociOpt.Tag == "" { + ociOpt.Tag = kMod.ModFile.Pkg.Version } } @@ -102,8 +87,28 @@ func (c *KpmClient) Push(opts ...PushOption) error { return err } - c.fillDefaultPushOptions(pushOpts, kMod) - ociOpts := pushOpts.OciOpts + source := pushOpts.Source + ociUrl, err := source.ToString() + if err != nil { + return err + } + if source.Oci == nil { + return fmt.Errorf("'%s' is not an oci source, only support oci source", ociUrl) + } + + var tag string + if source.Oci.Tag == "" { + tag = kMod.ModFile.Pkg.Version + } else { + tag = source.Oci.Tag + } + + ociOpts, err := c.ParseOciOptionFromString(ociUrl, tag) + if err != nil { + return err + } + c.fillDefaultPushOptions(ociOpts, kMod) + ociOpts.Annotations, err = kMod.GenOciManifestFromPkg() if err != nil { return err @@ -130,8 +135,7 @@ func (c *KpmClient) Push(opts ...PushOption) error { // PushToOci will push a kcl package to oci registry. func (c *KpmClient) pushToOci(localPath string, ociOpts *opt.OciOptions) error { - repoPath := utils.JoinPath(ociOpts.Reg, ociOpts.Repo) - repoPath = utils.JoinPath(repoPath, ociOpts.Ref) + repoPath := utils.JoinPath(ociOpts.Reg, ociOpts.Repo, ociOpts.Ref) cred, err := c.GetCredentials(ociOpts.Reg) if err != nil { return err diff --git a/pkg/client/push_test.go b/pkg/client/push_test.go index d360ffec..ff5f6525 100644 --- a/pkg/client/push_test.go +++ b/pkg/client/push_test.go @@ -8,7 +8,6 @@ import ( "github.com/stretchr/testify/assert" "kcl-lang.io/kpm/pkg/downloader" "kcl-lang.io/kpm/pkg/mock" - "kcl-lang.io/kpm/pkg/opt" pkg "kcl-lang.io/kpm/pkg/package" "kcl-lang.io/kpm/pkg/reporter" ) @@ -41,10 +40,14 @@ func TestPush(t *testing.T) { err = kpmcli.Push( WithPushModPath(pushedModPath), - WithPushOciOptions(&opt.OciOptions{ - Reg: "localhost:5001", - Repo: "test", - }), + WithPushSource( + downloader.Source{ + Oci: &downloader.Oci{ + Reg: "localhost:5001", + Repo: "test/push_0", + }, + }, + ), ) if err != (*reporter.KpmEvent)(nil) { diff --git a/pkg/cmd/cmd_push.go b/pkg/cmd/cmd_push.go index fff6661d..51aa7b2e 100644 --- a/pkg/cmd/cmd_push.go +++ b/pkg/cmd/cmd_push.go @@ -11,6 +11,7 @@ import ( "github.com/urfave/cli/v2" "kcl-lang.io/kpm/pkg/client" + "kcl-lang.io/kpm/pkg/downloader" "kcl-lang.io/kpm/pkg/errors" "kcl-lang.io/kpm/pkg/oci" "kcl-lang.io/kpm/pkg/opt" @@ -165,16 +166,18 @@ func pushPackage(ociUrl string, kclPkg *pkg.KclPkg, vendorMode bool, kpmcli *cli ) } - ociOpts.Annotations, err = kclPkg.GenOciManifestFromPkg() - if err != nil { - return err - } - - reporter.ReportMsgTo(fmt.Sprintf("package '%s' will be pushed", kclPkg.GetPkgName()), kpmcli.GetLogWriter()) // Push it. err = kpmcli.Push( client.WithPushModPath(kclPkg.HomePath), - client.WithPushOciOptions(ociOpts), + client.WithPushSource( + downloader.Source{ + Oci: &downloader.Oci{ + Reg: ociOpts.Reg, + Repo: utils.JoinPath(ociOpts.Repo, ociOpts.Ref), + Tag: ociOpts.Tag, + }, + }, + ), client.WithPushVendorMode(vendorMode), ) if err != (*reporter.KpmEvent)(nil) { diff --git a/pkg/opt/opt.go b/pkg/opt/opt.go index 78e34e9f..31d184c6 100644 --- a/pkg/opt/opt.go +++ b/pkg/opt/opt.go @@ -7,6 +7,7 @@ import ( "io" "net/url" "os" + gpath "path" "path/filepath" "strings" @@ -467,9 +468,18 @@ func ParseOciUrl(ociUrl string) (*OciOptions, *reporter.KpmEvent) { return nil, reporter.NewEvent(reporter.UrlSchemeNotOci) } + repo, ref := gpath.Split(u.Path) + repo = strings.TrimPrefix(strings.TrimSuffix(repo, "/"), "/") + + if repo == "" { + repo = strings.TrimPrefix(strings.TrimSuffix(u.Path, "/"), "/") + ref = "" + } + return &OciOptions{ Reg: u.Host, - Repo: u.Path, + Repo: repo, + Ref: ref, }, nil } @@ -491,7 +501,7 @@ func (oci *OciOptions) AddStoragePathSuffix(pathPrefix string) string { // SanitizePathSuffix will take 'Registry/Repo/Tag' as a path suffix and sanitize it. func (oci *OciOptions) SanitizePathWithSuffix(pathPrefix string) string { - return path.SanitizePath(filepath.Join(filepath.Join(filepath.Join(pathPrefix, oci.Reg), oci.Repo), oci.Tag)) + return path.SanitizePath(filepath.Join(pathPrefix, oci.Reg, oci.Repo, oci.Ref, oci.Tag)) } type OciManifestOptions struct { diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 61508ba9..baa5b1c0 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -552,10 +552,21 @@ func RmNewline(s string) string { } // JoinPath will join the 'elem' to the 'base' with '/'. -func JoinPath(base, elem string) string { - base = strings.TrimSuffix(base, "/") - elem = strings.TrimPrefix(elem, "/") - return base + "/" + elem +func JoinPath(elems ...string) string { + var result string + for i, elem := range elems { + if elem == "" { + continue + } + elem = strings.TrimPrefix(elem, "/") + elem = strings.TrimSuffix(elem, "/") + if i == 0 { + result = elem + } else { + result = result + "/" + elem + } + } + return result } // IsUrl will check whether the string 'str' is a url. diff --git a/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url/test_suite.stdout b/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url/test_suite.stdout index b6cf454b..2cc0b57c 100644 --- a/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url/test_suite.stdout +++ b/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url/test_suite.stdout @@ -1,4 +1,4 @@ start to pull 'oci://localhost:5001/test/k8s' the latest version '1.31.2' will be pulled -pulling '/test/k8s:1.31.2' from 'localhost:5001/test/k8s' +pulling 'test/k8s:1.31.2' from 'localhost:5001/test/k8s' pulled 'oci://localhost:5001/test/k8s' in '/localhost:5001/test/k8s/1.31.2' successfully \ No newline at end of file diff --git a/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url_tag/test_suite.stdout b/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url_tag/test_suite.stdout index 3656dc86..369fe05d 100644 --- a/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url_tag/test_suite.stdout +++ b/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url_tag/test_suite.stdout @@ -1,3 +1,3 @@ start to pull 'oci://localhost:5001/test/k8s' with tag '1.14' -pulling '/test/k8s:1.14' from 'localhost:5001/test/k8s' +pulling 'test/k8s:1.14' from 'localhost:5001/test/k8s' pulled 'oci://localhost:5001/test/k8s' in '/localhost:5001/test/k8s/1.14' successfully \ No newline at end of file diff --git a/test/e2e/test_suites/kpm/exec_outside_pkg/pull_private_pkg/test_suite.stdout b/test/e2e/test_suites/kpm/exec_outside_pkg/pull_private_pkg/test_suite.stdout index 2d980a93..46493f6e 100644 --- a/test/e2e/test_suites/kpm/exec_outside_pkg/pull_private_pkg/test_suite.stdout +++ b/test/e2e/test_suites/kpm/exec_outside_pkg/pull_private_pkg/test_suite.stdout @@ -1,4 +1,4 @@ start to pull 'oci://localhost:5001/test/helloworld' the latest version '0.1.2' will be pulled -pulling '/test/helloworld:0.1.2' from 'localhost:5001/test/helloworld' +pulling 'test/helloworld:0.1.2' from 'localhost:5001/test/helloworld' pulled 'oci://localhost:5001/test/helloworld' in '/localhost:5001/test/helloworld/0.1.2' successfully \ No newline at end of file diff --git a/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_url/test_suite.stderr b/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_url/test_suite.stderr index 7dfb5555..545b0fd2 100644 --- a/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_url/test_suite.stderr +++ b/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_url/test_suite.stderr @@ -1 +1 @@ -repository 'invalid_url/' not found \ No newline at end of file +repository 'invalid_url' not found \ No newline at end of file diff --git a/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_url_without_tag/test_suite.stderr b/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_url_without_tag/test_suite.stderr index affa8704..de1ddacb 100644 --- a/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_url_without_tag/test_suite.stderr +++ b/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_url_without_tag/test_suite.stderr @@ -1 +1 @@ -repository 'invalid_url/' not found +repository 'invalid_url' not found From 8b34ba011d75ae6ebcdc07c7174591f905a11207 Mon Sep 17 00:00:00 2001 From: zongz Date: Wed, 4 Dec 2024 17:39:10 +0800 Subject: [PATCH 07/10] fix: fix unit test TestModCheckerCheck_WithTrustedSum Signed-off-by: zongz --- pkg/mock/test_script/push_pkg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/mock/test_script/push_pkg.sh b/pkg/mock/test_script/push_pkg.sh index 64a41530..b6280ff0 100755 --- a/pkg/mock/test_script/push_pkg.sh +++ b/pkg/mock/test_script/push_pkg.sh @@ -29,4 +29,4 @@ echo $current_dir # Push the test_data package to the registry cd "$SCRIPT_DIR/../test_data" -"$current_dir/bin/kpm" push oci://$KPM_REG/$KPM_REPO +"$current_dir/bin/kpm" push oci://$KPM_REG/$KPM_REPO/test_data From 5d0c935c622cff0ac75363a52b60ea97f24c9d52 Mon Sep 17 00:00:00 2001 From: zongz Date: Wed, 4 Dec 2024 17:44:35 +0800 Subject: [PATCH 08/10] fix: fix unit test TestJoinPath Signed-off-by: zongz --- pkg/utils/utils.go | 2 +- pkg/utils/utils_test.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index baa5b1c0..50e6d803 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -560,7 +560,7 @@ func JoinPath(elems ...string) string { } elem = strings.TrimPrefix(elem, "/") elem = strings.TrimSuffix(elem, "/") - if i == 0 { + if i == 0 || result == "" { result = elem } else { result = result + "/" + elem diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go index dbf87bd5..129390cd 100644 --- a/pkg/utils/utils_test.go +++ b/pkg/utils/utils_test.go @@ -173,10 +173,10 @@ func TestJoinPath(t *testing.T) { assert.Equal(t, JoinPath("base", "elem"), "base/elem") assert.Equal(t, JoinPath("base/", "elem"), "base/elem") assert.Equal(t, JoinPath("base", "/elem"), "base/elem") - assert.Equal(t, JoinPath("", "/elem"), "/elem") - assert.Equal(t, JoinPath("", "elem"), "/elem") - assert.Equal(t, JoinPath("base/", ""), "base/") - assert.Equal(t, JoinPath("base", ""), "base/") + assert.Equal(t, JoinPath("", "elem"), "elem") + assert.Equal(t, JoinPath("", "elem"), "elem") + assert.Equal(t, JoinPath("base/", ""), "base") + assert.Equal(t, JoinPath("base", ""), "base") } func TestIsUrl(t *testing.T) { From 2dbd2c1b8da518de42a532becd35cd1f84b346bf Mon Sep 17 00:00:00 2001 From: zongz Date: Wed, 4 Dec 2024 17:48:01 +0800 Subject: [PATCH 09/10] fix: fix unit test TestParseOciOptionFromString Signed-off-by: zongz --- pkg/client/client_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 247c6de2..977fcb04 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -1029,7 +1029,7 @@ func TestParseOciOptionFromString(t *testing.T) { assert.Equal(t, err, nil) assert.Equal(t, ociOption.Ref, "") assert.Equal(t, ociOption.Reg, "test_reg") - assert.Equal(t, ociOption.Repo, "/test_oci_repo") + assert.Equal(t, ociOption.Repo, "test_oci_repo") assert.Equal(t, ociOption.Tag, "test_tag") } From 67662acd5c309129e6419d9ddc2fe86cbf0f2880 Mon Sep 17 00:00:00 2001 From: zongz Date: Wed, 4 Dec 2024 18:36:45 +0800 Subject: [PATCH 10/10] fix: fix windows test case Signed-off-by: zongz --- pkg/client/issues_test.go | 4 +++- pkg/client/push_test.go | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/client/issues_test.go b/pkg/client/issues_test.go index a6babb5e..19dba65f 100644 --- a/pkg/client/issues_test.go +++ b/pkg/client/issues_test.go @@ -436,7 +436,9 @@ func TestKpmIssue226(t *testing.T) { func TestKclIssue1768(t *testing.T) { testPath := "github.com/kcl-lang/kcl/issues/1768" test_push_with_tag := func(t *testing.T, kpmcli *KpmClient) { - + if runtime.GOOS == "windows" { + t.Skip("Skipping test on Windows") + } err := mock.StartDockerRegistry() if err != nil { t.Errorf("Error starting docker registry: %v", err) diff --git a/pkg/client/push_test.go b/pkg/client/push_test.go index ff5f6525..2aa2509e 100644 --- a/pkg/client/push_test.go +++ b/pkg/client/push_test.go @@ -3,6 +3,7 @@ package client import ( "bytes" "path/filepath" + "runtime" "testing" "github.com/stretchr/testify/assert" @@ -14,6 +15,9 @@ import ( func TestPush(t *testing.T) { testFunc := func(t *testing.T, kpmcli *KpmClient) { + if runtime.GOOS == "windows" { + t.Skip("Skipping test on Windows") + } err := mock.StartDockerRegistry() if err != nil { t.Errorf("Error starting docker registry: %v", err)