feat: add 'kpm update' to update the dependencies (#212)

* feat: add 'kpm update' to update the dependencies Signed-off-by: zongz <[email protected]> * fix: fix windows case Signed-off-by: zongz <[email protected]> * fix: fix CR comments Signed-off-by: zongz <[email protected]> * fix: fix type '// /' to '//' Signed-off-by: zongz <[email protected]> --------- Signed-off-by: zongz <[email protected]>
kcl-lang · Nov 14, 2023 · 1b58fe5 · 1b58fe5
1 parent 703dd7c
commit 1b58fe5
Show file tree

Hide file tree

Showing 27 changed files with 317 additions and 15 deletions.
diff --git a/go.mod b/go.mod
@@ -47,6 +47,7 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.3 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/iancoleman/strcase v0.3.0 // indirect
@@ -81,11 +82,13 @@ require (
 	golang.org/x/crypto v0.7.0 // indirect
 	golang.org/x/mod v0.9.0 // indirect
 	golang.org/x/net v0.9.0 // indirect
+	golang.org/x/oauth2 v0.6.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
 	golang.org/x/sys v0.7.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/tools v0.7.0 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect
 	google.golang.org/grpc v1.53.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
@@ -99,6 +102,7 @@ require (
 	github.com/go-git/go-git/v5 v5.6.1
 	github.com/gofrs/flock v0.8.1
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
+	github.com/google/go-github/v50 v50.2.0
 	github.com/hashicorp/go-version v1.6.0
 	github.com/moby/term v0.0.0-20221205130635-1aeaba878587
 	github.com/onsi/ginkgo/v2 v2.9.2

diff --git a/go.sum b/go.sum
@@ -210,10 +210,15 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-github/v50 v50.2.0 h1:j2FyongEHlO9nxXLc+LP3wuBSVU9mVxfpdYUexMpIfk=
+github.com/google/go-github/v50 v50.2.0/go.mod h1:VBY8FB6yPIjrtKhozXv4FQupxKLS6H4m6xFZlT43q8Q=
+github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
+github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@@ -511,6 +516,8 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
+golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw=
+golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -674,6 +681,8 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=

diff --git a/kpm.go b/kpm.go
@@ -37,6 +37,7 @@ func main() {
 		cmd.NewLogoutCmd(kpmcli),
 		cmd.NewPushCmd(kpmcli),
 		cmd.NewPullCmd(kpmcli),
+		cmd.NewUpdateCmd(kpmcli),
 	}
 	app.Flags = []cli.Flag{
 		&cli.BoolFlag{

diff --git a/pkg/api/test_data/test_run_pkg_in_path/test_kcl/kcl.mod b/pkg/api/test_data/test_run_pkg_in_path/test_kcl/kcl.mod
@@ -4,4 +4,4 @@ edition = "0.0.1"
 version = "0.0.1"
 
 [dependencies]
-konfig = { git = "https://github.com/awesome-kusion/konfig.git", tag = "v0.0.1" }
+konfig = { git = "https://github.com/awesome-kusion/konfig.git", tag = "v0.0.1" }
diff --git a/pkg/client/client.go b/pkg/client/client.go
@@ -9,6 +9,7 @@ import (
 	"reflect"
 	"strings"
 
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/otiai10/copy"
 	"kcl-lang.io/kcl-go/pkg/kcl"
 	"kcl-lang.io/kpm/pkg/constants"
@@ -22,6 +23,7 @@ import (
 	"kcl-lang.io/kpm/pkg/runner"
 	"kcl-lang.io/kpm/pkg/settings"
 	"kcl-lang.io/kpm/pkg/utils"
+	"oras.land/oras-go/v2"
 )
 
 // KpmClient is the client of kpm.
@@ -82,6 +84,50 @@ func (c *KpmClient) GetSettings() *settings.Settings {
 	return &c.settings
 }
 
+func (c *KpmClient) LoadPkgFromPath(pkgPath string) (*pkg.KclPkg, error) {
+	modFile, err := c.LoadModFile(pkgPath)
+	if err != nil {
+		return nil, reporter.NewErrorEvent(reporter.FailedLoadKclMod, err, fmt.Sprintf("could not load 'kcl.mod' in '%s'.", pkgPath))
+	}
+
+	// Get dependencies from kcl.mod.lock.
+	deps, err := c.LoadLockDeps(pkgPath)
+
+	if err != nil {
+		return nil, reporter.NewErrorEvent(reporter.FailedLoadKclMod, err, fmt.Sprintf("could not load 'kcl.mod.lock' in '%s'.", pkgPath))
+	}
+
+	return &pkg.KclPkg{
+		ModFile:      *modFile,
+		HomePath:     pkgPath,
+		Dependencies: *deps,
+	}, nil
+}
+
+func (c *KpmClient) LoadModFile(pkgPath string) (*pkg.ModFile, error) {
+	modFile := new(pkg.ModFile)
+	err := modFile.LoadModFile(filepath.Join(pkgPath, pkg.MOD_FILE))
+	if err != nil {
+		return nil, err
+	}
+
+	modFile.HomePath = pkgPath
+
+	if modFile.Dependencies.Deps == nil {
+		modFile.Dependencies.Deps = make(map[string]pkg.Dependency)
+	}
+	err = c.FillDependenciesInfo(modFile)
+	if err != nil {
+		return nil, err
+	}
+
+	return modFile, nil
+}
+
+func (c *KpmClient) LoadLockDeps(pkgPath string) (*pkg.Dependencies, error) {
+	return pkg.LoadLockDeps(pkgPath)
+}
+
 // ResolveDepsIntoMap will calculate the map of kcl package name and local storage path of the external packages.
 func (c *KpmClient) ResolveDepsIntoMap(kclPkg *pkg.KclPkg) (map[string]string, error) {
 	err := c.ResolvePkgDepsMetadata(kclPkg, true)
@@ -115,8 +161,29 @@ func (c *KpmClient) ResolvePkgDepsMetadata(kclPkg *pkg.KclPkg, update bool) erro
 	}
 
 	// alian the dependencies between kcl.mod and kcl.mod.lock
+	// clean the dependencies in kcl.mod.lock which not in kcl.mod
+	for name := range kclPkg.Dependencies.Deps {
+		if _, ok := kclPkg.ModFile.Dependencies.Deps[name]; !ok {
+			reporter.ReportEventTo(
+				reporter.NewEvent(
+					reporter.RemoveDep,
+					fmt.Sprintf("removing '%s'", name),
+				),
+				c.logWriter,
+			)
+			delete(kclPkg.Dependencies.Deps, name)
+		}
+	}
+	// add the dependencies in kcl.mod which not in kcl.mod.lock
 	for name, d := range kclPkg.ModFile.Dependencies.Deps {
 		if _, ok := kclPkg.Dependencies.Deps[name]; !ok {
+			reporter.ReportEventTo(
+				reporter.NewEvent(
+					reporter.AddDep,
+					fmt.Sprintf("adding '%s'", name),
+				),
+				c.logWriter,
+			)
 			kclPkg.Dependencies.Deps[name] = d
 		}
 	}
@@ -176,6 +243,20 @@ func (c *KpmClient) ResolvePkgDepsMetadata(kclPkg *pkg.KclPkg, update bool) erro
 	return nil
 }
 
+// UpdateDeps will update the dependencies.
+func (c *KpmClient) UpdateDeps(kclPkg *pkg.KclPkg) error {
+	_, err := c.ResolveDepsMetadataInJsonStr(kclPkg, true)
+	if err != nil {
+		return err
+	}
+
+	err = kclPkg.UpdateModAndLockFile()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 // ResolveDepsMetadataInJsonStr will calculate the local storage path of the external package,
 // and check whether the package exists locally. If the package does not exist, it will re-download to the local.
 // Finally, the calculated metadata of the dependent packages is serialized into a json string and returned.
@@ -564,10 +645,33 @@ func (c *KpmClient) VendorDeps(kclPkg *pkg.KclPkg) error {
 
 // FillDepInfo will fill registry information for a dependency.
 func (c *KpmClient) FillDepInfo(dep *pkg.Dependency) error {
-	if dep.Source.Oci != nil {
+	if dep.Source.Git == nil {
 		dep.Source.Oci.Reg = c.GetSettings().DefaultOciRegistry()
 		urlpath := utils.JoinPath(c.GetSettings().DefaultOciRepo(), dep.Name)
 		dep.Source.Oci.Repo = urlpath
+		manifest := ocispec.Manifest{}
+		jsonDesc, err := c.FetchOciManifestIntoJsonStr(opt.OciFetchOptions{
+			FetchBytesOptions: oras.DefaultFetchBytesOptions,
+			OciOptions: opt.OciOptions{
+				Reg:  c.GetSettings().DefaultOciRegistry(),
+				Repo: fmt.Sprintf("%s/%s", c.GetSettings().DefaultOciRepo(), dep.Name),
+				Tag:  dep.Version,
+			},
+		})
+
+		if err != nil {
+			return err
+		}
+
+		err = json.Unmarshal([]byte(jsonDesc), &manifest)
+		if err != nil {
+			return err
+		}
+
+		if value, ok := manifest.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_SUM]; ok {
+			dep.Sum = value
+		}
+		return nil
 	}
 	return nil
 }

diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go
@@ -8,8 +8,10 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
+	"github.com/otiai10/copy"
 	"github.com/stretchr/testify/assert"
 	"kcl-lang.io/kpm/pkg/env"
 	"kcl-lang.io/kpm/pkg/opt"
@@ -687,3 +689,64 @@ func TestParseOciOptionFromString(t *testing.T) {
 	assert.Equal(t, ociOption.Repo, "/test_oci_repo")
 	assert.Equal(t, ociOption.Tag, "test_tag")
 }
+
+func TestUpdateWithKclMod(t *testing.T) {
+	kpmcli, err := NewKpmClient()
+	assert.Equal(t, err, nil)
+
+	testDir := getTestDir("test_update")
+	src_testDir := filepath.Join(testDir, "test_update_kcl_mod")
+	dest_testDir := filepath.Join(testDir, "test_update_kcl_mod_tmp")
+	err = copy.Copy(src_testDir, dest_testDir)
+	assert.Equal(t, err, nil)
+
+	kclPkg, err := pkg.LoadKclPkg(dest_testDir)
+	assert.Equal(t, err, nil)
+	err = kpmcli.UpdateDeps(kclPkg)
+	assert.Equal(t, err, nil)
+	got_lock_file := filepath.Join(dest_testDir, "kcl.mod.lock")
+	got_content, err := os.ReadFile(got_lock_file)
+	assert.Equal(t, err, nil)
+
+	expected_path := filepath.Join(dest_testDir, "expected")
+	expected_content, err := os.ReadFile(expected_path)
+
+	assert.Equal(t, err, nil)
+	expect := strings.ReplaceAll(string(expected_content), "\r\n", "\n")
+	assert.Equal(t, string(got_content), expect)
+
+	defer func() {
+		err := os.RemoveAll(dest_testDir)
+		assert.Equal(t, err, nil)
+	}()
+}
+
+func TestUpdateWithKclModlock(t *testing.T) {
+	kpmcli, err := NewKpmClient()
+	assert.Equal(t, err, nil)
+
+	testDir := getTestDir("test_update")
+	src_testDir := filepath.Join(testDir, "test_update_kcl_mod_lock")
+	dest_testDir := filepath.Join(testDir, "test_update_kcl_mod_lock_tmp")
+	err = copy.Copy(src_testDir, dest_testDir)
+	assert.Equal(t, err, nil)
+
+	kclPkg, err := pkg.LoadKclPkg(dest_testDir)
+	assert.Equal(t, err, nil)
+	err = kpmcli.UpdateDeps(kclPkg)
+	assert.Equal(t, err, nil)
+	got_lock_file := filepath.Join(dest_testDir, "kcl.mod.lock")
+	got_content, err := os.ReadFile(got_lock_file)
+	assert.Equal(t, err, nil)
+
+	expected_path := filepath.Join(dest_testDir, "expected")
+	expected_content, err := os.ReadFile(expected_path)
+
+	assert.Equal(t, err, nil)
+	assert.Equal(t, string(got_content), string(expected_content))
+
+	defer func() {
+		err := os.RemoveAll(dest_testDir)
+		assert.Equal(t, err, nil)
+	}()
+}
diff --git a/pkg/client/test_data/test_update/test_update_kcl_mod/expected b/pkg/client/test_data/test_update/test_update_kcl_mod/expected
@@ -0,0 +1,9 @@
+[dependencies]
+  [dependencies.helloworld]
+    name = "helloworld"
+    full_name = "helloworld_0.1.1"
+    version = "0.1.1"
+    sum = "7OO4YK2QuRWPq9C7KTzcWcti5yUnueCjptT3OXiPVeQ="
+    reg = "ghcr.io"
+    repo = "kcl-lang/helloworld"
+    oci_tag = "0.1.1"
diff --git a/pkg/client/test_data/test_update/test_update_kcl_mod/kcl.mod b/pkg/client/test_data/test_update/test_update_kcl_mod/kcl.mod
@@ -0,0 +1,7 @@
+[package]
+name = "test_update"
+edition = "0.0.1"
+version = "0.0.1"
+
+[dependencies]
+helloworld = "0.1.1"
diff --git a/pkg/client/test_data/test_update/test_update_kcl_mod/kcl.mod.lock b/pkg/client/test_data/test_update/test_update_kcl_mod/kcl.mod.lock
diff --git a/pkg/client/test_data/test_update/test_update_kcl_mod/main.k b/pkg/client/test_data/test_update/test_update_kcl_mod/main.k
@@ -0,0 +1 @@
+The_first_kcl_program = 'Hello World!'
diff --git a/pkg/client/test_data/test_update/test_update_kcl_mod_lock/expected b/pkg/client/test_data/test_update/test_update_kcl_mod_lock/expected
diff --git a/pkg/client/test_data/test_update/test_update_kcl_mod_lock/kcl.mod b/pkg/client/test_data/test_update/test_update_kcl_mod_lock/kcl.mod
@@ -0,0 +1,4 @@
+[package]
+name = "test_update"
+edition = "0.0.1"
+version = "0.0.1"
diff --git a/pkg/client/test_data/test_update/test_update_kcl_mod_lock/kcl.mod.lock b/pkg/client/test_data/test_update/test_update_kcl_mod_lock/kcl.mod.lock
@@ -0,0 +1,9 @@
+[dependencies]
+  [dependencies.helloworld]
+    name = "helloworld"
+    full_name = "helloworld_0.1.1"
+    version = "0.1.1"
+    sum = "7OO4YK2QuRWPq9C7KTzcWcti5yUnueCjptT3OXiPVeQ="
+    reg = "ghcr.io"
+    repo = "kcl-lang/helloworld"
+    oci_tag = "0.1.1"
diff --git a/pkg/client/test_data/test_update/test_update_kcl_mod_lock/main.k b/pkg/client/test_data/test_update/test_update_kcl_mod_lock/main.k
@@ -0,0 +1 @@
+The_first_kcl_program = 'Hello World!'
diff --git a/pkg/cmd/cmd_push.go b/pkg/cmd/cmd_push.go
@@ -167,7 +167,11 @@ func pushPackage(ociUrl string, kclPkg *pkg.KclPkg, vendorMode bool, kpmcli *cli
 			"only support url scheme 'oci://'.",
 		)
 	}
-	ociOpts.Annotations = oci.GenOciManifestFromPkg(kclPkg)
+
+	ociOpts.Annotations, err = oci.GenOciManifestFromPkg(kclPkg)
+	if err != nil {
+		return err
+	}
 
 	reporter.ReportMsgTo(fmt.Sprintf("kpm: package '%s' will be pushed", kclPkg.GetPkgName()), kpmcli.GetLogWriter())
 	// 4. Push it.