Skip to content

Commit

Permalink
feat: support new file storage for Add with ModSpec
Browse files Browse the repository at this point in the history 
Signed-off-by: zongz <[email protected]>
  • Loading branch information
@zong-zhe
zong-zhe committed Nov 5, 2024
1 parent 58f2058 commit ff54d0d
Show file tree
Hide file tree
Showing 10 changed files with 284 additions and 148 deletions.
17 changes: 15 additions & 2 deletions pkg/client/client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"kcl-lang.io/kcl-go/pkg/kcl"
"kcl-lang.io/kpm/pkg/downloader"
"kcl-lang.io/kpm/pkg/env"
"kcl-lang.io/kpm/pkg/features"
"kcl-lang.io/kpm/pkg/opt"
pkg "kcl-lang.io/kpm/pkg/package"
"kcl-lang.io/kpm/pkg/reporter"
Expand Down Expand Up @@ -82,6 +83,18 @@ func TestWithGlobalLock(t *testing.T) {
test.RunTestWithGlobalLock(t, "TestDownloadGitWithPackage", testDownloadGitWithPackage)
test.RunTestWithGlobalLock(t, "TestModandLockFilesWithGitPackageDownload", testModandLockFilesWithGitPackageDownload)
test.RunTestWithGlobalLock(t, "TestDependencyGraph", testDependencyGraph)
test.RunTestWithGlobalLock(t, "TestAddWithModSpec", testAddWithModSpec)
test.RunTestWithGlobalLock(t, "TestRunRemoteWithArgsInvalid", testRunRemoteWithArgsInvalid)
test.RunTestWithGlobalLock(t, "TestRunRemoteWithArgs", testRunRemoteWithArgs)
test.RunTestWithGlobalLock(t, "TestRunWithNoSumCheck", testRunWithGitPackage)
test.RunTestWithGlobalLock(t, "TestRunGit", testRunGit)
test.RunTestWithGlobalLock(t, "TestRunOciWithSettingsFile", testRunOciWithSettingsFile)
test.RunTestWithGlobalLock(t, "TestVendorWithGlobalLock", testVendorWithGlobalLock)
test.RunTestWithGlobalLock(t, "TestPull", testPull)
test.RunTestWithGlobalLock(t, "TestPullWithInsecureSkipTLSverify", testPullWithInsecureSkipTLSverify)
test.RunTestWithGlobalLock(t, "TestPullWithModSpec", testPullWithModSpec)

features.Enable(features.SupportNewStorage)
test.RunTestWithGlobalLock(t, "testAddWithModSpec", testAddWithModSpec)
}

Expand Down Expand Up @@ -2004,7 +2017,7 @@ func TestRunLocalWithArgs(t *testing.T) {
}
}

func TestRunRemoteWithArgsInvalid(t *testing.T) {
func testRunRemoteWithArgsInvalid(t *testing.T) {
kpmcli, err := NewKpmClient()
assert.Equal(t, err, nil)

Expand Down Expand Up @@ -2033,7 +2046,7 @@ func TestRunRemoteWithArgsInvalid(t *testing.T) {
}
}

func TestRunRemoteWithArgs(t *testing.T) {
func testRunRemoteWithArgs(t *testing.T) {
pkgPath := getTestDir("test_run_options")
kpmcli, err := NewKpmClient()
assert.Equal(t, err, nil)
Expand Down
6 changes: 3 additions & 3 deletions pkg/client/deperated_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ func TestRunWithNoSumCheck(t *testing.T) {
}()
}

func TestRunWithGitPackage(t *testing.T) {
func testRunWithGitPackage(t *testing.T) {
pkgPath := getTestDir("test_run_git_package")

kpmcli, err := NewKpmClient()
Expand Down Expand Up @@ -84,7 +84,7 @@ func testRunWithOciDownloader(t *testing.T) {
assert.Equal(t, res.GetRawYamlResult(), "The_first_kcl_program: Hello World!")
}

func TestRunGit(t *testing.T) {
func testRunGit(t *testing.T) {
kpmcli, err := NewKpmClient()
assert.Equal(t, err, nil)

Expand All @@ -107,7 +107,7 @@ func TestRunGit(t *testing.T) {
assert.Equal(t, utils.RmNewline(string(bytes)), utils.RmNewline(string(resultStr)))
}

func TestRunOciWithSettingsFile(t *testing.T) {
func testRunOciWithSettingsFile(t *testing.T) {
kpmcli, err := NewKpmClient()
assert.Equal(t, err, nil)
kpmcli.SetLogWriter(nil)
Expand Down
6 changes: 3 additions & 3 deletions pkg/client/pull_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (
"kcl-lang.io/kpm/pkg/downloader"
)

func TestPull(t *testing.T) {
func testPull(t *testing.T) {
pulledPath := getTestDir("test_pull")
defer func() {
err := os.RemoveAll(filepath.Join(pulledPath, "oci"))
Expand Down Expand Up @@ -62,7 +62,7 @@ func TestPull(t *testing.T) {
}()
}

func TestPullWithInsecureSkipTLSverify(t *testing.T) {
func testPullWithInsecureSkipTLSverify(t *testing.T) {
pulledPath := getTestDir("test_pull")

kpmcli, err := NewKpmClient()
Expand Down Expand Up @@ -128,7 +128,7 @@ func TestInsecureSkipTLSverifyOCIRegistry(t *testing.T) {
assert.Equal(t, buf.String(), "Called Success\n")
}

func TestPullWithModSpec(t *testing.T) {
func testPullWithModSpec(t *testing.T) {
pulledPath := getTestDir("test_pull_with_modspec")
defer func() {
err := os.RemoveAll(filepath.Join(pulledPath, "oci"))
Expand Down
2 changes: 1 addition & 1 deletion pkg/client/vendor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ func testVendorWithMVS(t *testing.T) {
assert.Equal(t, utils.DirExists(filepath.Join(pkgPath, "vendor", "helloworld_0.1.1")), false)
}

func TestVendorWithGlobalLock(t *testing.T) {
func testVendorWithGlobalLock(t *testing.T) {
test.RunTestWithGlobalLock(t, "TestVendorDeps", testVendorDeps)
test.RunTestWithGlobalLock(t, "TestVendorWithMVS", testVendorWithMVS)
}
219 changes: 133 additions & 86 deletions pkg/downloader/downloader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"os"
"path/filepath"

gogit "github.com/go-git/go-git/v5"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/otiai10/copy"
"kcl-lang.io/kpm/pkg/constants"
Expand Down Expand Up @@ -139,34 +140,68 @@ func (d *GitDownloader) LatestVersion(opts *DownloadOptions) (string, error) {
// TODO：supports fetch the latest commit from the git bare repo,
// after totally transfer to the new storage.
// refer to cargo: https://github.com/rust-lang/cargo/blob/3dedb85a25604bdbbb8d3bf4b03162961a4facd0/crates/cargo-util-schemas/src/core/source_kind.rs#L133
var err error
tmp, err := os.MkdirTemp("", "")
if err != nil {
return "", err
}
tmp = filepath.Join(tmp, constants.GitScheme)

defer func() {
err = os.RemoveAll(tmp)
}()

repo, err := git.CloneWithOpts(
git.WithCommit(opts.Source.Commit),
git.WithBranch(opts.Source.Branch),
git.WithTag(opts.Source.Git.Tag),
git.WithRepoURL(opts.Source.Git.Url),
git.WithLocalPath(tmp),
)
var repo *gogit.Repository
if ok, err := features.Enabled(features.SupportNewStorage); err == nil && !ok && opts.EnableCache {
var err error
tmp, err := os.MkdirTemp("", "")
if err != nil {
return "", err
}
tmp = filepath.Join(tmp, constants.GitScheme)

defer func() {
err = os.RemoveAll(tmp)
}()

repo, err = git.CloneWithOpts(
git.WithCommit(opts.Source.Commit),
git.WithBranch(opts.Source.Branch),
git.WithTag(opts.Source.Git.Tag),
git.WithRepoURL(opts.Source.Git.Url),
git.WithLocalPath(tmp),
)

if err != nil {
return "", err
}
if err != nil {
return "", err
}
} else {
// Get the latest commit from the git repository cache
cacheFullPath := opts.CachePath
// If the cache bare git repository exists, fetch the latest commit from the cache.
if git.IsGitBareRepo(cacheFullPath) {
err := git.Fetch(cacheFullPath)
if err != nil {
return "", err
}
repo, err = gogit.PlainOpen(cacheFullPath)
if err != nil {
return "", err
}
} else {
// If not, clone the bare repository from the remote git repository, update the cache.
cloneOpts := []git.CloneOption{
git.WithCommit(opts.Source.Git.Commit),
git.WithBranch(opts.Source.Git.Branch),
git.WithTag(opts.Source.Git.Tag),
}

repo, err = git.CloneWithOpts(
append(
cloneOpts,
git.WithRepoURL(opts.Source.Git.Url),
git.WithLocalPath(cacheFullPath),
git.WithBare(true),
)...,
)
if err != nil {
return "", err
}
}
}
ref, err := repo.Head()
if err != nil {
return "", err
}

commit, err := repo.CommitObject(ref.Hash())
if err != nil {
return "", err
Expand Down Expand Up @@ -240,7 +275,6 @@ func (d *DepDownloader) Download(opts *DownloadOptions) error {
localPath := opts.LocalPath
cacheFullPath := opts.CachePath
if ok, err := features.Enabled(features.SupportNewStorage); err == nil && !ok && opts.EnableCache {
cacheFullPath = filepath.Join(opts.CachePath, opts.Source.LocalPath())
if utils.DirExists(cacheFullPath) && utils.DirExists(filepath.Join(cacheFullPath, constants.KCL_MOD)) &&
// If the version in modspec is empty, meanings the latest version is needed.
// The latest version should be requested first and the cache should be updated.
Expand All @@ -261,49 +295,56 @@ func (d *DepDownloader) Download(opts *DownloadOptions) error {
}
}

opts.LocalPath = tmpDir
// Dispatch the download to the specific downloader by package source.
if opts.Source.Oci != nil {
if d.OciDownloader == nil {
d.OciDownloader = &OciDownloader{}
}
err := d.OciDownloader.Download(opts)
if err != nil {
return err
// If the dependency package is already exist,
// Skip the download process.
if utils.DirExists(localPath) &&
utils.DirExists(filepath.Join(localPath, constants.KCL_MOD)) {
return nil
} else {
opts.LocalPath = tmpDir
// Dispatch the download to the specific downloader by package source.
if opts.Source.Oci != nil {
if d.OciDownloader == nil {
d.OciDownloader = &OciDownloader{}
}
err := d.OciDownloader.Download(opts)
if err != nil {
return err
}
}
}

if opts.Source.Git != nil {
if d.GitDownloader == nil {
d.GitDownloader = &GitDownloader{}
if opts.Source.Git != nil {
if d.GitDownloader == nil {
d.GitDownloader = &GitDownloader{}
}
err := d.GitDownloader.Download(opts)
if err != nil {
return err
}
}
err := d.GitDownloader.Download(opts)
if err != nil {
return err

// rename the tmp dir to the local path.
if utils.DirExists(localPath) {
err := os.RemoveAll(localPath)
if err != nil {
return err
}
}
}

// rename the tmp dir to the local path.
if utils.DirExists(localPath) {
err := os.RemoveAll(localPath)
// Move the downloaded package to the local path.
// On unix, after the move, the tmp dir will be removed.
err = utils.MoveOrCopy(tmpDir, localPath)
if err != nil {
return err
}
}

// Move the downloaded package to the local path.
// On unix, after the move, the tmp dir will be removed.
err = utils.MoveOrCopy(tmpDir, localPath)
if err != nil {
return err
}

if ok, err := features.Enabled(features.SupportNewStorage); err == nil && !ok && opts.EnableCache {
// Enable the cache, update the dependency package to the cache path.
if cacheFullPath != localPath {
err := copy.Copy(localPath, cacheFullPath)
if err != nil {
return err
if ok, err := features.Enabled(features.SupportNewStorage); err == nil && !ok && opts.EnableCache {
// Enable the cache, update the dependency package to the cache path.
if cacheFullPath != localPath {
err := copy.Copy(localPath, cacheFullPath)
if err != nil {
return err
}
}
}
}
Expand Down Expand Up @@ -368,12 +409,8 @@ func (d *OciDownloader) Download(opts *DownloadOptions) error {

if ok, err := features.Enabled(features.SupportNewStorage); err == nil && ok {
if opts.EnableCache {
hash, err := ociSource.Hash()
if err != nil {
return err
}
cacheFullPath := filepath.Join(opts.CachePath, hash)
localFullPath := filepath.Join(opts.LocalPath, hash)
cacheFullPath := opts.CachePath
localFullPath := opts.LocalPath

if utils.DirExists(localFullPath) &&
utils.DirExists(filepath.Join(localFullPath, constants.KCL_MOD)) {
Expand Down Expand Up @@ -463,13 +500,8 @@ func (d *GitDownloader) Download(opts *DownloadOptions) error {

if ok, err := features.Enabled(features.SupportNewStorage); err == nil && ok {
if opts.EnableCache {
hash, err := gitSource.Hash()
if err != nil {
return err
}

cacheFullPath := filepath.Join(opts.CachePath, hash)
localFullPath := filepath.Join(opts.LocalPath, hash)
cacheFullPath := opts.CachePath
localFullPath := opts.LocalPath
// Check if the package is already downloaded, if so, skip the download.
if utils.DirExists(localFullPath) &&
utils.DirExists(filepath.Join(localFullPath, constants.KCL_MOD)) {
Expand All @@ -486,30 +518,45 @@ func (d *GitDownloader) Download(opts *DownloadOptions) error {
// If failed to clone the bare repository from the cache path,
// clone the bare repository from the remote git repository, update the cache.
if err != nil {
_, err := git.CloneWithOpts(
// If the bare repository cache exists, fetch the latest commit from the cache.
if utils.DirExists(cacheFullPath) && git.IsGitBareRepo(cacheFullPath) {
err := git.Fetch(cacheFullPath)
if err != nil {
return err
}
} else {
// If not, clone the bare repository from the remote git repository, update the cache.
if utils.DirExists(cacheFullPath) {
err = os.Remove(cacheFullPath)
if err != nil {
return err
}
}
_, err := git.CloneWithOpts(
append(
cloneOpts,
git.WithRepoURL(gitSource.Url),
git.WithLocalPath(cacheFullPath),
git.WithBare(true),
)...,
)
if err != nil {
return err
}
}
// After cloning the bare repository,
// Clone the repository from the cache path to the local path.
_, err = git.CloneWithOpts(
append(
cloneOpts,
git.WithRepoURL(gitSource.Url),
git.WithLocalPath(cacheFullPath),
git.WithBare(true),
git.WithRepoURL(cacheFullPath),
git.WithLocalPath(localFullPath),
)...,
)
if err != nil {
return err
}
}
// After cloning the bare repository,
// Clone the repository from the cache path to the local path.
_, err = git.CloneWithOpts(
append(
cloneOpts,
git.WithRepoURL(cacheFullPath),
git.WithLocalPath(localFullPath),
)...,
)
if err != nil {
return err
}
}
}
} else {
Expand Down
Loading

0 comments on commit ff54d0d

Please sign in to comment.