From 4c44dedf05c300d86a025b4293f93a55f15e42a5 Mon Sep 17 00:00:00 2001 From: Nishant Bansal Date: Sat, 12 Oct 2024 23:51:25 +0530 Subject: [PATCH 1/7] Added functionality to include checksum in kcl packages Signed-off-by: Nishant Bansal --- Integrate-Checksum/main.go | 218 +++++++++++++++++++++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 Integrate-Checksum/main.go diff --git a/Integrate-Checksum/main.go b/Integrate-Checksum/main.go new file mode 100644 index 00000000..764d3999 --- /dev/null +++ b/Integrate-Checksum/main.go @@ -0,0 +1,218 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/registry/remote" + "oras.land/oras-go/v2/registry/remote/auth" + "oras.land/oras-go/v2/registry/remote/retry" + + "kcl-lang.io/kpm/pkg/client" + "kcl-lang.io/kpm/pkg/constants" + "kcl-lang.io/kpm/pkg/downloader" + "kcl-lang.io/kpm/pkg/opt" + pkg "kcl-lang.io/kpm/pkg/package" + "kcl-lang.io/kpm/pkg/utils" +) + +const KCLModFile = "kcl.mod" + +// findKCLModFiles searches the specified root directory for all kcl.mod files and returns their paths. +func findKCLModFiles(root string) ([]string, error) { + var modFilePaths []string + + err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if !info.IsDir() && info.Name() == KCLModFile { + modFilePaths = append(modFilePaths, filepath.Dir(path)) + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("error walking directory '%s': %w", root, err) + } + return modFilePaths, nil +} + +// parseMediaType extracts the media type from the manifest content. +func parseMediaType(content []byte) (string, error) { + var manifest struct { + MediaType string `json:"mediaType"` + } + if err := json.Unmarshal(content, &manifest); err != nil { + return "", fmt.Errorf("failed to unmarshal content for media type: %w", err) + } + if manifest.MediaType == "" { + return "", fmt.Errorf("media type is missing in manifest") + } + return manifest.MediaType, nil +} + +// resolveDependency loads the KCL package from a directory and constructs a dependency object with OCI source information. +func resolveDependency(kpmClient *client.KpmClient, packageDir string) (*pkg.Dependency, error) { + kclPkg, err := kpmClient.LoadPkgFromPath(packageDir) + if err != nil { + return nil, fmt.Errorf("failed to load KCL package from '%s': %w", packageDir, err) + } + + dependency := &pkg.Dependency{ + Name: kclPkg.ModFile.Pkg.Name, + Source: downloader.Source{ + Oci: &downloader.Oci{ + Reg: kpmClient.GetSettings().DefaultOciRegistry(), + Repo: utils.JoinPath(kpmClient.GetSettings().DefaultOciRepo(), kclPkg.GetPkgName()), + Tag: kclPkg.GetPkgTag(), + }, + }, + } + + if dependency.Sum, err = utils.HashDir(packageDir); err != nil { + return nil, fmt.Errorf("failed to hash directory '%s': %w", packageDir, err) + } + dependency.FromKclPkg(kclPkg) + + return dependency, nil +} + +// fetchManifest retrieves and unmarshals the OCI manifest for the given dependency. +func fetchManifest(kpmClient *client.KpmClient, dependency *pkg.Dependency) (ocispec.Manifest, error) { + var manifest ocispec.Manifest + + manifestJSON, err := kpmClient.FetchOciManifestIntoJsonStr(opt.OciFetchOptions{ + FetchBytesOptions: oras.DefaultFetchBytesOptions, + OciOptions: opt.OciOptions{ + Reg: dependency.Source.Oci.Reg, + Repo: dependency.Source.Oci.Repo, + Tag: dependency.Source.Oci.Tag, + }, + }) + if err != nil { + return manifest, fmt.Errorf("failed to fetch OCI manifest for '%s': %w", dependency.Name, err) + } + + if err := json.Unmarshal([]byte(manifestJSON), &manifest); err != nil { + return manifest, fmt.Errorf("failed to unmarshal OCI manifest: %w", err) + } + return manifest, nil +} + +// updateChecksum updates the checksum in the OCI manifest and pushes the manifest to the registry. +func updateChecksum(manifest ocispec.Manifest, kpmClient *client.KpmClient, dependency *pkg.Dependency) error { + if manifest.Annotations == nil { + manifest.Annotations = make(map[string]string) + } + manifest.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_SUM] = dependency.Sum + + repo, err := configureRepository(dependency, kpmClient) + if err != nil { + return fmt.Errorf("failed to configure repository: %w", err) + } + + manifestBytes, err := json.Marshal(manifest) + if err != nil { + return fmt.Errorf("failed to marshal updated manifest: %w", err) + } + + return tagManifest(repo, manifestBytes, dependency) +} + +// configureRepository initializes a repository reference and sets up the OCI client with credentials. +func configureRepository(dependency *pkg.Dependency, kpmClient *client.KpmClient) (*remote.Repository, error) { + repoReference := utils.JoinPath(dependency.Source.Oci.Reg, dependency.Source.Oci.Repo) + repo, err := remote.NewRepository(repoReference) + if err != nil { + return nil, fmt.Errorf("failed to create repository: %w", err) + } + + cred, err := kpmClient.GetCredentials(dependency.Source.Oci.Reg) + if err != nil { + return nil, fmt.Errorf("failed to retrieve credentials for registry '%s': %w", dependency.Source.Oci.Reg, err) + } + + repo.Client = &auth.Client{ + Client: &http.Client{ + Transport: retry.NewTransport(http.DefaultTransport.(*http.Transport).Clone()), + }, + Cache: auth.NewCache(), + Header: http.Header{"Accept": []string{"application/vnd.oci.image.manifest.v1+json"}}, + Credential: func(ctx context.Context, _ string) (auth.Credential, error) { + return *cred, nil + }, + } + + return repo, nil +} + +// tagManifest tags the updated manifest in the OCI registry. +func tagManifest(repo *remote.Repository, manifestBytes []byte, dependency *pkg.Dependency) error { + mediaType, err := parseMediaType(manifestBytes) + if err != nil { + return fmt.Errorf("failed to extract media type: %w", err) + } + + if _, err := oras.TagBytes(context.Background(), repo.Manifests(), mediaType, manifestBytes, dependency.Source.Oci.Tag); err != nil { + return fmt.Errorf("failed to tag manifest in OCI registry: %w", err) + } + + return nil +} + +// processPackage processes the package directory and updates the OCI manifest if needed. +func processPackage(packageDir string) error { + kpmClient, err := client.NewKpmClient() + if err != nil { + return fmt.Errorf("failed to create KPM client: %w", err) + } + + dependency, err := resolveDependency(kpmClient, packageDir) + if err != nil { + return fmt.Errorf("failed to resolve dependency: %w", err) + } + + manifest, err := fetchManifest(kpmClient, dependency) + if err != nil { + return fmt.Errorf("failed to fetch manifest: %w", err) + } + + if existingSum, ok := manifest.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_SUM]; ok && dependency.Sum == existingSum { + fmt.Println("Manifest already up to date with matching checksum.") + return nil + } + + if err := updateChecksum(manifest, kpmClient, dependency); err != nil { + return fmt.Errorf("failed to update checksum in manifest: %w", err) + } + + return nil +} + +func main() { + currentDir, err := os.Getwd() + if err != nil { + fmt.Printf("Error getting current directory: %v\n", err) + return + } + + modFilePaths, err := findKCLModFiles(currentDir) + if err != nil { + fmt.Printf("Error finding kcl.mod files: %v\n", err) + return + } + + for _, packageDir := range modFilePaths { + if err := processPackage(packageDir); err != nil { + fmt.Printf("Error processing package at '%s': %v\n", packageDir, err) + } + } +} From cd7fb71d6abeee452fc79aa9efb03f27289bebe0 Mon Sep 17 00:00:00 2001 From: Nishant Bansal Date: Sun, 13 Oct 2024 15:11:09 +0530 Subject: [PATCH 2/7] Added github action to run integrate checksum tool Signed-off-by: Nishant Bansal --- .github/workflows/include-kcl-checksums.yml | 31 ++++++ Integrate-Checksum/main.go | 2 + Integrate-Checksum/main_test.go | 100 +++++++++++++++++++ Integrate-Checksum/test_data/kcl.mod | 8 ++ Integrate-Checksum/test_data/kcl.mod.lock | 5 + Integrate-Checksum/test_data/main.k | 1 + mock/oci_env_mock.go | 23 +++++ mock/test_data/kcl.mod | 8 ++ mock/test_data/kcl.mod.lock | 5 + mock/test_data/main.k | 1 + mock/test_script/cleanup_test_environment.sh | 20 ++++ mock/test_script/push_pkg.sh | 31 ++++++ 12 files changed, 235 insertions(+) create mode 100644 .github/workflows/include-kcl-checksums.yml create mode 100644 Integrate-Checksum/main_test.go create mode 100644 Integrate-Checksum/test_data/kcl.mod create mode 100644 Integrate-Checksum/test_data/kcl.mod.lock create mode 100644 Integrate-Checksum/test_data/main.k create mode 100644 mock/oci_env_mock.go create mode 100644 mock/test_data/kcl.mod create mode 100644 mock/test_data/kcl.mod.lock create mode 100644 mock/test_data/main.k create mode 100755 mock/test_script/cleanup_test_environment.sh create mode 100755 mock/test_script/push_pkg.sh diff --git a/.github/workflows/include-kcl-checksums.yml b/.github/workflows/include-kcl-checksums.yml new file mode 100644 index 00000000..8a1fce9f --- /dev/null +++ b/.github/workflows/include-kcl-checksums.yml @@ -0,0 +1,31 @@ +name: Include KCL Modules Checksum + +on: + push: + +jobs: + include_modules_checksum: + runs-on: ubuntu-latest + + steps: + - name: Install kcl + run: wget -q https://kcl-lang.io/script/install-cli.sh -O - | /bin/bash + + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Login to GHCR + run: kcl registry login -u ${{ secrets.DEPLOY_ACCESS_NAME }} -p ${{ secrets.DEPLOY_ACCESS_TOKEN }} ghcr.io + + - name: Get dependencies + run: go get -v ./... + + - name: Run include checksum tool + run: go run ./Integrate-Checksum/main.go diff --git a/Integrate-Checksum/main.go b/Integrate-Checksum/main.go index 764d3999..afbd2a4e 100644 --- a/Integrate-Checksum/main.go +++ b/Integrate-Checksum/main.go @@ -215,4 +215,6 @@ func main() { fmt.Printf("Error processing package at '%s': %v\n", packageDir, err) } } + + fmt.Println("Checksum successfully included in all KCL packages") } diff --git a/Integrate-Checksum/main_test.go b/Integrate-Checksum/main_test.go new file mode 100644 index 00000000..589322fd --- /dev/null +++ b/Integrate-Checksum/main_test.go @@ -0,0 +1,100 @@ +package main + +import ( + "encoding/json" + "os" + "runtime" + "testing" + + "artifacthub/mock" + + "gotest.tools/assert" + "kcl-lang.io/kpm/pkg/client" + "kcl-lang.io/kpm/pkg/constants" +) + +// TestMainFunc tests the main functionality of the integrate checksum tool +func TestMainFunc(t *testing.T) { + // Skip the test on Windows due to platform-specific issues. + if runtime.GOOS == "windows" { + t.Skip("Skipping TestMainFunc on Windows due to platform-specific issues") + } + + // Start the local Docker registry required for testing + err := mock.StartDockerRegistry() + assert.NilError(t, err) + + // Push the test package to the local OCI registry + err = mock.PushTestPkgToRegistry() + assert.NilError(t, err) + + // Initialize the KPM client. + kpmClient, err := client.NewKpmClient() + assert.NilError(t, err, "Failed to initialize KPM client") + + // Get the current working directory. + currentDir, err := os.Getwd() + assert.NilError(t, err, "Failed to get current working directory") + + // Locate KCL module files in the current directory. + packageDirs, err := findKCLModFiles(currentDir) + assert.NilError(t, err, "Failed to locate KCL module files") + assert.Assert(t, len(packageDirs) > 0, "No KCL module files found") + + // Resolve the dependency for the first module found. + dependency, err := resolveDependency(kpmClient, packageDirs[0]) + assert.NilError(t, err, "Failed to resolve dependency") + + // Set custom OCI registry and repository for testing. + dependency.Source.Oci.Reg = "localhost:5001" + dependency.Source.Oci.Repo = "test" + + // Fetch the original manifest. + originalManifest, err := fetchManifest(kpmClient, dependency) + assert.NilError(t, err, "Failed to fetch original manifest") + + // Marshal the original manifest into JSON format. + originalManifestJSON, err := json.Marshal(originalManifest) + assert.NilError(t, err, "Failed to marshal original manifest to JSON") + + // Configure the repository for testing purposes. + repository, err := configureRepository(dependency, kpmClient) + assert.NilError(t, err, "Failed to configure repository") + repository.PlainHTTP = true // Enable plain HTTP for local testing. + + // Modify the manifest annotations for testing. + originalManifest.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_SUM] = "changes-for-testing-purpose" + + // Marshal the updated manifest into JSON format. + updatedManifestJSON, err := json.Marshal(originalManifest) + assert.NilError(t, err, "Failed to marshal updated manifest to JSON") + + // Tag the updated manifest in the repository. + err = tagManifest(repository, updatedManifestJSON, dependency) + assert.NilError(t, err, "Failed to tag updated manifest in repository") + + // Fetch the new manifest after tagging. + newManifest, err := fetchManifest(kpmClient, dependency) + assert.NilError(t, err, "Failed to fetch new manifest") + + // Marshal the new manifest into JSON format for comparison. + newManifestJSON, err := json.Marshal(newManifest) + assert.NilError(t, err, "Failed to marshal new manifest to JSON") + + // Check if the manifest was updated correctly. + if string(newManifestJSON) == string(originalManifestJSON) { + t.Errorf("Failed to update the manifest; got %v", string(originalManifestJSON)) + } + + // Revert the `Sum` field to its original value to ensure only that was changed. + newManifest.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_SUM] = dependency.Sum + newManifestJSON, err = json.Marshal(newManifest) + assert.NilError(t, err, "Failed to marshal reverted manifest to JSON") + + // Compare the new manifest data with the expected manifest data. + assert.Equal(t, string(newManifestJSON), string(originalManifestJSON), "New manifest data mismatch") + + // Clean the environment after all tests have been run + err = mock.CleanTestEnv() + assert.NilError(t, err) +} diff --git a/Integrate-Checksum/test_data/kcl.mod b/Integrate-Checksum/test_data/kcl.mod new file mode 100644 index 00000000..6c01364d --- /dev/null +++ b/Integrate-Checksum/test_data/kcl.mod @@ -0,0 +1,8 @@ +[package] +name = "test_data" +edition = "v0.9.0" +version = "0.0.1" + +[dependencies] +k8s = "1.31" + diff --git a/Integrate-Checksum/test_data/kcl.mod.lock b/Integrate-Checksum/test_data/kcl.mod.lock new file mode 100644 index 00000000..41b4867f --- /dev/null +++ b/Integrate-Checksum/test_data/kcl.mod.lock @@ -0,0 +1,5 @@ +[dependencies] + [dependencies.k8s] + name = "k8s" + full_name = "k8s_1.31" + version = "1.31" diff --git a/Integrate-Checksum/test_data/main.k b/Integrate-Checksum/test_data/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/Integrate-Checksum/test_data/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/mock/oci_env_mock.go b/mock/oci_env_mock.go new file mode 100644 index 00000000..4ebd5819 --- /dev/null +++ b/mock/oci_env_mock.go @@ -0,0 +1,23 @@ +package mock + +import ( + "os/exec" +) + +// StartDockerRegistry starts a local Docker registry by executing a shell script. +func StartDockerRegistry() error { + cmd := exec.Command("../scripts/reg.sh") + return cmd.Run() +} + +// PushTestPkgToRegistry pushes the test package to the local Docker registry. +func PushTestPkgToRegistry() error { + cmd := exec.Command("../mock/test_script/push_pkg.sh") + return cmd.Run() +} + +// CleanTestEnv cleans up the test environment by executing a cleanup script. +func CleanTestEnv() error { + cmd := exec.Command("../mock/test_script/cleanup_test_environment.sh") + return cmd.Run() +} diff --git a/mock/test_data/kcl.mod b/mock/test_data/kcl.mod new file mode 100644 index 00000000..6c01364d --- /dev/null +++ b/mock/test_data/kcl.mod @@ -0,0 +1,8 @@ +[package] +name = "test_data" +edition = "v0.9.0" +version = "0.0.1" + +[dependencies] +k8s = "1.31" + diff --git a/mock/test_data/kcl.mod.lock b/mock/test_data/kcl.mod.lock new file mode 100644 index 00000000..41b4867f --- /dev/null +++ b/mock/test_data/kcl.mod.lock @@ -0,0 +1,5 @@ +[dependencies] + [dependencies.k8s] + name = "k8s" + full_name = "k8s_1.31" + version = "1.31" diff --git a/mock/test_data/main.k b/mock/test_data/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/mock/test_data/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/mock/test_script/cleanup_test_environment.sh b/mock/test_script/cleanup_test_environment.sh new file mode 100755 index 00000000..4a244a01 --- /dev/null +++ b/mock/test_script/cleanup_test_environment.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Determine the directory where this script is located +SCRIPT_DIR="$(dirname "$(realpath "$0")")" + +# Stop and remove the Docker container then remove the Docker image +docker stop kcl-registry +docker rm kcl-registry +docker rmi registry + +# Delete all data stored in the Docker registry volume +rm -rf /var/lib/registry/* + +# Remove the directory that contains Docker authentication and related scripts +current_dir=$(pwd) +rm -rf "$current_dir/scripts/" + +# Delete the 'kcl' binary +cd "$SCRIPT_DIR/../../" +rm -rf ./bin/ diff --git a/mock/test_script/push_pkg.sh b/mock/test_script/push_pkg.sh new file mode 100755 index 00000000..999065aa --- /dev/null +++ b/mock/test_script/push_pkg.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Get the directory of the script +SCRIPT_DIR="$(dirname "$(realpath "$0")")" + +# Move to the root directory +cd "$SCRIPT_DIR/../../" + +# Install kcl binary +GOBIN=$(pwd)/bin go install kcl-lang.io/cli/cmd/kcl@latest + +# Check kpm version +version=$(./bin/kcl --version) +if ! echo "$version" ; then + echo "Incorrect version: '$version'." + exit 1 +fi + +export KPM_REG="localhost:5001" +export KPM_REPO="test" + +# Prepare the package on the registry +current_dir=$(pwd) +echo $current_dir + +# Log in to the local registry +"$current_dir/bin/kcl" registry login -u test -p 1234 localhost:5001 + +# Push the test_data package to the registry +cd "$SCRIPT_DIR/../test_data" +"$current_dir/bin/kcl" mod push oci://$KPM_REG/$KPM_REPO From a858b57262af0e0774cc522e566b9164bcb732af Mon Sep 17 00:00:00 2001 From: Nishant Bansal Date: Fri, 1 Nov 2024 00:42:12 +0530 Subject: [PATCH 3/7] fixes CI error and added more tests Signed-off-by: Nishant Bansal --- Integrate-Checksum/main_test.go | 34 +++++++++++++------- Integrate-Checksum/test_data/kcl.mod | 8 ----- Integrate-Checksum/test_data/kcl.mod.lock | 5 --- Integrate-Checksum/test_data/main.k | 1 - mock/oci_env_mock.go | 14 ++++++-- mock/test_data/kcl.mod | 6 +--- mock/test_data/kcl.mod.lock | 5 --- mock/test_script/cleanup_test_environment.sh | 3 ++ mock/test_script/pull_pkg.sh | 31 ++++++++++++++++++ 9 files changed, 69 insertions(+), 38 deletions(-) delete mode 100644 Integrate-Checksum/test_data/kcl.mod delete mode 100644 Integrate-Checksum/test_data/kcl.mod.lock delete mode 100644 Integrate-Checksum/test_data/main.k create mode 100755 mock/test_script/pull_pkg.sh diff --git a/Integrate-Checksum/main_test.go b/Integrate-Checksum/main_test.go index 589322fd..557c7912 100644 --- a/Integrate-Checksum/main_test.go +++ b/Integrate-Checksum/main_test.go @@ -2,7 +2,6 @@ package main import ( "encoding/json" - "os" "runtime" "testing" @@ -11,6 +10,7 @@ import ( "gotest.tools/assert" "kcl-lang.io/kpm/pkg/client" "kcl-lang.io/kpm/pkg/constants" + "kcl-lang.io/kpm/pkg/utils" ) // TestMainFunc tests the main functionality of the integrate checksum tool @@ -24,20 +24,16 @@ func TestMainFunc(t *testing.T) { err := mock.StartDockerRegistry() assert.NilError(t, err) - // Push the test package to the local OCI registry - err = mock.PushTestPkgToRegistry() + // Push the test package to the local OCI registry. + pkgDir, err := mock.PushTestPkgToRegistry() assert.NilError(t, err) // Initialize the KPM client. kpmClient, err := client.NewKpmClient() assert.NilError(t, err, "Failed to initialize KPM client") - // Get the current working directory. - currentDir, err := os.Getwd() - assert.NilError(t, err, "Failed to get current working directory") - // Locate KCL module files in the current directory. - packageDirs, err := findKCLModFiles(currentDir) + packageDirs, err := findKCLModFiles(pkgDir) assert.NilError(t, err, "Failed to locate KCL module files") assert.Assert(t, len(packageDirs) > 0, "No KCL module files found") @@ -82,9 +78,7 @@ func TestMainFunc(t *testing.T) { assert.NilError(t, err, "Failed to marshal new manifest to JSON") // Check if the manifest was updated correctly. - if string(newManifestJSON) == string(originalManifestJSON) { - t.Errorf("Failed to update the manifest; got %v", string(originalManifestJSON)) - } + assert.Assert(t, string(newManifestJSON) != string(originalManifestJSON), "Failed to update the manifest") // Revert the `Sum` field to its original value to ensure only that was changed. newManifest.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_SUM] = dependency.Sum @@ -94,6 +88,24 @@ func TestMainFunc(t *testing.T) { // Compare the new manifest data with the expected manifest data. assert.Equal(t, string(newManifestJSON), string(originalManifestJSON), "New manifest data mismatch") + // Pull the test package. + pkgPullPath, err := mock.PullTestPkg() + assert.NilError(t, err) + + // Find KCL module files in the pulled package. + packagePullDir, err := findKCLModFiles(pkgPullPath) + assert.NilError(t, err, "Failed to locate KCL module files") + + // Ensure that at least one KCL module file was found. + assert.Assert(t, len(packagePullDir) > 0, "No KCL module files found") + + // Calculate the hash of the pulled KCL module directory to verify its integrity. + pulledSum, err := utils.HashDir(packagePullDir[0]) + assert.NilError(t, err) + + // Compare the hash of the pulled files with the expected dependency sum to check for unintended changes. + assert.Equal(t, pulledSum, dependency.Sum, "Unexpected changes detected in the package contents") + // Clean the environment after all tests have been run err = mock.CleanTestEnv() assert.NilError(t, err) diff --git a/Integrate-Checksum/test_data/kcl.mod b/Integrate-Checksum/test_data/kcl.mod deleted file mode 100644 index 6c01364d..00000000 --- a/Integrate-Checksum/test_data/kcl.mod +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "test_data" -edition = "v0.9.0" -version = "0.0.1" - -[dependencies] -k8s = "1.31" - diff --git a/Integrate-Checksum/test_data/kcl.mod.lock b/Integrate-Checksum/test_data/kcl.mod.lock deleted file mode 100644 index 41b4867f..00000000 --- a/Integrate-Checksum/test_data/kcl.mod.lock +++ /dev/null @@ -1,5 +0,0 @@ -[dependencies] - [dependencies.k8s] - name = "k8s" - full_name = "k8s_1.31" - version = "1.31" diff --git a/Integrate-Checksum/test_data/main.k b/Integrate-Checksum/test_data/main.k deleted file mode 100644 index fa7048e6..00000000 --- a/Integrate-Checksum/test_data/main.k +++ /dev/null @@ -1 +0,0 @@ -The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/mock/oci_env_mock.go b/mock/oci_env_mock.go index 4ebd5819..f9b2a32b 100644 --- a/mock/oci_env_mock.go +++ b/mock/oci_env_mock.go @@ -10,10 +10,18 @@ func StartDockerRegistry() error { return cmd.Run() } -// PushTestPkgToRegistry pushes the test package to the local Docker registry. -func PushTestPkgToRegistry() error { +// PushTestPkgToRegistry pushes the test package to the local Docker registry and returns directory location. +func PushTestPkgToRegistry() (string, error) { cmd := exec.Command("../mock/test_script/push_pkg.sh") - return cmd.Run() + currentDir := "../mock" + return currentDir, cmd.Run() +} + +// PullTestPkg pulls the test package from the local Docker registry. +func PullTestPkg() (string, error) { + cmd := exec.Command("../mock/test_script/pull_pkg.sh") + pkgPullPath := "../mock/test_script" + return pkgPullPath, cmd.Run() } // CleanTestEnv cleans up the test environment by executing a cleanup script. diff --git a/mock/test_data/kcl.mod b/mock/test_data/kcl.mod index 6c01364d..d0044fa4 100644 --- a/mock/test_data/kcl.mod +++ b/mock/test_data/kcl.mod @@ -1,8 +1,4 @@ [package] name = "test_data" edition = "v0.9.0" -version = "0.0.1" - -[dependencies] -k8s = "1.31" - +version = "0.0.1" \ No newline at end of file diff --git a/mock/test_data/kcl.mod.lock b/mock/test_data/kcl.mod.lock index 41b4867f..e69de29b 100644 --- a/mock/test_data/kcl.mod.lock +++ b/mock/test_data/kcl.mod.lock @@ -1,5 +0,0 @@ -[dependencies] - [dependencies.k8s] - name = "k8s" - full_name = "k8s_1.31" - version = "1.31" diff --git a/mock/test_script/cleanup_test_environment.sh b/mock/test_script/cleanup_test_environment.sh index 4a244a01..2857c8a7 100755 --- a/mock/test_script/cleanup_test_environment.sh +++ b/mock/test_script/cleanup_test_environment.sh @@ -18,3 +18,6 @@ rm -rf "$current_dir/scripts/" # Delete the 'kcl' binary cd "$SCRIPT_DIR/../../" rm -rf ./bin/ + +cd "$SCRIPT_DIR" +rm -rf ./oci/ \ No newline at end of file diff --git a/mock/test_script/pull_pkg.sh b/mock/test_script/pull_pkg.sh new file mode 100755 index 00000000..9304665d --- /dev/null +++ b/mock/test_script/pull_pkg.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Get the directory of the script +SCRIPT_DIR="$(dirname "$(realpath "$0")")" + +# Move to the root directory +cd "$SCRIPT_DIR/../../" + +# Install kcl binary +GOBIN=$(pwd)/bin go install kcl-lang.io/cli/cmd/kcl@latest + +# Check kpm version +version=$(./bin/kcl --version) +if ! echo "$version" ; then + echo "Incorrect version: '$version'." + exit 1 +fi + +export KPM_REG="localhost:5001" +export KPM_REPO="test" + +# Prepare the package on the registry +current_dir=$(pwd) +echo $current_dir + +# Log in to the local registry +"$current_dir/bin/kcl" registry login -u test -p 1234 localhost:5001 + +# Pull the test_data package from the registry +cd "$SCRIPT_DIR" +"$current_dir/bin/kcl" mod pull oci://$KPM_REG/$KPM_REPO From df2711986ac463f0f3ae744218aeb39171af642d Mon Sep 17 00:00:00 2001 From: Nishant Bansal Date: Fri, 8 Nov 2024 21:46:43 +0530 Subject: [PATCH 4/7] fixes workflow action Signed-off-by: Nishant Bansal --- .github/workflows/include-kcl-checksums.yml | 23 ++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/include-kcl-checksums.yml b/.github/workflows/include-kcl-checksums.yml index 8a1fce9f..7f94c3b7 100644 --- a/.github/workflows/include-kcl-checksums.yml +++ b/.github/workflows/include-kcl-checksums.yml @@ -1,7 +1,16 @@ name: Include KCL Modules Checksum on: - push: + workflow_dispatch: + inputs: + kpm_reg: + description: "Enter the KPM Registry" + required: true + default: "localhost:5001" + kpm_repo: + description: "Enter the KPM Repository" + required: true + default: "test" jobs: include_modules_checksum: @@ -21,11 +30,19 @@ jobs: with: go-version-file: go.mod - - name: Login to GHCR - run: kcl registry login -u ${{ secrets.DEPLOY_ACCESS_NAME }} -p ${{ secrets.DEPLOY_ACCESS_TOKEN }} ghcr.io + - name: Run local registry for localhost + if: ${{ github.event.inputs.kpm_reg == 'localhost:5001' }} + run: ./scripts/reg.sh + + - name: Login to registry + run: | + kcl registry login -u ${{ github.event.inputs.kpm_repo }} -p ${{ secrets.REGISTRY_ACCESS_TOKEN }} ${{ github.event.inputs.kpm_reg }} - name: Get dependencies run: go get -v ./... - name: Run include checksum tool + env: + KPM_REG: ${{ github.event.inputs.kpm_reg }} + KPM_REPO: ${{ github.event.inputs.kpm_repo }} run: go run ./Integrate-Checksum/main.go From a258701ef493cd26214a5b24b87ec70e233d5119 Mon Sep 17 00:00:00 2001 From: Nishant Bansal Date: Mon, 18 Nov 2024 10:24:06 +0530 Subject: [PATCH 5/7] added new changes based on review Signed-off-by: Nishant Bansal --- .github/workflows/include-kcl-checksums.yml | 12 ++++++- Integrate-Checksum/main.go | 38 +++++++++++++++------ 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/.github/workflows/include-kcl-checksums.yml b/.github/workflows/include-kcl-checksums.yml index 7f94c3b7..c0b7cb8a 100644 --- a/.github/workflows/include-kcl-checksums.yml +++ b/.github/workflows/include-kcl-checksums.yml @@ -11,6 +11,14 @@ on: description: "Enter the KPM Repository" required: true default: "test" + pkg_name: + description: "Enter the Package Name" + required: true + default: "test" + pkg_version: + description: "Enter the Package Version" + required: true + default: "0.0.1" jobs: include_modules_checksum: @@ -36,7 +44,7 @@ jobs: - name: Login to registry run: | - kcl registry login -u ${{ github.event.inputs.kpm_repo }} -p ${{ secrets.REGISTRY_ACCESS_TOKEN }} ${{ github.event.inputs.kpm_reg }} + kcl registry login -u ${{ secrets.REGISTRY_ACCESS_NAME }} -p ${{ secrets.REGISTRY_ACCESS_TOKEN }} ${{ github.event.inputs.kpm_reg }} - name: Get dependencies run: go get -v ./... @@ -45,4 +53,6 @@ jobs: env: KPM_REG: ${{ github.event.inputs.kpm_reg }} KPM_REPO: ${{ github.event.inputs.kpm_repo }} + PKG_NAME: ${{ github.event.inputs.pkg_name }} + PKG_VERSION: ${{ github.event.inputs.pkg_version }} run: go run ./Integrate-Checksum/main.go diff --git a/Integrate-Checksum/main.go b/Integrate-Checksum/main.go index afbd2a4e..96dabcc7 100644 --- a/Integrate-Checksum/main.go +++ b/Integrate-Checksum/main.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "log" "net/http" "os" "path/filepath" @@ -22,7 +23,11 @@ import ( "kcl-lang.io/kpm/pkg/utils" ) -const KCLModFile = "kcl.mod" +const ( + KCLModFile = "kcl.mod" + PKG_NAME_ENV = "PKG_NAME" + PKG_VERSION_ENV = "PKG_VERSION" +) // findKCLModFiles searches the specified root directory for all kcl.mod files and returns their paths. func findKCLModFiles(root string) ([]string, error) { @@ -169,17 +174,16 @@ func tagManifest(repo *remote.Repository, manifestBytes []byte, dependency *pkg. } // processPackage processes the package directory and updates the OCI manifest if needed. -func processPackage(packageDir string) error { - kpmClient, err := client.NewKpmClient() - if err != nil { - return fmt.Errorf("failed to create KPM client: %w", err) - } - +func processPackage(packageDir string, kpmClient *client.KpmClient, pkgName string, pkgVersion string) error { dependency, err := resolveDependency(kpmClient, packageDir) if err != nil { return fmt.Errorf("failed to resolve dependency: %w", err) } + if dependency.Name != pkgName || dependency.Version != pkgVersion { + return nil + } + manifest, err := fetchManifest(kpmClient, dependency) if err != nil { return fmt.Errorf("failed to fetch manifest: %w", err) @@ -210,11 +214,23 @@ func main() { return } + pkgName := os.Getenv(PKG_NAME_ENV) + pkgVersion := os.Getenv(PKG_VERSION_ENV) + + if pkgName == "" || pkgVersion == "" { + log.Fatal("Environment variables PKG_NAME or PKG_VERSION are not set") + } + + kpmClient, err := client.NewKpmClient() + if err != nil { + log.Fatalf("failed to create KPM client: %v", err) + } + for _, packageDir := range modFilePaths { - if err := processPackage(packageDir); err != nil { - fmt.Printf("Error processing package at '%s': %v\n", packageDir, err) + if err := processPackage(packageDir, kpmClient, pkgName, pkgVersion); err != nil { + log.Fatalf("Error processing package at '%s': %v", packageDir, err) } } - - fmt.Println("Checksum successfully included in all KCL packages") + + fmt.Printf("Checksum successfully included in the package '%s' of version '%s'\n", pkgName, pkgVersion) } From 49a29ed1afa4b1d3945f06be297881f31f94e757 Mon Sep 17 00:00:00 2001 From: Nishant Bansal Date: Thu, 21 Nov 2024 08:52:09 +0530 Subject: [PATCH 6/7] added log messages Signed-off-by: Nishant Bansal --- Integrate-Checksum/main.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Integrate-Checksum/main.go b/Integrate-Checksum/main.go index 96dabcc7..684e587f 100644 --- a/Integrate-Checksum/main.go +++ b/Integrate-Checksum/main.go @@ -47,6 +47,9 @@ func findKCLModFiles(root string) ([]string, error) { if err != nil { return nil, fmt.Errorf("error walking directory '%s': %w", root, err) } + + log.Printf("Selected paths: %v", modFilePaths) + return modFilePaths, nil } @@ -85,6 +88,8 @@ func resolveDependency(kpmClient *client.KpmClient, packageDir string) (*pkg.Dep if dependency.Sum, err = utils.HashDir(packageDir); err != nil { return nil, fmt.Errorf("failed to hash directory '%s': %w", packageDir, err) } + log.Printf("Successfully hashed directory '%s': Sum = %s", packageDir, dependency.Sum) + dependency.FromKclPkg(kclPkg) return dependency, nil @@ -190,7 +195,7 @@ func processPackage(packageDir string, kpmClient *client.KpmClient, pkgName stri } if existingSum, ok := manifest.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_SUM]; ok && dependency.Sum == existingSum { - fmt.Println("Manifest already up to date with matching checksum.") + fmt.Printf("Manifest already up to date with matching checksum. ExistingSum: %s\n", existingSum) return nil } @@ -198,6 +203,8 @@ func processPackage(packageDir string, kpmClient *client.KpmClient, pkgName stri return fmt.Errorf("failed to update checksum in manifest: %w", err) } + log.Printf("Successfully updated manifest with new checksum: %s\n", dependency.Sum) + return nil } @@ -221,6 +228,8 @@ func main() { log.Fatal("Environment variables PKG_NAME or PKG_VERSION are not set") } + log.Printf("Acquired package info - Name: %s, Version: %s", pkgName, pkgVersion) + kpmClient, err := client.NewKpmClient() if err != nil { log.Fatalf("failed to create KPM client: %v", err) From 9d94efff92b38303073187d7975f24df33163684 Mon Sep 17 00:00:00 2001 From: Nishant Bansal Date: Thu, 21 Nov 2024 09:12:33 +0530 Subject: [PATCH 7/7] updated log messages Signed-off-by: Nishant Bansal --- Integrate-Checksum/main.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Integrate-Checksum/main.go b/Integrate-Checksum/main.go index 684e587f..81e15a70 100644 --- a/Integrate-Checksum/main.go +++ b/Integrate-Checksum/main.go @@ -195,7 +195,7 @@ func processPackage(packageDir string, kpmClient *client.KpmClient, pkgName stri } if existingSum, ok := manifest.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_SUM]; ok && dependency.Sum == existingSum { - fmt.Printf("Manifest already up to date with matching checksum. ExistingSum: %s\n", existingSum) + log.Printf("Manifest already up to date with matching checksum. ExistingSum: %s\n", existingSum) return nil } @@ -211,13 +211,13 @@ func processPackage(packageDir string, kpmClient *client.KpmClient, pkgName stri func main() { currentDir, err := os.Getwd() if err != nil { - fmt.Printf("Error getting current directory: %v\n", err) + log.Printf("Error getting current directory: %v\n", err) return } modFilePaths, err := findKCLModFiles(currentDir) if err != nil { - fmt.Printf("Error finding kcl.mod files: %v\n", err) + log.Printf("Error finding kcl.mod files: %v\n", err) return } @@ -241,5 +241,5 @@ func main() { } } - fmt.Printf("Checksum successfully included in the package '%s' of version '%s'\n", pkgName, pkgVersion) + log.Printf("Checksum successfully included in the package '%s' of version '%s'\n", pkgName, pkgVersion) }