From bd347db164bb22149583658ef7c1076064022a72 Mon Sep 17 00:00:00 2001 From: Nalin Dahyabhai Date: Thu, 5 Dec 2024 10:31:15 -0500 Subject: [PATCH] manifest add: add a --artifact-annotation for --artifact When adding an artifact to a manifest list / image index with `manifest add --artifact` provide a `--artifact-annotation` flag which can be used to set annotations in the artifact manifest that we're generating. Fix a client-side hang when using --artifact. Signed-off-by: Nalin Dahyabhai --- cmd/podman/manifest/add.go | 41 ++++--- .../markdown/podman-manifest-add.1.md.in | 5 + pkg/bindings/manifests/manifests.go | 1 + test/e2e/manifest_test.go | 106 +++++++++++++++++- 4 files changed, 138 insertions(+), 15 deletions(-) diff --git a/cmd/podman/manifest/add.go b/cmd/podman/manifest/add.go index eafc9de3dc..5c2645336f 100644 --- a/cmd/podman/manifest/add.go +++ b/cmd/podman/manifest/add.go @@ -24,12 +24,13 @@ type manifestAddOptsWrapper struct { entities.ManifestAddOptions artifactOptions entities.ManifestAddArtifactOptions - tlsVerifyCLI bool // CLI only - insecure bool // CLI only - credentialsCLI string // CLI only - artifact bool // CLI only - artifactConfigFile string // CLI only - artifactType string // CLI only + tlsVerifyCLI bool // CLI only + insecure bool // CLI only + credentialsCLI string // CLI only + artifact bool // CLI only + artifactConfigFile string // CLI only + artifactType string // CLI only + artifactAnnotations []string // CLI only } var ( @@ -55,16 +56,20 @@ func init() { flags.BoolVar(&manifestAddOpts.All, "all", false, "add all of the list's images if the image is a list") annotationFlagName := "annotation" - flags.StringArrayVar(&manifestAddOpts.Annotation, annotationFlagName, nil, "set an `annotation` for the specified image") + flags.StringArrayVar(&manifestAddOpts.Annotation, annotationFlagName, nil, "set an `annotation` for the specified image or artifact") _ = addCmd.RegisterFlagCompletionFunc(annotationFlagName, completion.AutocompleteNone) archFlagName := "arch" - flags.StringVar(&manifestAddOpts.Arch, archFlagName, "", "override the `architecture` of the specified image") + flags.StringVar(&manifestAddOpts.Arch, archFlagName, "", "override the `architecture` of the specified image or artifact") _ = addCmd.RegisterFlagCompletionFunc(archFlagName, completion.AutocompleteArch) artifactFlagName := "artifact" flags.BoolVar(&manifestAddOpts.artifact, artifactFlagName, false, "add all arguments as artifact files rather than as images") + artifactAnnotationFlagName := "artifact-annotation" + flags.StringArrayVar(&manifestAddOpts.artifactAnnotations, artifactAnnotationFlagName, nil, "set an `annotation` in the artifact") + _ = addCmd.RegisterFlagCompletionFunc(artifactAnnotationFlagName, completion.AutocompleteNone) + artifactExcludeTitlesFlagName := "artifact-exclude-titles" flags.BoolVar(&manifestAddOpts.artifactOptions.ExcludeTitles, artifactExcludeTitlesFlagName, false, fmt.Sprintf(`refrain from setting %q annotations on "layers"`, imgspecv1.AnnotationTitle)) @@ -101,15 +106,15 @@ func init() { _ = addCmd.RegisterFlagCompletionFunc(credsFlagName, completion.AutocompleteNone) featuresFlagName := "features" - flags.StringSliceVar(&manifestAddOpts.Features, featuresFlagName, nil, "override the `features` of the specified image") + flags.StringSliceVar(&manifestAddOpts.Features, featuresFlagName, nil, "override the `features` of the specified image or artifact") _ = addCmd.RegisterFlagCompletionFunc(featuresFlagName, completion.AutocompleteNone) osFlagName := "os" - flags.StringVar(&manifestAddOpts.OS, osFlagName, "", "override the `OS` of the specified image") + flags.StringVar(&manifestAddOpts.OS, osFlagName, "", "override the `OS` of the specified image or artifact") _ = addCmd.RegisterFlagCompletionFunc(osFlagName, completion.AutocompleteOS) osVersionFlagName := "os-version" - flags.StringVar(&manifestAddOpts.OSVersion, osVersionFlagName, "", "override the OS `version` of the specified image") + flags.StringVar(&manifestAddOpts.OSVersion, osVersionFlagName, "", "override the OS `version` of the specified image or artifact") _ = addCmd.RegisterFlagCompletionFunc(osVersionFlagName, completion.AutocompleteNone) flags.BoolVar(&manifestAddOpts.insecure, "insecure", false, "neither require HTTPS nor verify certificates when accessing the registry") @@ -117,7 +122,7 @@ func init() { flags.BoolVar(&manifestAddOpts.tlsVerifyCLI, "tls-verify", true, "require HTTPS and verify certificates when accessing the registry") variantFlagName := "variant" - flags.StringVar(&manifestAddOpts.Variant, variantFlagName, "", "override the `Variant` of the specified image") + flags.StringVar(&manifestAddOpts.Variant, variantFlagName, "", "override the `variant` of the specified image or artifact") _ = addCmd.RegisterFlagCompletionFunc(variantFlagName, completion.AutocompleteNone) if registry.IsRemote() { @@ -157,7 +162,7 @@ func add(cmd *cobra.Command, args []string) error { if !manifestAddOpts.artifact { var changedArtifactFlags []string - for _, artifactOption := range []string{"artifact-type", "artifact-config", "artifact-config-type", "artifact-layer-type", "artifact-subject", "artifact-exclude-titles"} { + for _, artifactOption := range []string{"artifact-type", "artifact-config", "artifact-config-type", "artifact-layer-type", "artifact-subject", "artifact-exclude-titles", "artifact-annotation"} { if cmd.Flags().Changed(artifactOption) { changedArtifactFlags = append(changedArtifactFlags, "--"+artifactOption) } @@ -183,6 +188,16 @@ func add(cmd *cobra.Command, args []string) error { } manifestAddOpts.artifactOptions.Config = string(configBytes) } + if len(manifestAddOpts.artifactAnnotations) > 0 { + manifestAddOpts.artifactOptions.Annotations = make(map[string]string, len(manifestAddOpts.artifactAnnotations)) + for _, annotation := range manifestAddOpts.artifactAnnotations { + key, val, hasVal := strings.Cut(annotation, "=") + if !hasVal { + return fmt.Errorf("no value given for annotation %q", key) + } + manifestAddOpts.artifactOptions.Annotations[key] = val + } + } manifestAddOpts.artifactOptions.ManifestAnnotateOptions = manifestAddOpts.ManifestAnnotateOptions listID, err = registry.ImageEngine().ManifestAddArtifact(context.Background(), args[0], args[1:], manifestAddOpts.artifactOptions) if err != nil { diff --git a/docs/source/markdown/podman-manifest-add.1.md.in b/docs/source/markdown/podman-manifest-add.1.md.in index db02b00c05..e9576b114a 100644 --- a/docs/source/markdown/podman-manifest-add.1.md.in +++ b/docs/source/markdown/podman-manifest-add.1.md.in @@ -39,6 +39,11 @@ Create an artifact manifest and add it to the image index. Arguments after the index name will be interpreted as file names rather than as image references. In most scenarios, the **--artifact-type** option should also be specified. +#### **--artifact-annotation**=*annotation=value* + +When creating an artifact manifest and adding it to the image index, set the +specified annotation in the artifact manifest. + #### **--artifact-config**=*path* When creating an artifact manifest and adding it to the image index, use the diff --git a/pkg/bindings/manifests/manifests.go b/pkg/bindings/manifests/manifests.go index 9ac41a85eb..20197109ff 100644 --- a/pkg/bindings/manifests/manifests.go +++ b/pkg/bindings/manifests/manifests.go @@ -356,6 +356,7 @@ func Modify(ctx context.Context, name string, images []string, options *ModifyOp artifactContentType = writer.FormDataContentType() artifactWriterGroup.Add(1) go func() { + defer artifactWriterGroup.Done() defer bodyWriter.Close() defer writer.Close() // start with the body we would have uploaded if we weren't diff --git a/test/e2e/manifest_test.go b/test/e2e/manifest_test.go index 783e9418a9..f9d67b8f8e 100644 --- a/test/e2e/manifest_test.go +++ b/test/e2e/manifest_test.go @@ -3,8 +3,11 @@ package integration import ( + "bytes" + "crypto/rand" "encoding/json" "fmt" + "io" "os" "path/filepath" "strings" @@ -19,6 +22,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gexec" + digest "github.com/opencontainers/go-digest" imgspec "github.com/opencontainers/image-spec/specs-go" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -427,6 +431,102 @@ add_compression = ["zstd"]`), 0o644) Expect(session.OutputToString()).To(MatchJSON(encoded)) }) + It("artifact", func() { + session := podmanTest.Podman([]string{"manifest", "create", "foo"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(ExitCleanly()) + // generate some random data + tmpData := make([]byte, 1024) + nRead, err := rand.Read(tmpData) + Expect(err).ToNot(HaveOccurred()) + Expect(nRead).To(Equal(len(tmpData))) + // put that random data into a file + tmpFile, err := os.CreateTemp(podmanTest.TmpDir, "artifact-test") + Expect(err).ToNot(HaveOccurred()) + nCopied, err := io.Copy(tmpFile, bytes.NewReader(tmpData)) + Expect(err).ToNot(HaveOccurred()) + Expect(int(nCopied)).To(Equal(len(tmpData))) + tmpFileName := tmpFile.Name() + err = tmpFile.Close() + Expect(err).ToNot(HaveOccurred()) + configData := `{"key2":"value"}` + configFile, err := os.CreateTemp(podmanTest.TmpDir, "artifact-test") + Expect(err).ToNot(HaveOccurred()) + nCopied, err = io.Copy(configFile, strings.NewReader(configData)) + Expect(err).ToNot(HaveOccurred()) + Expect(int(nCopied)).To(Equal(len(configData))) + configFileName := configFile.Name() + err = configFile.Close() + Expect(err).ToNot(HaveOccurred()) + // add that file to the list as an artifact + artifactType := "application/x-custom" + artifactConfigType := "text/json" + artifactLayerType := "text/plain" + session = podmanTest.Podman([]string{"manifest", "add", "foo", tmpFileName, + "--artifact", + "--artifact-type", artifactType, + "--artifact-annotation", "key1=value", + "--artifact-config", configFileName, + "--artifact-config-type", artifactConfigType, + "--artifact-layer-type", artifactLayerType, + }) + session.WaitWithDefaultTimeout() + Expect(session).Should(ExitCleanly()) + // push the list to an OCI layout + layoutDir, err := os.MkdirTemp(podmanTest.TmpDir, "artifact-test") + Expect(err).ToNot(HaveOccurred()) + session = podmanTest.Podman([]string{"manifest", "push", "-q", "foo", "oci:" + layoutDir}) + session.WaitWithDefaultTimeout() + Expect(session).Should(ExitCleanly()) + // start following the breadcrumbs + var topIndex, builtIndex imgspecv1.Index + topIndexBytes, err := os.ReadFile(filepath.Join(layoutDir, "index.json")) + Expect(err).ToNot(HaveOccurred()) + err = json.Unmarshal(topIndexBytes, &topIndex) + Expect(err).ToNot(HaveOccurred()) + Expect(topIndex.Manifests).To(HaveLen(1)) + // only thing in the layout is the index we built + builtIndexDigest := topIndex.Manifests[0].Digest + builtIndexBytes, err := os.ReadFile(filepath.Join(layoutDir, "blobs", builtIndexDigest.Algorithm().String(), builtIndexDigest.Encoded())) + Expect(err).ToNot(HaveOccurred()) + err = json.Unmarshal(builtIndexBytes, &builtIndex) + Expect(err).ToNot(HaveOccurred()) + Expect(builtIndex.Manifests).To(HaveLen(1)) + // only thing in the index is the artifact manifest we built + artifactManifestDigest := builtIndex.Manifests[0].Digest + artifactManifestBytes, err := os.ReadFile(filepath.Join(layoutDir, "blobs", artifactManifestDigest.Algorithm().String(), artifactManifestDigest.Encoded())) + Expect(err).ToNot(HaveOccurred()) + // construct what we think we've been building + expected, err := json.Marshal(&imgspecv1.Manifest{ + Versioned: imgspec.Versioned{ + SchemaVersion: 2, + }, + MediaType: imgspecv1.MediaTypeImageManifest, + ArtifactType: artifactType, + Config: imgspecv1.Descriptor{ + MediaType: artifactConfigType, + Digest: digest.FromBytes([]byte(configData)), + Size: int64(len(configData)), + Data: []byte(configData), + }, + Layers: []imgspecv1.Descriptor{ + { + MediaType: artifactLayerType, + Digest: digest.FromBytes(tmpData), + Size: int64(len(tmpData)), + Annotations: map[string]string{ + imgspecv1.AnnotationTitle: filepath.Base(tmpFileName), + }, + }, + }, + Annotations: map[string]string{ + "key1": "value", + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(artifactManifestBytes).To(MatchJSON(expected)) + }) + It("remove digest", func() { session := podmanTest.Podman([]string{"manifest", "create", "foo"}) session.WaitWithDefaultTimeout() @@ -450,8 +550,10 @@ add_compression = ["zstd"]`), 0o644) ContainSubstring(imageListPPC64LEInstanceDigest), ContainSubstring(imageListS390XInstanceDigest), Not( - ContainSubstring(imageListARM64InstanceDigest)), - )) + ContainSubstring(imageListARM64InstanceDigest), + ), + ), + ) }) It("remove not-found", func() {