diff --git a/cmd/podman/manifest/annotate.go b/cmd/podman/manifest/annotate.go
index 60765a9b22..9a54fe2166 100644
--- a/cmd/podman/manifest/annotate.go
+++ b/cmd/podman/manifest/annotate.go
@@ -119,7 +119,7 @@ func annotate(cmd *cobra.Command, args []string) error {
 	} else {
 		opts.Annotations = annotations
 	}
-	id, err := registry.ImageEngine().ManifestAnnotate(registry.Context(), args[0], args[1], opts)
+	id, err := registry.ImageEngine().ManifestAnnotate(registry.Context(), listImageSpec, instanceSpec, opts)
 	if err != nil {
 		return err
 	}
diff --git a/test/e2e/manifest_test.go b/test/e2e/manifest_test.go
index 22bf064b4e..497307d35f 100644
--- a/test/e2e/manifest_test.go
+++ b/test/e2e/manifest_test.go
@@ -10,12 +10,16 @@ import (
 	"strings"
 
 	"github.com/containers/common/libimage/define"
+	"github.com/containers/image/v5/docker/reference"
+	manifest "github.com/containers/image/v5/manifest"
+	"github.com/containers/image/v5/transports/alltransports"
 	podmanRegistry "github.com/containers/podman/v5/hack/podman-registry-go"
 	. "github.com/containers/podman/v5/test/utils"
 	"github.com/containers/storage/pkg/archive"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	. "github.com/onsi/gomega/gexec"
+	imgspec "github.com/opencontainers/image-spec/specs-go"
 	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
@@ -377,12 +381,49 @@ add_compression = ["zstd"]`), 0o644)
 		session = podmanTest.Podman([]string{"manifest", "annotate", "--annotation", "hello=world,withcomma", "--arch", "bar", "foo", imageListARM64InstanceDigest})
 		session.WaitWithDefaultTimeout()
 		Expect(session).Should(ExitCleanly())
+		session = podmanTest.Podman([]string{"manifest", "annotate", "--index", "--annotation", "left=right", "foo"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(ExitCleanly())
 		session = podmanTest.Podman([]string{"manifest", "inspect", "foo"})
 		session.WaitWithDefaultTimeout()
 		Expect(session).Should(ExitCleanly())
-		Expect(session.OutputToString()).To(ContainSubstring(`"architecture": "bar"`))
-		// Check added annotation
-		Expect(session.OutputToString()).To(ContainSubstring(`"hello": "world,withcomma"`))
+		// Extract the digest from the name of the image that we just added to the list
+		ref, err := alltransports.ParseImageName(imageListInstance)
+		Expect(err).ToNot(HaveOccurred())
+		dockerReference := ref.DockerReference()
+		referenceWithDigest, ok := dockerReference.(reference.Canonical)
+		Expect(ok).To(BeTrueBecause("we started with a canonical reference"))
+		// Check that the index has all of the information we've just added to it
+		encoded, err := json.Marshal(&imgspecv1.Index{
+			Versioned: imgspec.Versioned{
+				SchemaVersion: 2,
+			},
+			// media type forced because we need to be able to represent annotations
+			MediaType: imgspecv1.MediaTypeImageIndex,
+			Manifests: []imgspecv1.Descriptor{
+				{
+					// from imageListInstance
+					MediaType: manifest.DockerV2Schema2MediaType,
+					Digest:    referenceWithDigest.Digest(),
+					Size:      527,
+					// OS from imageListInstance(?), Architecture/Variant as above
+					Platform: &imgspecv1.Platform{
+						Architecture: "bar",
+						OS:           "linux",
+					},
+					// added above
+					Annotations: map[string]string{
+						"hello": "world,withcomma",
+					},
+				},
+			},
+			// added above
+			Annotations: map[string]string{
+				"left": "right",
+			},
+		})
+		Expect(err).ToNot(HaveOccurred())
+		Expect(session.OutputToString()).To(MatchJSON(encoded))
 	})
 
 	It("remove digest", func() {