From 7c7bdfb8466d4212665abca97589e848dd09b940 Mon Sep 17 00:00:00 2001 From: Uwe Krueger Date: Tue, 24 Sep 2024 16:58:30 +0200 Subject: [PATCH] transfer uploader tests for relativeociref --- api/oci/utils.go | 6 +- .../accessmethods/ociartifact/method.go | 10 +- api/ocm/tools/transfer/options.go | 3 + api/ocm/tools/transfer/transfer.go | 2 +- .../transferhandler/standard/transfer_test.go | 302 ++++++++++++++++++ 5 files changed, 320 insertions(+), 3 deletions(-) create mode 100644 api/ocm/tools/transfer/transferhandler/standard/transfer_test.go diff --git a/api/oci/utils.go b/api/oci/utils.go index 577d44fc7..fad2b5500 100644 --- a/api/oci/utils.go +++ b/api/oci/utils.go @@ -18,11 +18,15 @@ func AsTags(tag string) []string { } func StandardOCIRef(host, repository, version string) string { + return fmt.Sprintf("%s%s%s", host, grammar.RepositorySeparator, RelativeOCIRef(repository, version)) +} + +func RelativeOCIRef(repository, version string) string { sep := grammar.TagSeparator if ok, _ := artdesc.IsDigest(version); ok { sep = grammar.DigestSeparator } - return fmt.Sprintf("%s%s%s%s%s", host, grammar.RepositorySeparator, repository, sep, version) + return fmt.Sprintf("%s%s%s", repository, sep, version) } func IsIntermediate(spec RepositorySpec) bool { diff --git a/api/ocm/extensions/accessmethods/ociartifact/method.go b/api/ocm/extensions/accessmethods/ociartifact/method.go index f6660c56a..29c75cb32 100644 --- a/api/ocm/extensions/accessmethods/ociartifact/method.go +++ b/api/ocm/extensions/accessmethods/ociartifact/method.go @@ -78,11 +78,19 @@ func (a *AccessSpec) Describe(ctx accspeccpi.Context) string { func (a *AccessSpec) Info(ctx accspeccpi.Context) *accspeccpi.UniformAccessSpecInfo { ref, _ := oci.ParseRef(a.ImageReference) host, port := ref.HostPort() + + r := ref.Repository + if ref.Tag != nil { + r += ":" + *ref.Tag + } + if ref.Digest != nil { + r += "@" + ref.Digest.String() + } return &accspeccpi.UniformAccessSpecInfo{ Kind: Type, Host: host, Port: port, - Info: ref.Version(), + Info: r, } } diff --git a/api/ocm/tools/transfer/options.go b/api/ocm/tools/transfer/options.go index 4c9e66dcf..cd1d688ca 100644 --- a/api/ocm/tools/transfer/options.go +++ b/api/ocm/tools/transfer/options.go @@ -41,6 +41,9 @@ type localOptions struct { func (opts *localOptions) Eval(optlist ...transferhandler.TransferOption) error { for _, o := range optlist { + if o == nil { + continue + } if _, ok := o.(transferhandler.TransferOptionsCreator); !ok { err := o.ApplyTransferOption(opts) if err != nil { diff --git a/api/ocm/tools/transfer/transfer.go b/api/ocm/tools/transfer/transfer.go index 849013886..022ac4662 100644 --- a/api/ocm/tools/transfer/transfer.go +++ b/api/ocm/tools/transfer/transfer.go @@ -255,7 +255,7 @@ func copyVersion(printer common.Printer, log logging.Logger, hist common.History hint := ocmcpi.ArtifactNameHint(a, src) old, err = cur.GetResourceByIdentity(r.Meta().GetIdentity(srccd.Resources)) - changed := err != nil || old.Digest == nil || !old.Digest.Equal(r.Meta().Digest) + changed := err != nil || old.Digest == nil || r.Meta().Digest == nil || !old.Digest.Equal(r.Meta().Digest) valueNeeded := err == nil && needsTransport(src.GetContext(), r, &old) if changed || valueNeeded { var msgs []interface{} diff --git a/api/ocm/tools/transfer/transferhandler/standard/transfer_test.go b/api/ocm/tools/transfer/transferhandler/standard/transfer_test.go new file mode 100644 index 000000000..76c26bae8 --- /dev/null +++ b/api/ocm/tools/transfer/transferhandler/standard/transfer_test.go @@ -0,0 +1,302 @@ +package standard_test + +import ( + . "github.com/mandelsoft/goutils/testutils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "ocm.software/ocm/api/helper/builder" + "ocm.software/ocm/api/oci" + "ocm.software/ocm/api/oci/artdesc" + "ocm.software/ocm/api/oci/extensions/repositories/artifactset" + ocictf "ocm.software/ocm/api/oci/extensions/repositories/ctf" + . "ocm.software/ocm/api/oci/testhelper" + "ocm.software/ocm/api/ocm" + metav1 "ocm.software/ocm/api/ocm/compdesc/meta/v1" + "ocm.software/ocm/api/ocm/extensions/accessmethods/localblob" + "ocm.software/ocm/api/ocm/extensions/accessmethods/ociartifact" + "ocm.software/ocm/api/ocm/extensions/accessmethods/relativeociref" + resourcetypes "ocm.software/ocm/api/ocm/extensions/artifacttypes" + "ocm.software/ocm/api/ocm/extensions/attrs/keepblobattr" + "ocm.software/ocm/api/ocm/extensions/blobhandler" + storagecontext "ocm.software/ocm/api/ocm/extensions/blobhandler/handlers/oci" + "ocm.software/ocm/api/ocm/extensions/blobhandler/handlers/oci/ocirepo" + "ocm.software/ocm/api/ocm/extensions/repositories/ctf" + "ocm.software/ocm/api/ocm/tools/transfer" + "ocm.software/ocm/api/ocm/tools/transfer/transferhandler" + "ocm.software/ocm/api/ocm/tools/transfer/transferhandler/standard" + "ocm.software/ocm/api/utils" + "ocm.software/ocm/api/utils/accessio" + "ocm.software/ocm/api/utils/accessobj" + common "ocm.software/ocm/api/utils/misc" +) + +const OCIHOST2 = "target" + +var _ = Describe("value transport with relative ocireg", func() { + var env *Builder + + BeforeEach(func() { + env = NewBuilder() + + env.RSAKeyPair(SIGNATURE) + + env.OCICommonTransport(OCIPATH, accessio.FormatDirectory, func() { + OCIManifest1(env) + }) + + FakeOCIRepo(env, OCIPATH, OCIHOST) + FakeOCIRepo(env, ARCH, OCIHOST2) + + env.OCMCommonTransport(OCIPATH, accessio.FormatDirectory, func() { + env.Component(COMPONENT, func() { + env.Version(VERSION, func() { + env.Provider(PROVIDER) + env.Resource("artifact", "", resourcetypes.OCI_IMAGE, metav1.LocalRelation, func() { + env.Access( + relativeociref.New(oci.RelativeOCIRef(OCINAMESPACE, OCIVERSION)), + ) + }) + }) + }) + }) + }) + + AfterEach(func() { + env.Cleanup() + }) + + It("it should use additional resolver to resolve component ref", func() { + src := Must(ctf.Open(env.OCMContext(), accessobj.ACC_READONLY, OCIPATH, 0, env)) + defer Close(src, "src") + + cv := Must(src.LookupComponentVersion(COMPONENT, VERSION)) + defer Close(cv, "cv") + + r := Must(cv.GetResourceByIndex(0)) + CheckBlob(r, D_OCIMANIFEST1, 628) + }) + + DescribeTable("transfers per value", func(keep bool, mod func(env *Builder), dig string, size int, opts ...transferhandler.TransferOption) { + env.OCMContext().BlobHandlers().Register(ocirepo.NewArtifactHandler(FakeOCIRegBaseFunction), + blobhandler.ForRepo(oci.CONTEXT_TYPE, ocictf.Type), blobhandler.ForMimeType(artdesc.ToContentMediaType(artdesc.MediaTypeImageManifest))) + keepblobattr.Set(env.OCMContext(), keep) + + p, buf := common.NewBufferedPrinter() + topts := append([]transferhandler.TransferOption{ + standard.ResourcesByValue(), transfer.WithPrinter(p), + }, opts...) + + src := Must(ctf.Open(env.OCMContext(), accessobj.ACC_READONLY, OCIPATH, 0, env)) + defer Close(src, "src") + + cv := Must(src.LookupComponentVersion(COMPONENT, VERSION)) + defer Close(cv, "cv") + + mod(env) + + tgt := Must(ctf.Open(env, accessobj.ACC_WRITABLE, ARCH, 0, env)) + ctgt := accessio.OnceCloser(tgt) + defer Close(ctgt, "tgt") + + MustBeSuccessful(transfer.Transfer(cv, tgt, topts...)) + + options := &standard.Options{} + transferhandler.ApplyOptions(options, topts...) + + out := ` + transferring version "github.com/mandelsoft/test:v1"... + ...resource 0 artifact\[ociImage\]\(ocm/value:v2.0\)... + ...adding component version... +` + if options.IsOverwrite() { + out = ` + transferring version "github.com/mandelsoft/test:v1"... + warning: version "github.com/mandelsoft/test:v1" already present, but differs because some artifact.*changed \(transport enforced by overwrite option\) + ...resource 0 artifact\[ociImage\]\(ocm/value:v2.0\).* + ...adding component version... +` + } + Expect(string(buf.Bytes())).To(StringMatchTrimmedWithContext(utils.Crop(out, 2))) + MustBeSuccessful(ctgt.Close()) + + tgt = Must(ctf.Open(env, accessobj.ACC_READONLY, ARCH, 0, env)) + ctgt = accessio.OnceCloser(tgt) + defer Close(ctgt, "tgt2") + + tcv := Must(tgt.LookupComponentVersion(COMPONENT, VERSION)) + ctcv := accessio.OnceCloser(tcv) + defer Close(ctcv, "tcv") + + r := Must(tcv.GetResourceByIndex(0)) + acc := Must(r.Access()) + + atype := ociartifact.Type + if keep { + atype = localblob.Type + } + Expect(acc.GetKind()).To(Equal(atype)) + + info := acc.Info(env.OCMContext()) + if keep { + Expect(info.Info).To(Equal("sha256:" + H_OCIARCHMANIFEST1)) + + } else { + Expect(info.Host).To(Equal(OCIHOST2 + ".alias")) + Expect(info.Info).To(Equal("ocm/value:v2.0")) + } + + CheckBlob(r, dig, size) + + MustBeSuccessful(ctcv.Close()) + MustBeSuccessful(ctgt.Close()) + + CheckAritifact(env, dig, size) + + // re-transport + buf.Reset() + tgt = Must(ctf.Open(env, accessobj.ACC_WRITABLE, ARCH, 0, env)) + ctgt = accessio.OnceCloser(tgt) + defer Close(ctgt, "tgt3") + + MustBeSuccessful(transfer.Transfer(cv, tgt, topts...)) + + tcv = Must(tgt.LookupComponentVersion(COMPONENT, VERSION)) + ctcv = accessio.OnceCloser(tcv) + defer Close(ctcv, "ctcv") + + r = Must(tcv.GetResourceByIndex(0)) + acc = Must(r.Access()) + Expect(acc.GetKind()).To(Equal(atype)) + + info = acc.Info(env.OCMContext()) + if keep { + Expect(info.Info).To(Equal("sha256:" + H_OCIARCHMANIFEST1)) + + } else { + Expect(info.Info).To(Equal("ocm/value:v2.0")) + } + + MustBeSuccessful(ctcv.Close()) + MustBeSuccessful(ctgt.Close()) + + CheckAritifact(env, dig, size) + + }, + Entry("empty target", false, EmptyTarget, D_OCIMANIFEST1, 628), + Entry("identical target", false, IdenticalTarget, D_OCIMANIFEST1, 628), + Entry("different target", false, DifferentTarget, D_OCIMANIFEST1, 628), + Entry("different CV", false, DifferentCV, D_OCIMANIFEST1, 628, standard.Overwrite()), + Entry("different namespace", false, DifferentNamespace, D_OCIMANIFEST1, 628, standard.Overwrite()), + Entry("different name", false, DifferentName, D_OCIMANIFEST1, 628, standard.Overwrite()), + Entry("keep, empty target", true, EmptyTarget, D_OCIMANIFEST1, 628), + + Entry("keep, identical target", true, IdenticalTarget, D_OCIMANIFEST1, 628), + Entry("keep, different target", true, DifferentTarget, D_OCIMANIFEST1, 628), + Entry("keep, different CV", true, DifferentCV, D_OCIMANIFEST1, 628, standard.Overwrite()), + Entry("keep, different namespace", true, DifferentNamespace, D_OCIMANIFEST1, 628, standard.Overwrite()), + Entry("keep, different name", true, DifferentName, D_OCIMANIFEST1, 628, standard.Overwrite()), + ) +}) + +func EmptyTarget(env *Builder) { + env.OCMCommonTransport(ARCH, accessio.FormatDirectory) +} + +func IdenticalTarget(env *Builder) { + env.OCMCommonTransport(ARCH, accessio.FormatDirectory, func() { + OCIManifest1For(env, OCINAMESPACE, OCIVERSION) + }) +} + +func DifferentTarget(env *Builder) { + env.OCMCommonTransport(ARCH, accessio.FormatDirectory, func() { + OCIManifest2For(env, OCINAMESPACE, OCIVERSION) + }) +} + +func DifferentCV(env *Builder) { + env.OCICommonTransport(ARCH, accessio.FormatDirectory, func() { + OCIManifest2For(env, OCINAMESPACE, OCIVERSION) + }) + env.OCMCommonTransport(ARCH, accessio.FormatDirectory, func() { + env.Component(COMPONENT, func() { + env.Version(VERSION, func() { + env.Provider(PROVIDER) + env.Resource("artifact", "", resourcetypes.OCI_IMAGE, metav1.LocalRelation, func() { + env.Access( + ociartifact.New(oci.StandardOCIRef(OCIHOST2+".alias", OCINAMESPACE, OCIVERSION)), + ) + }) + }) + }) + }) +} + +func DifferentNamespace(env *Builder) { + env.OCICommonTransport(ARCH, accessio.FormatDirectory, func() { + OCIManifest2For(env, OCINAMESPACE2, OCIVERSION) + }) + env.OCMCommonTransport(ARCH, accessio.FormatDirectory, func() { + env.Component(COMPONENT, func() { + env.Version(VERSION, func() { + env.Provider(PROVIDER) + env.Resource("artifact", "", resourcetypes.OCI_IMAGE, metav1.LocalRelation, func() { + env.Access( + relativeociref.New(oci.RelativeOCIRef(OCINAMESPACE2, OCIVERSION)), + ) + }) + }) + }) + }) +} + +func DifferentName(env *Builder) { + env.OCICommonTransport(ARCH, accessio.FormatDirectory, func() { + OCIManifest1For(env, OCINAMESPACE, OCIVERSION) + }) + env.OCMCommonTransport(ARCH, accessio.FormatDirectory, func() { + env.Component(COMPONENT, func() { + env.Version(VERSION, func() { + env.Provider(PROVIDER) + env.Resource("other", "", resourcetypes.OCI_IMAGE, metav1.LocalRelation, func() { + env.Access( + ociartifact.New(oci.StandardOCIRef(OCIHOST2+".alias", OCINAMESPACE, OCIVERSION)), + ) + }) + }) + }) + }) +} + +func FakeOCIRegBaseFunction(ctx *storagecontext.StorageContext) string { + return OCIHOST2 + ".alias" +} + +func CheckBlob(r ocm.ResourceAccess, dig string, size int) { + blob := Must(r.BlobAccess()) + defer Close(blob, "blob") + + ExpectWithOffset(1, int(blob.Size())).To(Equal(size)) + set := Must(artifactset.OpenFromBlob(accessobj.ACC_READONLY, blob)) + defer Close(set, "set") + + digest := set.GetMain() + ExpectWithOffset(1, digest.Hex()).To(Equal(dig)) + + acc := Must(set.GetArtifact(digest.String())) + defer Close(acc, "acc") + + ExpectWithOffset(1, acc.IsManifest()).To(BeTrue()) + ExpectWithOffset(1, acc.Digest().Hex()).To(Equal(dig)) +} + +func CheckAritifact(env *Builder, dig string, size int) { + repo := Must(ocictf.Open(env, accessobj.ACC_READONLY, ARCH, 0, env)) + defer Close(repo, "oci repo") + + art := Must(repo.LookupArtifact(OCINAMESPACE, OCIVERSION)) + defer Close(art, "art") + + ExpectWithOffset(1, art.IsManifest()).To(BeTrue()) + ExpectWithOffset(1, art.Digest().Hex()).To(Equal(dig)) +}