Validate method for oci/ocm repository specs (#866)

#### What this PR does / why we need it: This PR introduces a `Validate` method for OCI and OCM repository specifications. Its task is to validate the spec as well as the connectivity (and potentially the credentials), if possible. Checking credentials makes only sense, if there is no possibility to use different credentials for different shards of the repository (for example organizations in the GitHub OCI registry).
open-component-model · Aug 23, 2024 · e77f8c6 · e77f8c6
1 parent e46befa
commit e77f8c6
Show file tree

Hide file tree

Showing 20 changed files with 359 additions and 17 deletions.
diff --git a/api/oci/extensions/repositories/artifactset/format.go b/api/oci/extensions/repositories/artifactset/format.go
@@ -6,6 +6,7 @@ import (
 	"github.com/mandelsoft/goutils/errors"
 	"github.com/mandelsoft/vfs/pkg/vfs"
 
+	"ocm.software/ocm/api/oci/artdesc"
 	"ocm.software/ocm/api/utils/accessio"
 	"ocm.software/ocm/api/utils/accessobj"
 	"ocm.software/ocm/api/utils/blobaccess/blobaccess"
@@ -45,14 +46,22 @@ type accessObjectInfo struct {
 
 var _ accessobj.AccessObjectInfo = (*accessObjectInfo)(nil)
 
+var baseInfo = accessobj.DefaultAccessObjectInfo{
+	ObjectTypeName:           "artifactset",
+	ElementDirectoryName:     BlobsDirectoryName,
+	ElementTypeName:          "blob",
+	DescriptorHandlerFactory: NewStateHandler,
+	DescriptorValidator:      validateDescriptor,
+}
+
+func validateDescriptor(data []byte) error {
+	_, err := artdesc.DecodeIndex(data)
+	return err
+}
+
 func NewAccessObjectInfo(fmts ...string) accessobj.AccessObjectInfo {
 	a := &accessObjectInfo{
-		accessobj.DefaultAccessObjectInfo{
-			ObjectTypeName:           "artifactset",
-			ElementDirectoryName:     BlobsDirectoryName,
-			ElementTypeName:          "blob",
-			DescriptorHandlerFactory: NewStateHandler,
-		},
+		baseInfo,
 	}
 	oci := IsOCIDefaultFormat()
 	if len(fmts) > 0 {

diff --git a/api/oci/extensions/repositories/artifactset/type.go b/api/oci/extensions/repositories/artifactset/type.go
@@ -4,6 +4,7 @@ import (
 	"github.com/mandelsoft/vfs/pkg/vfs"
 
 	"ocm.software/ocm/api/credentials"
+	"ocm.software/ocm/api/datacontext/attrs/vfsattr"
 	"ocm.software/ocm/api/oci/cpi"
 	"ocm.software/ocm/api/utils/accessio"
 	"ocm.software/ocm/api/utils/accessobj"
@@ -78,11 +79,19 @@ func (a *RepositorySpec) Repository(ctx cpi.Context, creds credentials.Credentia
 	return NewRepository(ctx, a)
 }
 
-func (a *RepositorySpec) AsUniformSpec(cpi.Context) cpi.UniformRepositorySpec {
+func (a *RepositorySpec) AsUniformSpec(ctx cpi.Context) cpi.UniformRepositorySpec {
 	opts, _ := NewOptions(&a.Options) // now unknown option possible (same Options type)
+	opts.Default(vfsattr.Get(ctx))
 	p, err := vfs.Canonical(opts.GetPathFileSystem(), a.FilePath, false)
 	if err != nil {
 		return cpi.UniformRepositorySpec{Type: a.GetKind(), Info: a.FilePath}
 	}
 	return cpi.UniformRepositorySpec{Type: a.GetKind(), Info: p}
 }
+
+func (a *RepositorySpec) Validate(ctx cpi.Context, creds credentials.Credentials, context ...credentials.UsageContext) error {
+	opts := a.Options
+	opts.Default(vfsattr.Get(ctx))
+
+	return accessobj.ValidateDescriptor(&baseInfo, a.FilePath, opts.GetPathFileSystem())
+}
diff --git a/api/oci/extensions/repositories/ctf/format.go b/api/oci/extensions/repositories/ctf/format.go
@@ -10,6 +10,7 @@ import (
 
 	"ocm.software/ocm/api/oci/cpi"
 	"ocm.software/ocm/api/oci/extensions/repositories/ctf/format"
+	"ocm.software/ocm/api/oci/extensions/repositories/ctf/index"
 	"ocm.software/ocm/api/utils/accessio"
 	"ocm.software/ocm/api/utils/accessobj"
 	"ocm.software/ocm/api/utils/blobaccess/blobaccess"
@@ -26,6 +27,12 @@ var accessObjectInfo = &accessobj.DefaultAccessObjectInfo{
 	ElementDirectoryName:     BlobsDirectoryName,
 	ElementTypeName:          "blob",
 	DescriptorHandlerFactory: NewStateHandler,
+	DescriptorValidator:      validateDescriptor,
+}
+
+func validateDescriptor(data []byte) error {
+	_, err := index.Decode(data)
+	return err
 }
 
 type Object = Repository

diff --git a/api/oci/extensions/repositories/ctf/type.go b/api/oci/extensions/repositories/ctf/type.go
@@ -4,6 +4,7 @@ import (
 	"strings"
 
 	"ocm.software/ocm/api/credentials"
+	"ocm.software/ocm/api/datacontext/attrs/vfsattr"
 	"ocm.software/ocm/api/oci/cpi"
 	"ocm.software/ocm/api/utils/accessio"
 	"ocm.software/ocm/api/utils/accessobj"
@@ -81,3 +82,10 @@ func (s *RepositorySpec) UniformRepositorySpec() *cpi.UniformRepositorySpec {
 func (a *RepositorySpec) Repository(ctx cpi.Context, creds credentials.Credentials) (cpi.Repository, error) {
 	return Open(ctx, a.AccessMode, a.FilePath, 0o700, &a.StandardOptions)
 }
+
+func (a *RepositorySpec) Validate(ctx cpi.Context, creds credentials.Credentials, context ...credentials.UsageContext) error {
+	opts := a.StandardOptions
+	opts.Default(vfsattr.Get(ctx))
+
+	return accessobj.ValidateDescriptor(accessObjectInfo, a.FilePath, opts.GetPathFileSystem())
+}
diff --git a/api/oci/extensions/repositories/docker/docker_test.go b/api/oci/extensions/repositories/docker/docker_test.go
@@ -0,0 +1,20 @@
+//go:build docker_test
+
+package docker_test
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+
+	"github.com/mandelsoft/goutils/testutils"
+
+	"ocm.software/ocm/api/oci"
+	"ocm.software/ocm/api/oci/extensions/repositories/docker"
+)
+
+var _ = Describe("Local Docker Daemon", func() {
+	It("validated access", func() {
+		octx := oci.DefaultContext()
+		spec := docker.NewRepositorySpec()
+		testutils.MustBeSuccessful(spec.Validate(octx, nil))
+	})
+})
diff --git a/api/oci/extensions/repositories/docker/suite_test.go b/api/oci/extensions/repositories/docker/suite_test.go
@@ -0,0 +1,13 @@
+package docker_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestConfig(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "docker daemon Test Suite")
+}
diff --git a/api/oci/extensions/repositories/docker/type.go b/api/oci/extensions/repositories/docker/type.go
@@ -1,6 +1,8 @@
 package docker
 
 import (
+	"context"
+
 	"ocm.software/ocm/api/credentials"
 	"ocm.software/ocm/api/oci/cpi"
 	"ocm.software/ocm/api/utils"
@@ -46,3 +48,13 @@ func (a *RepositorySpec) UniformRepositorySpec() *cpi.UniformRepositorySpec {
 func (a *RepositorySpec) Repository(ctx cpi.Context, creds credentials.Credentials) (cpi.Repository, error) {
 	return NewRepository(ctx, a)
 }
+
+func (a *RepositorySpec) Validate(ctx cpi.Context, creds credentials.Credentials, usageContext ...credentials.UsageContext) error {
+	client, err := newDockerClient(a.DockerHost)
+	if err != nil {
+		return err
+	}
+
+	_, err = client.Ping(context.Background())
+	return err
+}
diff --git a/api/oci/extensions/repositories/empty/type.go b/api/oci/extensions/repositories/empty/type.go
@@ -49,3 +49,7 @@ func (a *RepositorySpec) UniformRepositorySpec() *cpi.UniformRepositorySpec {
 func (a *RepositorySpec) Repository(ctx cpi.Context, creds credentials.Credentials) (cpi.Repository, error) {
 	return ctx.GetAttributes().GetOrCreateAttribute(ATTR_REPOS, func(datacontext.Context) interface{} { return NewRepository(ctx) }).(cpi.Repository), nil
 }
+
+func (a *RepositorySpec) Validate(cpi.Context, credentials.Credentials, ...credentials.UsageContext) error {
+	return nil
+}
diff --git a/api/oci/extensions/repositories/ocireg/type.go b/api/oci/extensions/repositories/ocireg/type.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/containerd/containerd/reference"
 
@@ -12,6 +13,7 @@ import (
 	"ocm.software/ocm/api/oci/cpi"
 	"ocm.software/ocm/api/utils"
 	"ocm.software/ocm/api/utils/runtime"
+	"ocm.software/ocm/api/utils/tcp"
 )
 
 const (
@@ -128,6 +130,18 @@ func (a *RepositorySpec) Repository(ctx cpi.Context, creds credentials.Credentia
 	return NewRepository(ctx, a, info)
 }
 
+func (a *RepositorySpec) Validate(ctx cpi.Context, creds credentials.Credentials, context ...credentials.UsageContext) error {
+	info, err := a.getInfo(creds)
+	if err != nil {
+		return err
+	}
+	h, p, _ := info.HostInfo()
+	if p == "" {
+		p = "443"
+	}
+	return tcp.PingTCPServer(h+":"+p, time.Second)
+}
+
 func (a *RepositorySpec) GetConsumerId(uctx ...credentials.UsageContext) credentials.ConsumerIdentity {
 	info, err := a.getInfo(nil)
 	if err != nil {

diff --git a/api/oci/internal/repotypes.go b/api/oci/internal/repotypes.go
@@ -26,6 +26,8 @@ type RepositorySpec interface {
 	Name() string
 	UniformRepositorySpec() *UniformRepositorySpec
 	Repository(Context, credentials.Credentials) (Repository, error)
+
+	Validate(Context, credentials.Credentials, ...credentials.UsageContext) error
 }
 
 type (
@@ -95,6 +97,10 @@ func (r *UnknownRepositorySpec) Repository(Context, credentials.Credentials) (Re
 	return nil, errors.ErrUnknown("repository type", r.GetType())
 }
 
+func (r *UnknownRepositorySpec) Validate(Context, credentials.Credentials, ...credentials.UsageContext) error {
+	return errors.ErrUnknown("repository type", r.GetType())
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 
 type GenericRepositorySpec struct {
@@ -157,4 +163,12 @@ func (s *GenericRepositorySpec) Repository(ctx Context, creds credentials.Creden
 	return spec.Repository(ctx, creds)
 }
 
+func (s *GenericRepositorySpec) Validate(ctx Context, creds credentials.Credentials, context ...credentials.UsageContext) error {
+	spec, err := s.Evaluate(ctx)
+	if err != nil {
+		return err
+	}
+	return spec.Validate(ctx, creds)
+}
+
 ////////////////////////////////////////////////////////////////////////////////
diff --git a/api/ocm/extensions/repositories/comparch/comparch_test.go b/api/ocm/extensions/repositories/comparch/comparch_test.go
@@ -49,6 +49,22 @@ var _ = Describe("Repository", func() {
 		// spec will not equal r as the filesystem cannot be serialized
 	})
 
+	It("validates component archive with resource stored as tar", func() {
+		// this is the typical use case
+		octx := ocm.DefaultContext()
+		spec := Must(comparch.NewRepositorySpec(accessobj.ACC_READONLY, TAR_COMPARCH))
+
+		MustBeSuccessful(spec.Validate(octx, nil))
+	})
+
+	It("validates component archive with resource stored as dir", func() {
+		// this is the typical use case
+		octx := ocm.DefaultContext()
+		spec := Must(comparch.NewRepositorySpec(accessobj.ACC_READONLY, DIR_COMPARCH))
+
+		MustBeSuccessful(spec.Validate(octx, nil))
+	})
+
 	It("component archive with resource stored as tar", func() {
 		// this is the typical use case
 		octx := ocm.DefaultContext()
@@ -87,7 +103,7 @@ var _ = Describe("Repository", func() {
 		Expect(bufferA).To(Equal(bufferB))
 	})
 
-	It("creates component archive", func() {
+	It("creates component archive directory", func() {
 		octx := ocm.DefaultContext()
 		memfs := memoryfs.New()
 
@@ -111,6 +127,50 @@ var _ = Describe("Repository", func() {
 		}))
 
 		MustBeSuccessful(finalize.Finalize())
+		Expect(vfs.DirExists(memfs, "test")).To(BeTrue())
+
+		spec := Must(comparch.NewRepositorySpec(accessobj.ACC_READONLY, "test", accessio.PathFileSystem(memfs)))
+		MustBeSuccessful(spec.Validate(octx, nil))
+
+		arch = Must(comparch.Open(octx, accessobj.ACC_WRITABLE, "test", 0o0700, accessio.PathFileSystem(memfs)))
+		finalize.Close(arch, "comparch)")
+
+		res = Must(arch.GetResourcesByName("blob"))
+		Expect(res[0].Meta().Digest).To(DeepEqual(&metav1.DigestSpec{
+			HashAlgorithm:          sha256.Algorithm,
+			NormalisationAlgorithm: blob.GenericBlobDigestV1,
+			Value:                  D_TESTDATA,
+		}))
+	})
+
+	It("creates component archive tgz", func() {
+		octx := ocm.DefaultContext()
+		memfs := memoryfs.New()
+
+		var finalize finalizer.Finalizer
+		defer Defer(finalize.Finalize)
+
+		arch := Must(comparch.Create(octx, accessobj.ACC_WRITABLE, "test", 0o0700, accessio.FormatTGZ, accessio.PathFileSystem(memfs)))
+		finalize.Close(arch, "comparch)")
+
+		arch.SetName("acme.org/test")
+		arch.SetVersion("v1.0.1")
+
+		MustBeSuccessful(arch.SetResourceBlob(compdesc.NewResourceMeta("blob", resourcetypes.PLAIN_TEXT, metav1.LocalRelation),
+			blobaccess.ForString(mime.MIME_TEXT, S_TESTDATA), "", nil))
+
+		res := Must(arch.GetResourcesByName("blob"))
+		Expect(res[0].Meta().Digest).To(DeepEqual(&metav1.DigestSpec{
+			HashAlgorithm:          sha256.Algorithm,
+			NormalisationAlgorithm: blob.GenericBlobDigestV1,
+			Value:                  D_TESTDATA,
+		}))
+
+		MustBeSuccessful(finalize.Finalize())
+		Expect(vfs.FileExists(memfs, "test")).To(BeTrue())
+
+		spec := Must(comparch.NewRepositorySpec(accessobj.ACC_READONLY, "test", accessio.PathFileSystem(memfs)))
+		MustBeSuccessful(spec.Validate(octx, nil))
 
 		arch = Must(comparch.Open(octx, accessobj.ACC_WRITABLE, "test", 0o0700, accessio.PathFileSystem(memfs)))
 		finalize.Close(arch, "comparch)")

diff --git a/api/ocm/extensions/repositories/comparch/format.go b/api/ocm/extensions/repositories/comparch/format.go
@@ -20,10 +20,16 @@ const BlobsDirectoryName = "blobs"
 
 var accessObjectInfo = &accessobj.DefaultAccessObjectInfo{
 	DescriptorFileName:       ComponentDescriptorFileName,
-	ObjectTypeName:           "artifactset",
+	ObjectTypeName:           "component archive",
 	ElementDirectoryName:     BlobsDirectoryName,
 	ElementTypeName:          "blob",
 	DescriptorHandlerFactory: NewStateHandler,
+	DescriptorValidator:      validateDescriptor,
+}
+
+func validateDescriptor(data []byte) error {
+	_, err := compdesc.Decode(data)
+	return err
 }
 
 type Object = ComponentArchive

diff --git a/api/ocm/extensions/repositories/comparch/type.go b/api/ocm/extensions/repositories/comparch/type.go
@@ -4,6 +4,7 @@ import (
 	"github.com/mandelsoft/vfs/pkg/vfs"
 
 	"ocm.software/ocm/api/credentials"
+	"ocm.software/ocm/api/datacontext/attrs/vfsattr"
 	"ocm.software/ocm/api/ocm/cpi"
 	"ocm.software/ocm/api/utils/accessio"
 	"ocm.software/ocm/api/utils/accessobj"
@@ -63,12 +64,20 @@ func (a *RepositorySpec) Repository(ctx cpi.Context, creds credentials.Credentia
 	return NewRepository(ctx, a)
 }
 
-func (a *RepositorySpec) AsUniformSpec(cpi.Context) *cpi.UniformRepositorySpec {
-	opts := &accessio.StandardOptions{}
-	opts.Default()
+func (a *RepositorySpec) AsUniformSpec(ctx cpi.Context) *cpi.UniformRepositorySpec {
+	opts := a.StandardOptions
+	opts.Default(vfsattr.Get(ctx))
+
 	p, err := vfs.Canonical(opts.GetPathFileSystem(), a.FilePath, false)
 	if err != nil {
 		return &cpi.UniformRepositorySpec{Type: a.GetKind(), SubPath: a.FilePath}
 	}
 	return &cpi.UniformRepositorySpec{Type: a.GetKind(), SubPath: p}
 }
+
+func (a *RepositorySpec) Validate(ctx cpi.Context, creds credentials.Credentials, context ...credentials.UsageContext) error {
+	opts := a.StandardOptions
+	opts.Default(vfsattr.Get(ctx))
+
+	return accessobj.ValidateDescriptor(accessObjectInfo, a.FilePath, opts.GetPathFileSystem())
+}
diff --git a/api/ocm/extensions/repositories/composition/type.go b/api/ocm/extensions/repositories/composition/type.go
@@ -38,4 +38,8 @@ func (r *RepositorySpec) Repository(ctx cpi.Context, credentials credentials.Cre
 	return NewRepository(ctx, r.Name), nil
 }
 
+func (a *RepositorySpec) Validate(ctx cpi.Context, creds credentials.Credentials, context ...credentials.UsageContext) error {
+	return nil
+}
+
 var _ cpi.RepositorySpec = (*RepositorySpec)(nil)
diff --git a/api/ocm/extensions/repositories/genericocireg/type.go b/api/ocm/extensions/repositories/genericocireg/type.go
@@ -180,6 +180,10 @@ func (s *RepositorySpec) GetIdentityMatcher() string {
 	return credentials.GetProvidedIdentityMatcher(s.RepositorySpec)
 }
 
+func (s *RepositorySpec) Validate(ctx cpi.Context, creds credentials.Credentials, uctx ...credentials.UsageContext) error {
+	return s.RepositorySpec.Validate(ctx.OCIContext(), creds, uctx...)
+}
+
 func DefaultComponentRepositoryMeta(meta *ComponentRepositoryMeta) *ComponentRepositoryMeta {
 	if meta == nil {
 		meta = &ComponentRepositoryMeta{}