diff --git a/.github/workflows/components.yaml b/.github/workflows/components.yaml index c6f8d53bd..520bc7563 100644 --- a/.github/workflows/components.yaml +++ b/.github/workflows/components.yaml @@ -30,7 +30,7 @@ permissions: env: REF: ${{ inputs.ref == '' && github.ref || inputs.ref }} CTF_TYPE: directory - components: '["ocmcli", "helminstaller", "helmdemo", "subchartsdemo", "ecrplugin"]' + components: '["ocmcli", "helminstaller", "helmdemo", "subchartsdemo", "ecrplugin", "jfrogplugin"]' IMAGE_PLATFORMS: 'linux/amd64 linux/arm64' PLATFORMS: 'windows/amd64 darwin/arm64 darwin/amd64 linux/amd64 linux/arm64' BUILDX_CACHE_PUSH: false diff --git a/api/ocm/plugin/cache/updater.go b/api/ocm/plugin/cache/updater.go index 17978059e..22441765b 100644 --- a/api/ocm/plugin/cache/updater.go +++ b/api/ocm/plugin/cache/updater.go @@ -296,44 +296,57 @@ func (o *PluginUpdater) download(session ocm.Session, cv ocm.ComponentVersionAcc return nil } } + dir := plugindirattr.Get(o.Context) - if dir != "" { - lock, err := filelock.LockDir(dir) + if dir == "" { + home, err := os.UserHomeDir() // use home if provided if err != nil { - return err + return fmt.Errorf("failed to determine home directory to determine default plugin directory: %w", err) } - defer lock.Close() - - target := filepath.Join(dir, desc.PluginName) - - verb := "installing" - if ok, _ := vfs.FileExists(fs, target); ok { - if !o.Force && (cv.GetVersion() == o.Current || !o.UpdateMode) { - return fmt.Errorf("plugin %s already found in %s", desc.PluginName, dir) - } - if o.UpdateMode { - verb = "updating" - } - fs.Remove(target) + dir = filepath.Join(home, plugindirattr.DEFAULT_PLUGIN_DIR) + if err := os.Mkdir(dir, os.ModePerm|os.ModeDir); err != nil { + return fmt.Errorf("failed to create default plugin directory: %w", err) } - o.Printer.Printf("%s plugin %s[%s] in %s...\n", verb, desc.PluginName, desc.PluginVersion, dir) - dst, err := fs.OpenFile(target, vfs.O_CREATE|vfs.O_TRUNC|vfs.O_WRONLY, 0o755) - if err != nil { - return errors.Wrapf(err, "cannot create plugin file %s", target) + if err := plugindirattr.Set(o.Context, dir); err != nil { + return fmt.Errorf("failed to set plugin dir after defaulting: %w", err) } - src, err := fs.OpenFile(file.Name(), vfs.O_RDONLY, 0) - if err != nil { - dst.Close() - return errors.Wrapf(err, "cannot open plugin executable %s", file.Name()) + } + + lock, err := filelock.LockDir(dir) + if err != nil { + return err + } + defer lock.Close() + + target := filepath.Join(dir, desc.PluginName) + + verb := "installing" + if ok, _ := vfs.FileExists(fs, target); ok { + if !o.Force && (cv.GetVersion() == o.Current || !o.UpdateMode) { + return fmt.Errorf("plugin %s already found in %s", desc.PluginName, dir) } - _, err = io.Copy(dst, src) - dst.Close() - utils.IgnoreError(src.Close()) - utils.IgnoreError(os.Remove(file.Name())) - utils.IgnoreError(SetPluginSourceInfo(dir, cv, found.Meta().Name, desc.PluginName)) - if err != nil { - return errors.Wrapf(err, "cannot copy plugin file %s", target) + if o.UpdateMode { + verb = "updating" } + fs.Remove(target) + } + o.Printer.Printf("%s plugin %s[%s] in %s...\n", verb, desc.PluginName, desc.PluginVersion, dir) + dst, err := fs.OpenFile(target, vfs.O_CREATE|vfs.O_TRUNC|vfs.O_WRONLY, 0o755) + if err != nil { + return errors.Wrapf(err, "cannot create plugin file %s", target) + } + src, err := fs.OpenFile(file.Name(), vfs.O_RDONLY, 0) + if err != nil { + dst.Close() + return errors.Wrapf(err, "cannot open plugin executable %s", file.Name()) + } + _, err = io.Copy(dst, src) + dst.Close() + utils.IgnoreError(src.Close()) + utils.IgnoreError(os.Remove(file.Name())) + utils.IgnoreError(SetPluginSourceInfo(dir, cv, found.Meta().Name, desc.PluginName)) + if err != nil { + return errors.Wrapf(err, "cannot copy plugin file %s", target) } } return nil diff --git a/api/ocm/plugin/plugin.go b/api/ocm/plugin/plugin.go index f546a0781..a3190d304 100644 --- a/api/ocm/plugin/plugin.go +++ b/api/ocm/plugin/plugin.go @@ -6,10 +6,12 @@ import ( "fmt" "io" "os" + "strings" "sync" "github.com/mandelsoft/goutils/errors" "github.com/mandelsoft/goutils/finalizer" + mlog "github.com/mandelsoft/logging" "github.com/mandelsoft/vfs/pkg/vfs" "ocm.software/ocm/api/credentials" @@ -112,11 +114,32 @@ func (p *pluginImpl) Exec(r io.Reader, w io.Writer, args ...string) (result []by args = append([]string{"--" + ppi.OptPlugingLogConfig, string(data)}, args...) } - if len(p.config) == 0 { - p.ctx.Logger(TAG).Debug("execute plugin action", "path", p.Path(), "args", args) - } else { - p.ctx.Logger(TAG).Debug("execute plugin action", "path", p.Path(), "args", args, "config", p.config) + if p.ctx.Logger(TAG).Enabled(mlog.DebugLevel) { + // Plainly kill any credentials found in the logger. + // Stupidly match for "credentials" arg. + // Not totally safe, but better than nothing. + logargs := make([]string, len(args)) + for i, arg := range args { + if logargs[i] != "" { + continue + } + if strings.Contains(arg, "credentials") { + if strings.Contains(arg, "=") { + logargs[i] = "***" + } else if i+1 < len(args)-1 { + logargs[i+1] = "***" + } + } + logargs[i] = arg + } + + if len(p.config) == 0 { + p.ctx.Logger(TAG).Debug("execute plugin action", "path", p.Path(), "args", logargs) + } else { + p.ctx.Logger(TAG).Debug("execute plugin action", "path", p.Path(), "args", logargs, "config", p.config) + } } + data, err := cache.Exec(p.Path(), p.config, r, w, args...) if logfile != nil { diff --git a/api/ocm/plugin/ppi/cmds/upload/put/cmd.go b/api/ocm/plugin/ppi/cmds/upload/put/cmd.go index 7bdac8633..bcb013ab1 100644 --- a/api/ocm/plugin/ppi/cmds/upload/put/cmd.go +++ b/api/ocm/plugin/ppi/cmds/upload/put/cmd.go @@ -3,7 +3,6 @@ package put import ( "encoding/json" "fmt" - "io" "os" "github.com/mandelsoft/goutils/errors" @@ -73,7 +72,7 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { func (o *Options) Complete(args []string) error { o.Name = args[0] if err := runtime.DefaultYAMLEncoding.Unmarshal([]byte(args[1]), &o.Specification); err != nil { - return errors.Wrapf(err, "invalid repository specification") + return fmt.Errorf("invalid repository specification: %w", err) } return nil } @@ -81,30 +80,33 @@ func (o *Options) Complete(args []string) error { func Command(p ppi.Plugin, cmd *cobra.Command, opts *Options) error { spec, err := p.DecodeUploadTargetSpecification(opts.Specification) if err != nil { - return errors.Wrapf(err, "target specification") + return fmt.Errorf("target specification: %w", err) } u := p.GetUploader(opts.Name) if u == nil { return errors.ErrNotFound(descriptor.KIND_UPLOADER, fmt.Sprintf("%s:%s", opts.ArtifactType, opts.MediaType)) } - w, h, err := u.Writer(p, opts.ArtifactType, opts.MediaType, opts.Hint, spec, opts.Credentials) + + fi, err := os.Stdin.Stat() if err != nil { - return err + return fmt.Errorf("failed to stat stdin: %w", err) } - _, err = io.Copy(w, os.Stdin) - if err != nil { - w.Close() - return err + if size := fi.Size(); size == 0 { + return fmt.Errorf("stdin is empty, and nothing can be uploaded") } - err = w.Close() + + h, err := u.Upload(p, opts.ArtifactType, opts.MediaType, opts.Hint, spec, opts.Credentials, os.Stdin) if err != nil { - return err + return fmt.Errorf("upload failed: %w", err) } + acc := h() + data, err := json.Marshal(acc) if err == nil { cmd.Printf("%s\n", string(data)) } + return err } diff --git a/api/ocm/plugin/ppi/interface.go b/api/ocm/plugin/ppi/interface.go index bb35c71b0..152352d81 100644 --- a/api/ocm/plugin/ppi/interface.go +++ b/api/ocm/plugin/ppi/interface.go @@ -113,7 +113,7 @@ type Uploader interface { Description() string ValidateSpecification(p Plugin, spec UploadTargetSpec) (info *UploadTargetSpecInfo, err error) - Writer(p Plugin, arttype, mediatype string, hint string, spec UploadTargetSpec, creds credentials.Credentials) (io.WriteCloser, AccessSpecProvider, error) + Upload(p Plugin, arttype, mediatype string, hint string, spec UploadTargetSpec, creds credentials.Credentials, reader io.Reader) (AccessSpecProvider, error) } type UploadTargetSpec = runtime.TypedObject diff --git a/cmds/demoplugin/uploaders/demo.go b/cmds/demoplugin/uploaders/demo.go index 706d7ffd5..26a61b7f8 100644 --- a/cmds/demoplugin/uploaders/demo.go +++ b/cmds/demoplugin/uploaders/demo.go @@ -56,7 +56,7 @@ func (a *Uploader) Decoders() ppi.UploadFormats { return types } -func (a *Uploader) ValidateSpecification(p ppi.Plugin, spec ppi.UploadTargetSpec) (*ppi.UploadTargetSpecInfo, error) { +func (a *Uploader) ValidateSpecification(_ ppi.Plugin, spec ppi.UploadTargetSpec) (*ppi.UploadTargetSpecInfo, error) { var info ppi.UploadTargetSpecInfo my := spec.(*TargetSpec) @@ -72,13 +72,13 @@ func (a *Uploader) ValidateSpecification(p ppi.Plugin, spec ppi.UploadTargetSpec return &info, nil } -func (a *Uploader) Writer(p ppi.Plugin, arttype, mediatype, hint string, repo ppi.UploadTargetSpec, creds credentials.Credentials) (io.WriteCloser, ppi.AccessSpecProvider, error) { +func (a *Uploader) Upload(p ppi.Plugin, _, mediatype, hint string, repo ppi.UploadTargetSpec, _ credentials.Credentials, reader io.Reader) (ppi.AccessSpecProvider, error) { var file *os.File var err error cfg, err := p.GetConfig() if err != nil { - return nil, nil, errors.Wrapf(err, "can't get config for access method %s", mediatype) + return nil, errors.Wrapf(err, "can't get config for access method %s", mediatype) } root := os.TempDir() @@ -86,7 +86,7 @@ func (a *Uploader) Writer(p ppi.Plugin, arttype, mediatype, hint string, repo pp root = cfg.(*config.Config).Uploaders.Path err := os.MkdirAll(root, 0o700) if err != nil { - return nil, nil, errors.Wrapf(err, "cannot create root dir") + return nil, errors.Wrapf(err, "cannot create root dir") } } @@ -106,7 +106,7 @@ func (a *Uploader) Writer(p ppi.Plugin, arttype, mediatype, hint string, repo pp err = os.MkdirAll(dir, 0o700) if err != nil { - return nil, nil, err + return nil, err } if hint == "" { @@ -115,8 +115,13 @@ func (a *Uploader) Writer(p ppi.Plugin, arttype, mediatype, hint string, repo pp file, err = os.OpenFile(filepath.Join(os.TempDir(), path), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) } if err != nil { - return nil, nil, err + return nil, err } writer := NewWriter(file, path, mediatype, hint == "", accessmethods.NAME, accessmethods.VERSION) - return writer, writer.Specification, nil + + if _, err = io.Copy(writer, reader); err != nil { + return nil, fmt.Errorf("cannot write to %q: %w", file.Name(), err) + } + + return writer.Specification, nil } diff --git a/cmds/jfrogplugin/main.go b/cmds/jfrogplugin/main.go new file mode 100644 index 000000000..5bc535fdb --- /dev/null +++ b/cmds/jfrogplugin/main.go @@ -0,0 +1,70 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "strconv" + + "ocm.software/ocm/api/config" + "ocm.software/ocm/api/ocm/extensions/artifacttypes" + "ocm.software/ocm/api/ocm/extensions/blobhandler" + "ocm.software/ocm/api/ocm/plugin" + "ocm.software/ocm/api/ocm/plugin/ppi" + "ocm.software/ocm/api/ocm/plugin/ppi/cmds" + "ocm.software/ocm/api/version" + "ocm.software/ocm/cmds/jfrogplugin/uploaders/helm" +) + +const NAME = "jfrog" + +func main() { + p := ppi.NewPlugin(NAME, version.Get().String()) + + p.SetShort(NAME + " plugin") + p.SetLong(`ALPHA GRADE plugin providing custom functions related to interacting with JFrog Repositories (e.g. Artifactory). + +This plugin is solely for interacting with JFrog Servers and cannot be used for generic repository types. +Thus, you should only consider this plugin if +- You need to use a JFrog specific API +- You cannot use any of the generic (non-jfrog) implementations. + +Examples: + +You can configure the JFrog plugin as an Uploader in an ocm config file with: + +- type: ` + fmt.Sprintf("%s.ocm.%s", plugin.KIND_UPLOADER, config.OCM_CONFIG_TYPE_SUFFIX) + ` + registrations: + - name: ` + fmt.Sprintf("%s/%s/%s", plugin.KIND_PLUGIN, NAME, helm.NAME) + ` + artifactType: ` + artifacttypes.HELM_CHART + ` + priority: 200 # must be > ` + strconv.Itoa(blobhandler.DEFAULT_BLOBHANDLER_PRIO) + ` to be used over the default handler + config: + type: ` + fmt.Sprintf("%s/%s", helm.NAME, helm.VERSION) + ` + # this is only a sample JFrog Server URL, do NOT append /artifactory + url: int.repositories.ocm.software + repository: ocm-helm-test +`) + p.SetConfigParser(GetConfig) + + u := helm.New() + if err := p.RegisterUploader(artifacttypes.HELM_CHART, "", u); err != nil { + panic(err) + } + err := cmds.NewPluginCommand(p).Execute(os.Args[1:]) + if err != nil { + fmt.Fprintf(os.Stderr, "error while running plugin: %v\n", err) + os.Exit(1) + } +} + +type Config struct { +} + +func GetConfig(raw json.RawMessage) (interface{}, error) { + var cfg Config + + if err := json.Unmarshal(raw, &cfg); err != nil { + return nil, fmt.Errorf("could not get config: %w", err) + } + return &cfg, nil +} diff --git a/cmds/jfrogplugin/uploaders/helm/headers.go b/cmds/jfrogplugin/uploaders/helm/headers.go new file mode 100644 index 000000000..7c677cf68 --- /dev/null +++ b/cmds/jfrogplugin/uploaders/helm/headers.go @@ -0,0 +1,25 @@ +package helm + +import ( + "net/http" + + "ocm.software/ocm/api/credentials" +) + +func SetHeadersFromCredentials(req *http.Request, creds credentials.Credentials) { + if creds == nil { + return + } + if creds.ExistsProperty(credentials.ATTR_TOKEN) { + req.Header.Set("Authorization", "Bearer "+creds.GetProperty(credentials.ATTR_TOKEN)) + } else { + var user, pass string + if creds.ExistsProperty(credentials.ATTR_USERNAME) { + user = creds.GetProperty(credentials.ATTR_USERNAME) + } + if creds.ExistsProperty(credentials.ATTR_PASSWORD) { + pass = creds.GetProperty(credentials.ATTR_PASSWORD) + } + req.SetBasicAuth(user, pass) + } +} diff --git a/cmds/jfrogplugin/uploaders/helm/helm.go b/cmds/jfrogplugin/uploaders/helm/helm.go new file mode 100644 index 000000000..6c08a931c --- /dev/null +++ b/cmds/jfrogplugin/uploaders/helm/helm.go @@ -0,0 +1,252 @@ +package helm + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strings" + "time" + + "github.com/containerd/containerd/reference" + + "ocm.software/ocm/api/credentials" + "ocm.software/ocm/api/credentials/cpi" + "ocm.software/ocm/api/credentials/identity/hostpath" + "ocm.software/ocm/api/ocm/extensions/artifacttypes" + "ocm.software/ocm/api/ocm/plugin/ppi" + "ocm.software/ocm/api/utils/runtime" +) + +const ( + NAME = "JFrogHelm" + + // VERSION of the Uploader TODO Increment once stable + VERSION = "v1alpha1" + + // ID_HOSTNAME is the hostname of the artifactory server to upload to + ID_HOSTNAME = hostpath.ID_HOSTNAME + // ID_PORT is the port of the artifactory server to upload to + ID_PORT = hostpath.ID_PORT + // ID_REPOSITORY is the repository name in JFrog Artifactory to upload to + ID_REPOSITORY = "repository" + + // DEFAULT_TIMEOUT is the default timeout for http requests issued by the uploader. + DEFAULT_TIMEOUT = time.Minute +) + +type JFrogHelmUploaderSpec struct { + runtime.ObjectVersionedType `json:",inline"` + + // URL is the hostname of the JFrog instance. + // Required for correct reference to Artifactory. + URL string `json:"url"` + + // Repository is the repository to upload to. + // Required for correct reference to Artifactory. + Repository string `json:"repository"` + + JFrogHelmChart `json:",inline"` + + // Timeout is the maximum duration the upload of the chart can take + // before aborting and failing. + // OPTIONAL: If not set, set to the internal DEFAULT_TIMEOUT. + Timeout *time.Duration `json:"timeout,omitempty"` +} + +type JFrogHelmChart struct { + // ChartName is the desired name of the chart in the repository. + // OPTIONAL: If not set, defaulted from the passed Hint. + Name string `json:"name,omitempty"` + // Version is the desired version of the chart + // OPTIONAL: If not set, defaulted from the passed Hint. + Version string `json:"version,omitempty"` +} + +func (s *JFrogHelmUploaderSpec) GetTimeout() time.Duration { + if s.Timeout == nil { + return DEFAULT_TIMEOUT + } + return *s.Timeout +} + +var types ppi.UploadFormats + +func init() { + decoder, err := runtime.NewDirectDecoder[runtime.TypedObject](&JFrogHelmUploaderSpec{}) + if err != nil { + panic(err) + } + types = ppi.UploadFormats{NAME + runtime.VersionSeparator + VERSION: decoder} +} + +func (a *Uploader) Decoders() ppi.UploadFormats { + return types +} + +type Uploader struct { + ppi.UploaderBase + *http.Client +} + +var _ ppi.Uploader = (*Uploader)(nil) + +func New() ppi.Uploader { + return &Uploader{ + UploaderBase: ppi.MustNewUploaderBase(NAME, "upload artifacts to JFrog HELM repositories by using the JFrog REST API."), + Client: http.DefaultClient, + } +} + +func (a *Uploader) ValidateSpecification(_ ppi.Plugin, spec ppi.UploadTargetSpec) (*ppi.UploadTargetSpecInfo, error) { + targetSpec, ok := spec.(*JFrogHelmUploaderSpec) + if !ok { + return nil, fmt.Errorf("invalid spec type %T", spec) + } + + info, err := ConvertTargetSpecToInfo(targetSpec) + if err != nil { + return nil, fmt.Errorf("failed to convert target spec to info: %w", err) + } + + return info, nil +} + +// Upload uploads any artifact that is of type artifacttypes.HELM_CHART. +// Process: +// 1. introspect the JFrogHelmUploaderSpec (cast from ppi.UploadTargetSpec) and hint parameter +// (the hint is expected to be an OCI style reference, such as `repo/comp:version`) +// 2. building an Artifactory Style JFrog Upload URL out of it (see ConvertTargetSpecToHelmUploadURL), +// 3. creating a request respecting the passed credentials based on SetHeadersFromCredentials +// 4. uploading the passed blob as is (expected to be a tgz byte stream) +// 5. intepreting the JFrog API response, and converting it from ArtifactoryUploadResponse to ppi.AccessSpec +func (a *Uploader) Upload(_ ppi.Plugin, artifactType, _, hint string, targetSpec ppi.UploadTargetSpec, creds credentials.Credentials, reader io.Reader) (ppi.AccessSpecProvider, error) { + if artifactType != artifacttypes.HELM_CHART { + return nil, fmt.Errorf("unsupported artifact type %s", artifactType) + } + + spec, ok := targetSpec.(*JFrogHelmUploaderSpec) + if !ok { + return nil, fmt.Errorf("the type %T is not a valid target spec type", spec) + } + + if err := EnsureSpecWithHelpFromHint(spec, hint); err != nil { + return nil, fmt.Errorf("could not ensure spec to be ready for upload: %w", err) + } + + targetURL, err := ConvertTargetSpecToHelmUploadURL(spec) + if err != nil { + return nil, fmt.Errorf("failed to convert target spec to URL: %w", err) + } + + ctx, cancel := context.WithTimeout(context.TODO(), spec.GetTimeout()) + defer cancel() + + access, err := Upload(ctx, reader, a.Client, targetURL, creds) + if err != nil { + return nil, fmt.Errorf("failed to upload") + } + + return func() ppi.AccessSpec { + return access + }, nil +} + +// ConvertTargetSpecToInfo converts the JFrogHelmUploaderSpec +// to a valid info block containing the consumer ID used +// in the library to identify the correct credentials that need to +// be passed to it. +func ConvertTargetSpecToInfo(spec *JFrogHelmUploaderSpec) (*ppi.UploadTargetSpecInfo, error) { + purl, err := parseURLAllowNoScheme(spec.URL) + if err != nil { + return nil, fmt.Errorf("failed to parse URL: %w", err) + } + + var info ppi.UploadTargetSpecInfo + + // By default, we identify an artifactory repository as a combination + // of Host & Repository + info.ConsumerId = credentials.ConsumerIdentity{ + cpi.ID_TYPE: NAME, + ID_HOSTNAME: purl.Hostname(), + ID_REPOSITORY: spec.Repository, + } + if purl.Port() != "" { + info.ConsumerId.SetNonEmptyValue(ID_PORT, purl.Port()) + } + + return &info, nil +} + +// EnsureSpecWithHelpFromHint introspects the hint and fills the target spec based on it. +// It makes sure that the spec can be used to access a JFrog Artifactory HELM Repository. +func EnsureSpecWithHelpFromHint(spec *JFrogHelmUploaderSpec, hint string) error { + if refFromHint, err := reference.Parse(hint); err == nil { + if refFromHint.Digest() != "" && refFromHint.Object == "" { + return fmt.Errorf("the hint contained a valid reference but it was a digest, so it cannot be used to deduce a version of the helm chart: %s", refFromHint) + } + if spec.Version == "" { + spec.Version = refFromHint.Object + } + if spec.Name == "" { + spec.Name = path.Base(refFromHint.Locator) + } + } + if spec.Name == "" { + return fmt.Errorf("the chart name could not be deduced from the hint (%s) or the config (%s)", hint, spec) + } + if spec.Version == "" { + return fmt.Errorf("the chart version could not be deduced from the hint (%s) or the config (%s)", hint, spec) + } + return nil +} + +// ConvertTargetSpecToHelmUploadURL interprets the JFrogHelmUploaderSpec into a valid REST API Endpoint URL to upload to. +// It requires a valid ChartName and ChartVersion to determine the correct URL endpoint. +// +// See https://jfrog.com/help/r/jfrog-rest-apis/deploy-artifact for the URL endpoint +// See https://jfrog.com/help/r/jfrog-artifactory-documentation/deploying-artifacts for artifact deployment reference +// See https://jfrog.com/help/r/jfrog-artifactory-documentation/use-the-jfrog-helm-client for the HELM Client reference. +// +// Example: +// +// JFrogHelmUploaderSpec.URL => demo.jfrog.ocm.software +// JFrogHelmUploaderSpec.Repository => my-charts +// JFrogHelmUploaderSpec.ChartName => podinfo +// JFrogHelmUploaderSpec.ChartVersion => 0.0.1 +// +// will result in +// +// url.URL => https://demo.jfrog.ocm.software/artifactory/my-charts/podinfo-0.0.1.tgz +func ConvertTargetSpecToHelmUploadURL(spec *JFrogHelmUploaderSpec) (*url.URL, error) { + requestURL := path.Join(spec.URL, "artifactory", spec.Repository, fmt.Sprintf("%s-%s.tgz", spec.Name, spec.Version)) + requestURLParsed, err := parseURLAllowNoScheme(requestURL) + if err != nil { + return nil, fmt.Errorf("failed to parse full request URL: %w", err) + } + return requestURLParsed, nil +} + +// parseURLAllowNoScheme is an adaptation / hack on url.Parse because +// url.Parse does not support parsing a URL without a prefixed scheme. +// However, we would like to accept these kind of URLs because we default them +// to "https://" out of convenience. +func parseURLAllowNoScheme(urlToParse string) (*url.URL, error) { + const dummyScheme = "dummy" + if !strings.Contains(urlToParse, "://") { + urlToParse = dummyScheme + "://" + urlToParse + } + parsedURL, err := url.Parse(urlToParse) + if err != nil { + return nil, err + } + if parsedURL.Scheme == dummyScheme { + parsedURL.Scheme = "" + } + if parsedURL.Scheme == "" { + parsedURL.Scheme = "https" + } + return parsedURL, nil +} diff --git a/cmds/jfrogplugin/uploaders/helm/upload.go b/cmds/jfrogplugin/uploaders/helm/upload.go new file mode 100644 index 000000000..b2ea079d2 --- /dev/null +++ b/cmds/jfrogplugin/uploaders/helm/upload.go @@ -0,0 +1,105 @@ +package helm + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strings" + + "ocm.software/ocm/api/credentials" + "ocm.software/ocm/api/ocm/extensions/accessmethods/helm" + "ocm.software/ocm/api/ocm/plugin/ppi" +) + +func Upload( + ctx context.Context, + data io.Reader, + client *http.Client, + url *url.URL, + creds credentials.Credentials, +) (_ ppi.AccessSpec, err error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + var req *http.Request + var res *http.Response + if req, err = http.NewRequestWithContext(ctx, http.MethodPut, url.String(), data); err != nil { + return nil, fmt.Errorf("failed to create HTTP request for upload: %w", err) + } + SetHeadersFromCredentials(req, creds) + + if res, err = client.Do(req); err != nil { + return nil, fmt.Errorf("failed to store blob in artifactory: %w", err) + } + defer func() { + err = errors.Join(err, res.Body.Close()) + }() + + if invalid := 200 > res.StatusCode || res.StatusCode >= 300; invalid { + var responseBytes []byte + if responseBytes, err = io.ReadAll(res.Body); err != nil { + var body string + if len(responseBytes) > 0 { + body = fmt.Sprintf(": %s", string(responseBytes)) + } + return nil, fmt.Errorf("invalid response (status %v)%s", res.StatusCode, body) + } + } + + uploadResponse := &ArtifactoryUploadResponse{} + if err = json.NewDecoder(res.Body).Decode(uploadResponse); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + + return uploadResponse.ToHelmAccessSpec() +} + +type ArtifactoryUploadResponse struct { + Repo string `json:"repo,omitempty"` + Path string `json:"path,omitempty"` + Created string `json:"created,omitempty"` + CreatedBy string `json:"createdBy,omitempty"` + DownloadUri string `json:"downloadUri,omitempty"` + MimeType string `json:"mimeType,omitempty"` + Size string `json:"size,omitempty"` + Checksums struct { + Sha1 string `json:"sha1,omitempty"` + Sha256 string `json:"sha256,omitempty"` + Md5 string `json:"md5,omitempty"` + } `json:"checksums,omitempty"` + Uri string `json:"uri"` +} + +func (r *ArtifactoryUploadResponse) URL() string { + if r.DownloadUri != "" { + return r.DownloadUri + } + return r.Uri +} + +func (r *ArtifactoryUploadResponse) ToHelmAccessSpec() (ppi.AccessSpec, error) { + u := r.URL() + urlp, err := url.Parse(u) + if err != nil { + return nil, err + } + chart := path.Base(urlp.Path) + chart = strings.TrimSuffix(chart, path.Ext(chart)) + + // this is needed so that the chart version constructor for OCM is happy + // OCM encodes helm charts with a ":"... + if idx := strings.LastIndex(chart, "-"); idx > 0 { + chart = chart[:idx] + ":" + chart[idx+1:] + } + + urlp.Path = "" + urlp = urlp.JoinPath("artifactory", "api", "helm", r.Repo) + repo := urlp.String() + + return helm.New(chart, repo), nil +} diff --git a/cmds/jfrogplugin/uploaders/helm/upload_test.go b/cmds/jfrogplugin/uploaders/helm/upload_test.go new file mode 100644 index 000000000..8d11feacc --- /dev/null +++ b/cmds/jfrogplugin/uploaders/helm/upload_test.go @@ -0,0 +1,82 @@ +package helm + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + neturl "net/url" + "strings" + "testing" + + "ocm.software/ocm/api/credentials" + "ocm.software/ocm/api/ocm/extensions/accessmethods/helm" +) + +func TestUpload(t *testing.T) { + const artifactory = "https://mocked.artifactory.localhost:9999" + const chartName, chartVersion, repo = "chart", "1.0.0", "repo" + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPut { + t.Fatalf("method is not PUT as expected by JFrog") + } + if user, pass, ok := r.BasicAuth(); !ok { + t.Fatalf("invalid basic auth: %s - %s", user, pass) + } + res := ArtifactoryUploadResponse{ + Repo: repo, + DownloadUri: fmt.Sprintf("%s/path/to/%s/%s-%s.tgz", artifactory, repo, chartName, chartVersion), + } + data, err := json.Marshal(res) + if err != nil { + t.Fatalf("failed to marshal response: %v", err) + } + if _, err := w.Write(data); err != nil { + t.Fatalf("failed to write response: %v", err) + } + })) + t.Cleanup(func() { + srv.Close() + }) + client := srv.Client() + + url, err := neturl.Parse(srv.URL) + if err != nil { + t.Fatalf("unexpected test client URL: %v", err) + } + + ctx := context.Background() + + data := strings.NewReader("testdata") + + accessSpec, err := Upload(ctx, data, client, url, credentials.DirectCredentials{ + credentials.ATTR_USERNAME: "foo", + credentials.ATTR_PASSWORD: "bar", + }) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if typ := accessSpec.GetType(); typ != helm.Type { + t.Fatalf("unexpected type: %v", typ) + } + + helmAccessSpec, ok := accessSpec.(*helm.AccessSpec) + if !ok { + t.Fatalf("unexpected cast failure to helm access spec") + } + + if specChart := helmAccessSpec.GetChartName(); specChart != chartName { + t.Fatalf("unexpected chart name: %v", specChart) + } + if specVersion := helmAccessSpec.GetVersion(); specVersion != chartVersion { + t.Fatalf("unexpected chart version: %v", specVersion) + } + + if helmAccessSpec.HelmRepository != fmt.Sprintf("%s/artifactory/api/helm/%s", artifactory, repo) { + t.Fatalf("expected an injected helm api reference to artifactory") + } + +} diff --git a/components/jfrogplugin/Makefile b/components/jfrogplugin/Makefile new file mode 100644 index 000000000..ef51b171e --- /dev/null +++ b/components/jfrogplugin/Makefile @@ -0,0 +1,127 @@ +NAME = jfrogplugin +PROVIDER ?= ocm.software +GITHUBORG ?= open-component-model +COMPONENT = $(PROVIDER)/plugins/$(NAME) +OCMREPO ?= ghcr.io/$(GITHUBORG)/ocm +PLATFORMS ?= linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 +CTF_TYPE ?= directory + + +REPO_ROOT := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))/../.. +ifeq ($(VERSION),) +VERSION := $(shell go run ../../api/version/generate/release_generate.go print-rc-version $(CANDIDATE)) +endif +COMMIT = $(shell git rev-parse --verify HEAD) +# if EFFECTIVE_VERSION is not set, set it to VERSION+COMMIT +# this is not the same as '?=' because it will also set the value if EFFECTIVE_VERSION is set to an empty string +ifeq ($(EFFECTIVE_VERSION),) +EFFECTIVE_VERSION := $(VERSION)+$(COMMIT) +endif +GIT_TREE_STATE := $(shell [ -z "$$(git status --porcelain 2>/dev/null)" ] && echo clean || echo dirty) + +CMDSRCS=$(shell find $(REPO_ROOT)/cmds/$(NAME) -type f) +OCMSRCS=$(shell find $(REPO_ROOT)/api -type f) $(REPO_ROOT)/go.* + +CREDS ?= +# Define the path to the binary +OCM_BIN = $(REPO_ROOT)/bin/ocm + +# Rule to build the binary if it doesn't exist or if the source code has changed +$(OCM_BIN): $(REPO_ROOT)/cmds/ocm/main.go + mkdir -p $(REPO_ROOT)/bin + go build -ldflags $(BUILD_FLAGS) -o $(OCM_BIN) $(REPO_ROOT)/cmds/ocm + +# Use the binary for the OCM command +OCM = $(OCM_BIN) $(CREDS) + +GEN = $(REPO_ROOT)/gen/$(NAME) + + $(GEN): + @mkdir -p $(GEN) + +NOW := $(shell date -u +%FT%T%z) +BUILD_FLAGS := "-s -w \ + -X ocm.software/ocm/api/version.gitVersion=$(EFFECTIVE_VERSION) \ + -X ocm.software/ocm/api/version.gitTreeState=$(GIT_TREE_STATE) \ + -X ocm.software/ocm/api/version.gitCommit=$(COMMIT) \ + -X ocm.software/ocm/api/version.buildDate=$(NOW)" + +.PHONY: build +build: $(GEN)/build + +$(GEN)/build: $(GEN) $(CMDSRCS) $(OCMSRCS) + @for i in $(PLATFORMS); do \ + tag=$$(echo $$i | sed -e s:/:-:g); \ + echo GOARCH=$$(basename $$i) GOOS=$$(dirname $$i) CGO_ENABLED=0 go build -ldflags $(BUILD_FLAGS) -o $(GEN)/$(NAME).$$tag ../../cmds/$(NAME); \ + GOARCH=$$(basename $$i) GOOS=$$(dirname $$i) CGO_ENABLED=0 go build -ldflags $(BUILD_FLAGS) -o $(GEN)/$(NAME).$$tag ../../cmds/$(NAME) & \ + done; \ + wait + @touch $(GEN)/build + + +.PHONY: ctf +ctf: $(GEN)/ctf + +$(GEN)/ctf: $(OCM_BIN) $(GEN)/.exists $(GEN)/build component-constructor.yaml $(CHARTSRCS) + @rm -rf "$(GEN)/ctf" + $(OCM) add componentversions \ + --create \ + --file $(GEN)/ctf \ + --type $(CTF_TYPE) \ + --templater=spiff \ + COMPONENT="$(COMPONENT)" \ + NAME="$(NAME)" \ + VERSION="$(VERSION)" \ + PROVIDER="$(PROVIDER)" \ + COMMIT="$(COMMIT)" \ + GEN="$(GEN)" \ + PLATFORMS="$(PLATFORMS)" \ + component-constructor.yaml + touch "$(GEN)/ctf" + +.PHONY: version +version: + @echo $(VERSION) + +.PHONY: push +push: $(GEN)/ctf $(GEN)/push.$(NAME) + +$(GEN)/push.$(NAME): $(GEN)/ctf $(OCM_BIN) + $(OCM) transfer ctf -f $(GEN)/ctf $(OCMREPO) + @touch $(GEN)/push.$(NAME) + +.PHONY: plain-push +plain-push: $(GEN) $(OCM_BIN) + $(OCM) transfer ctf -f $(GEN)/ctf $(OCMREPO) + @touch $(GEN)/push.$(NAME) + +.PHONY: transport +transport: $(OCM_BIN) +ifneq ($(TARGETREPO),) + $(OCM) transfer component -Vc $(OCMREPO)//$(COMPONENT):$(VERSION) $(TARGETREPO) +endif + +$(GEN)/.exists: + @mkdir -p $(GEN) + @touch $@ + +.PHONY: info +info: + @echo "ROOT: $(REPO_ROOT)" + @echo "VERSION: $(VERSION)" + @echo "COMMIT; $(COMMIT)" + +.PHONY: describe +describe: $(GEN)/ctf $(OCM_BIN) + $(OCM) get resources --lookup $(OCMREPO) -r -o treewide $(GEN)/ctf + +.PHONY: descriptor +descriptor: $(GEN)/ctf $(OCM_BIN) + $(OCM) get component -S v3alpha1 -o yaml $(GEN)/ctf + +.PHONY: clean +clean: + rm -rf $(GEN) + +install: $(GEN)/ctf $(OCM_BIN) + $(OCM) install plugin -f $(GEN)/ctf \ No newline at end of file diff --git a/components/jfrogplugin/bindings.yaml b/components/jfrogplugin/bindings.yaml new file mode 100644 index 000000000..ae8fb3bd1 --- /dev/null +++ b/components/jfrogplugin/bindings.yaml @@ -0,0 +1,4 @@ +values: + GEN: ../../gen/jfrogplugin + PLATFORMS: linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 + VERSION: "1.0" diff --git a/components/jfrogplugin/component-constructor.yaml b/components/jfrogplugin/component-constructor.yaml new file mode 100644 index 000000000..b5b7d3f51 --- /dev/null +++ b/components/jfrogplugin/component-constructor.yaml @@ -0,0 +1,23 @@ +--- +helper: + <<<: (( &temporary )) + executable: + <<<: (( &template )) + name: jfrog + type: ocmPlugin + version: (( values.VERSION )) + extraIdentity: + os: ((dirname(p) )) + architecture: (( basename(p) )) + input: + type: file + # Generate the path to the plugin binary by looking into the base path and encoding the platform + path: (( values.GEN "/" values.NAME "." replace(p,"/","-") )) + +components: + - name: (( values.COMPONENT)) + version: (( values.VERSION)) + provider: + name: (( values.PROVIDER)) + # use all platforms and create a resource for each + resources: (( map[split(" ", values.PLATFORMS)|p|-> *helper.executable] )) \ No newline at end of file diff --git a/components/jfrogplugin/sources.yaml b/components/jfrogplugin/sources.yaml new file mode 100644 index 000000000..dcc9c5f89 --- /dev/null +++ b/components/jfrogplugin/sources.yaml @@ -0,0 +1,7 @@ +name: source +type: filesytem +access: + type: github + repoUrl: github.com/open-component-model/ocm + commit: ${COMMIT} +version: ${VERSION}