diff --git a/.github/TODO.md b/.github/TODO.md deleted file mode 100644 index d49ae130..00000000 --- a/.github/TODO.md +++ /dev/null @@ -1,9 +0,0 @@ -# TODO - -- [x] ~~The code is currently super dirty, need to refactor heavily.~~ -- [x] ~~Get rid of Golang dependency. Plugin "install" hook should download -prebuilt **helms3** binary file from github releases.~~ -- [x] ~~Make `helm s3` command able to upload the charts to the repo. Remember -that helm has no build-in command like `push` or `publish`, so we need to provide -easy way to push charts to the repository.~~ -- [ ] On `helm s3 push` need to check that the file is a valid Helm chart. \ No newline at end of file diff --git a/README.md b/README.md index 3df35e50..a4efd3c9 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ The installation itself is simple as: You can install a specific release version: - $ helm plugin install https://github.com/hypnoglow/helm-s3.git --version 0.2.0 + $ helm plugin install https://github.com/hypnoglow/helm-s3.git --version 0.4.0 To use the plugin, you do not need any special dependencies. The installer will download versioned release with prebuilt binary from [github releases](https://github.com/hypnoglow/helm-s3/releases). @@ -58,7 +58,7 @@ Fetching also works: $ helm fetch s3://bucket-name/charts/epicservice-0.5.1.tgz -### Init & Push +### Init To create a new repository, use **init**: @@ -67,10 +67,12 @@ To create a new repository, use **init**: This command generates an empty **index.yaml** and uploads it to the S3 bucket under `/charts` key. -To push to this repo by it's name, you need to add it first: +To work with this repo by it's name, first you need to add it using native helm command: $ helm repo add mynewrepo s3://bucket-name/charts +### Push + Now you can push your chart to this repo: $ helm s3 push ./epicservice-0.7.2.tgz mynewrepo @@ -85,6 +87,21 @@ Now your pushed chart is available: NAME VERSION DESCRIPTION mynewrepo/epicservice 0.7.2 A Helm chart. +### Delete + +To delete specific chart version from the repository: + + $ helm s3 delete epicservice --version 0.7.2 mynewrepo + +As always, remote repo index updated automatically again. To sync local, run: + + $ helm repo update + +The chart is deleted from the repo: + + $ helm search mynewrepo/epicservice + No results found + ## Uninstall $ helm plugin remove s3 diff --git a/cmd/helms3/delete.go b/cmd/helms3/delete.go new file mode 100644 index 00000000..e8a1fc6f --- /dev/null +++ b/cmd/helms3/delete.go @@ -0,0 +1,73 @@ +package main + +import ( + "context" + "fmt" + + "github.com/pkg/errors" + + "github.com/hypnoglow/helm-s3/pkg/awss3" + "github.com/hypnoglow/helm-s3/pkg/awsutil" + "github.com/hypnoglow/helm-s3/pkg/helmutil" + "github.com/hypnoglow/helm-s3/pkg/index" +) + +func runDelete(name, version, repoName string) error { + repoEntry, err := helmutil.LookupRepoEntry(repoName) + if err != nil { + return err + } + + awsConfig, err := awsutil.Config() + if err != nil { + return errors.Wrap(err, "get aws config") + } + + storage := awss3.NewStorage(awsConfig) + + // Fetch current index. + + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + + b, err := storage.FetchRaw(ctx, repoEntry.URL+"/index.yaml") + if err != nil { + return errors.WithMessage(err, "fetch current repo index") + } + + idx, err := index.LoadBytes(b) + if err != nil { + return errors.WithMessage(err, "load index from downloaded file") + } + + // Update index. + + chartVersion, err := idx.Delete(name, version) + if err != nil { + return err + } + + idxReader, err := idx.Reader() + if err != nil { + return errors.Wrap(err, "get index reader") + } + + // Delete the file from S3 and replace index file. + + if len(chartVersion.URLs) < 1 { + return fmt.Errorf("chart version index record has no urls") + } + uri := chartVersion.URLs[0] + + ctx, cancel = context.WithTimeout(context.Background(), defaultTimeout*2) + defer cancel() + + if err := storage.Delete(ctx, uri); err != nil { + return errors.WithMessage(err, "delete chart file from s3") + } + if _, err := storage.Upload(ctx, repoEntry.URL+"/index.yaml", idxReader); err != nil { + return errors.WithMessage(err, "upload new index to s3") + } + + return nil +} diff --git a/cmd/helms3/init.go b/cmd/helms3/init.go index 2193126e..826794ba 100644 --- a/cmd/helms3/init.go +++ b/cmd/helms3/init.go @@ -21,10 +21,12 @@ func runInit(uri string) error { return errors.WithMessage(err, "get aws config") } + storage := awss3.NewStorage(awsConfig) + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) defer cancel() - if _, err := awss3.Upload(ctx, uri+"/index.yaml", r, awsConfig); err != nil { + if _, err := storage.Upload(ctx, uri+"/index.yaml", r); err != nil { return errors.WithMessage(err, "upload index to s3") } diff --git a/cmd/helms3/main.go b/cmd/helms3/main.go index 6e108d1f..774d0fca 100644 --- a/cmd/helms3/main.go +++ b/cmd/helms3/main.go @@ -14,8 +14,10 @@ var ( ) const ( - actionPush = "push" - actionInit = "init" + actionVersion = "version" + actionInit = "init" + actionPush = "push" + actionDelete = "delete" defaultTimeout = time.Second * 5 ) @@ -29,18 +31,32 @@ func main() { } cli := kingpin.New("helm s3", "") - cli.Version(version) + cli.Command(actionVersion, "Show plugin version.") + initCmd := cli.Command(actionInit, "Initialize empty repository on AWS S3.") initURI := initCmd.Arg("uri", "URI of repository, e.g. s3://awesome-bucket/charts"). Required(). String() - pushCmd := cli.Command(actionPush, "Push chart to repository.") + + pushCmd := cli.Command(actionPush, "Push chart to the repository.") pushChartPath := pushCmd.Arg("chartPath", "Path to a chart, e.g. ./epicservice-0.5.1.tgz"). Required(). String() - pushTargetRepository := pushCmd.Arg("repo", "Target repository to runPush"). + pushTargetRepository := pushCmd.Arg("repo", "Target repository to push to"). + Required(). + String() + + deleteCmd := cli.Command(actionDelete, "Delete chart from the repository.").Alias("del") + deleteChartName := deleteCmd.Arg("chartName", "Name of chart to delete"). + Required(). + String() + deleteChartVersion := deleteCmd.Flag("version", "Version of chart to delete"). Required(). String() + deleteTargetRepository := deleteCmd.Arg("repo", "Target repository to delete from"). + Required(). + String() + action := kingpin.MustParse(cli.Parse(os.Args[1:])) if action == "" { cli.Usage(os.Args[1:]) @@ -48,6 +64,9 @@ func main() { } switch action { + case actionVersion: + fmt.Print(version) + return case actionInit: if err := runInit(*initURI); err != nil { @@ -62,5 +81,10 @@ func main() { } return + case actionDelete: + if err := runDelete(*deleteChartName, *deleteChartVersion, *deleteTargetRepository); err != nil { + log.Fatal(err) + } + return } } diff --git a/cmd/helms3/proxy.go b/cmd/helms3/proxy.go index 0e647b80..bfbe1fda 100644 --- a/cmd/helms3/proxy.go +++ b/cmd/helms3/proxy.go @@ -16,10 +16,12 @@ func runProxy(uri string) error { return errors.WithMessage(err, "get aws config") } + storage := awss3.NewStorage(awsConfig) + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) defer cancel() - b, err := awss3.FetchRaw(ctx, uri, awsConfig) + b, err := storage.FetchRaw(ctx, uri) if err != nil { return errors.WithMessage(err, "fetch from s3") } diff --git a/cmd/helms3/push.go b/cmd/helms3/push.go index 08b39305..c150365a 100644 --- a/cmd/helms3/push.go +++ b/cmd/helms3/push.go @@ -8,13 +8,11 @@ import ( "github.com/pkg/errors" "k8s.io/helm/pkg/chartutil" - "k8s.io/helm/pkg/helm/environment" - "k8s.io/helm/pkg/helm/helmpath" "k8s.io/helm/pkg/provenance" - "k8s.io/helm/pkg/repo" "github.com/hypnoglow/helm-s3/pkg/awss3" "github.com/hypnoglow/helm-s3/pkg/awsutil" + "github.com/hypnoglow/helm-s3/pkg/helmutil" "github.com/hypnoglow/helm-s3/pkg/index" ) @@ -26,7 +24,17 @@ func runPush(chartPath string, repoName string) error { dir := filepath.Dir(fpath) fname := filepath.Base(fpath) - os.Chdir(dir) + + if err := os.Chdir(dir); err != nil { + return errors.Wrapf(err, "change dir to %s", dir) + } + + awsConfig, err := awsutil.Config() + if err != nil { + return errors.Wrap(err, "get aws config") + } + + storage := awss3.NewStorage(awsConfig) // Load chart and calculate required params like hash. @@ -35,7 +43,7 @@ func runPush(chartPath string, repoName string) error { return fmt.Errorf("file %s is not a helm chart archive", fname) } - repoURL, err := lookupRepoURL(repoName) + repoEntry, err := helmutil.LookupRepoEntry(repoName) if err != nil { return err } @@ -47,14 +55,10 @@ func runPush(chartPath string, repoName string) error { // Fetch current index. - awsConfig, err := awsutil.Config() - if err != nil { - return errors.Wrap(err, "get aws config") - } - ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) defer cancel() - b, err := awss3.FetchRaw(ctx, repoURL+"/index.yaml", awsConfig) + + b, err := storage.FetchRaw(ctx, repoEntry.URL+"/index.yaml") if err != nil { return errors.WithMessage(err, "fetch current repo index") } @@ -66,7 +70,7 @@ func runPush(chartPath string, repoName string) error { // Update index. - idx.Add(chart.GetMetadata(), fname, repoURL, hash) + idx.Add(chart.GetMetadata(), fname, repoEntry.URL, hash) idx.SortEntries() // Finally, upload both chart file and index. @@ -82,32 +86,13 @@ func runPush(chartPath string, repoName string) error { ctx, cancel = context.WithTimeout(context.Background(), defaultTimeout*2) defer cancel() - if _, err := awss3.Upload(ctx, repoURL+"/"+fname, fchart, awsConfig); err != nil { + + if _, err := storage.Upload(ctx, repoEntry.URL+"/"+fname, fchart); err != nil { return errors.WithMessage(err, "upload chart to s3") } - if _, err := awss3.Upload(ctx, repoURL+"/index.yaml", idxReader, awsConfig); err != nil { + if _, err := storage.Upload(ctx, repoEntry.URL+"/index.yaml", idxReader); err != nil { return errors.WithMessage(err, "upload index to s3") } return nil } - -func lookupRepoURL(name string) (string, error) { - h := helmpath.Home(environment.DefaultHelmHome) - if os.Getenv("HELM_HOME") != "" { - h = helmpath.Home(os.Getenv("HELM_HOME")) - } - - repoFile, err := repo.LoadRepositoriesFile(h.RepositoryFile()) - if err != nil { - return "", errors.Wrap(err, "load repo file") - } - - for _, r := range repoFile.Repositories { - if r.Name == name { - return r.URL, nil - } - } - - return "", errors.Errorf("repo with name %s not found, try `helm repo add %s `", name, name) -} diff --git a/pkg/awss3/awss3.go b/pkg/awss3/awss3.go deleted file mode 100644 index 371e52ff..00000000 --- a/pkg/awss3/awss3.go +++ /dev/null @@ -1,71 +0,0 @@ -package awss3 - -import ( - "context" - "io" - "net/url" - "strings" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" - "github.com/pkg/errors" -) - -// FetchRaw downloads the file from URI and returns it in the form of byte slice. -// URI must be in the form of s3://bucket-name/key[/file.ext]. -func FetchRaw(ctx context.Context, uri string, awsConfig *aws.Config) ([]byte, error) { - sess, err := session.NewSession(awsConfig) - if err != nil { - return nil, errors.Wrap(err, "failed to create new aws session") - } - - u, err := url.Parse(uri) - if err != nil { - return nil, errors.Wrapf(err, "failed to parse uri %s", uri) - } - - bucket, key := u.Host, strings.TrimPrefix(u.Path, "/") - - buf := &aws.WriteAtBuffer{} - _, err = s3manager.NewDownloader(sess).DownloadWithContext( - ctx, - buf, - &s3.GetObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - }) - if err != nil { - return nil, errors.Wrap(err, "failed to download object from s3") - } - - return buf.Bytes(), nil -} - -// Upload uploads the file read from r to S3 by path uri. URI must be in the form -// of s3://bucket-name/key[/file.ext]. -func Upload(ctx context.Context, uri string, r io.Reader, awsConfig *aws.Config) (string, error) { - sess, err := session.NewSession(awsConfig) - if err != nil { - return "", errors.Wrap(err, "failed to create new aws session") - } - - u, err := url.Parse(uri) - if err != nil { - return "", errors.Wrapf(err, "failed to parse uri %s", uri) - } - - bucket, key := u.Host, strings.TrimPrefix(u.Path, "/") - - result, err := s3manager.NewUploader(sess).UploadWithContext(ctx, &s3manager.UploadInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - Body: r, - }) - if err != nil { - return "", errors.Wrap(err, "failed to upload file to s3") - } - - return result.Location, nil -} diff --git a/pkg/awss3/storage.go b/pkg/awss3/storage.go new file mode 100644 index 00000000..e425007e --- /dev/null +++ b/pkg/awss3/storage.go @@ -0,0 +1,133 @@ +package awss3 + +import ( + "context" + "fmt" + "io" + "net/url" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/pkg/errors" +) + +// NewStorage returns a new Storage. +func NewStorage(awsConfig *aws.Config) *Storage { + return &Storage{ + config: awsConfig, + } +} + +// Storage provides an interface to work with AWS S3 objects by s3 protocol. +type Storage struct { + config *aws.Config + session *session.Session +} + +// FetchRaw downloads the object from URI and returns it in the form of byte slice. +// uri must be in the form of s3 protocol: s3://bucket-name/key[...]. +func (s *Storage) FetchRaw(ctx context.Context, uri string) ([]byte, error) { + if err := s.initSession(); err != nil { + return nil, err + } + + bucket, key, err := parseURI(uri) + if err != nil { + return nil, err + } + + buf := &aws.WriteAtBuffer{} + _, err = s3manager.NewDownloader(s.session).DownloadWithContext( + ctx, + buf, + &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + }) + if err != nil { + return nil, errors.Wrap(err, "fetch object from s3") + } + + return buf.Bytes(), nil +} + +// Upload uploads the object read from r to S3 by path uri. +// uri must be in the form of s3 protocol: s3://bucket-name/key[...]. +func (s *Storage) Upload(ctx context.Context, uri string, r io.Reader) (string, error) { + if err := s.initSession(); err != nil { + return "", err + } + + bucket, key, err := parseURI(uri) + if err != nil { + return "", err + } + + result, err := s3manager.NewUploader(s.session).UploadWithContext( + ctx, + &s3manager.UploadInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Body: r, + }) + if err != nil { + return "", errors.Wrap(err, "upload object to s3") + } + + return result.Location, nil +} + +// Delete deletes the object by uri. +// uri must be in the form of s3 protocol: s3://bucket-name/key[...]. +func (s *Storage) Delete(ctx context.Context, uri string) error { + if err := s.initSession(); err != nil { + return err + } + + bucket, key, err := parseURI(uri) + if err != nil { + return err + } + + _, err = s3.New(s.session).DeleteObjectWithContext( + ctx, + &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + }, + ) + if err != nil { + return errors.Wrap(err, "delete object from s3") + } + + return nil +} + +func (s *Storage) initSession() (err error) { + if s.session != nil { + return nil + } + + s.session, err = session.NewSession(s.config) + return errors.Wrap(err, "init aws session") +} + +// parseURI returns bucket and key from URIs like: +// - s3://bucket-name/dir +// - s3://bucket-name/dir/file.ext +func parseURI(uri string) (bucket, key string, err error) { + if !strings.HasPrefix(uri, "s3://") { + return "", "", fmt.Errorf("uri %s protocol is not s3", uri) + } + + u, err := url.Parse(uri) + if err != nil { + return "", "", errors.Wrapf(err, "parse uri %s", uri) + } + + bucket, key = u.Host, strings.TrimPrefix(u.Path, "/") + return bucket, key, nil +} diff --git a/pkg/helmutil/repo_entry.go b/pkg/helmutil/repo_entry.go new file mode 100644 index 00000000..389d69d0 --- /dev/null +++ b/pkg/helmutil/repo_entry.go @@ -0,0 +1,35 @@ +package helmutil + +import ( + "os" + + "github.com/pkg/errors" + "k8s.io/helm/pkg/helm/environment" + "k8s.io/helm/pkg/helm/helmpath" + "k8s.io/helm/pkg/repo" +) + +const ( + envHelmHome = "HELM_HOME" +) + +// LookupRepoEntry returns an entry from helm's repositories.yaml file by name. +func LookupRepoEntry(name string) (*repo.Entry, error) { + h := helmpath.Home(environment.DefaultHelmHome) + if os.Getenv(envHelmHome) != "" { + h = helmpath.Home(os.Getenv(envHelmHome)) + } + + repoFile, err := repo.LoadRepositoriesFile(h.RepositoryFile()) + if err != nil { + return nil, errors.Wrap(err, "load repo file") + } + + for _, r := range repoFile.Repositories { + if r.Name == name { + return r, nil + } + } + + return nil, errors.Errorf("repo with name %s not found, try `helm repo add %s `", name, name) +} diff --git a/pkg/index/index.go b/pkg/index/index.go index 531d45c8..cfb18f9b 100644 --- a/pkg/index/index.go +++ b/pkg/index/index.go @@ -2,6 +2,7 @@ package index import ( "bytes" + "fmt" "io" "github.com/ghodss/yaml" @@ -15,7 +16,7 @@ type Index struct { } // Reader returns io.Reader for index. -func (i Index) Reader() (io.Reader, error) { +func (i *Index) Reader() (io.Reader, error) { b, err := yaml.Marshal(i) if err != nil { return nil, err @@ -24,19 +25,40 @@ func (i Index) Reader() (io.Reader, error) { return bytes.NewReader(b), nil } +// Delete removes chart version from index and returns deleted item. +func (idx *Index) Delete(name, version string) (*repo.ChartVersion, error) { + for chartName, chartVersions := range idx.Entries { + if chartName != name { + continue + } + + for i, chartVersion := range chartVersions { + if chartVersion.Version == version { + idx.Entries[chartName] = append( + idx.Entries[chartName][:i], + idx.Entries[chartName][i+1:]..., + ) + return chartVersion, nil + } + } + } + + return nil, fmt.Errorf("chart %s version %s not found in index", name, version) +} + // New returns a new index. -func New() Index { - return Index{ +func New() *Index { + return &Index{ repo.NewIndexFile(), } } // LoadBytes returns an index read from bytes. -func LoadBytes(b []byte) (Index, error) { +func LoadBytes(b []byte) (*Index, error) { i := &repo.IndexFile{} if err := yaml.Unmarshal(b, i); err != nil { - return Index{}, err + return nil, err } i.SortEntries() - return Index{i}, nil + return &Index{i}, nil } diff --git a/plugin.yaml b/plugin.yaml index d45f48b2..303892db 100644 --- a/plugin.yaml +++ b/plugin.yaml @@ -1,5 +1,5 @@ name: "s3" -version: "0.3.0" +version: "0.4.0" usage: "The plugin allows to use s3 protocol to upload, fetch charts and to work with repositories." description: |- Provides AWS S3 protocol support. diff --git a/sh/build.sh b/sh/build.sh index bad5548b..505679d8 100755 --- a/sh/build.sh +++ b/sh/build.sh @@ -11,5 +11,7 @@ if [ ! -e "${GOPATH}/src/${pkg}" ]; then ln -sfn "${projectRoot}" "${GOPATH}/src/${pkg}" fi +version="$(cat plugin.yaml | grep "version" | cut -d '"' -f 2)" + cd "${GOPATH}/src/${pkg}" -go build -o bin/helms3 ./cmd/helms3 \ No newline at end of file +go build -o bin/helms3 -ldflags "-X main.version=${version}" ./cmd/helms3 \ No newline at end of file diff --git a/sh/integration-tests.sh b/sh/integration-tests.sh index f8498924..f14bd32c 100755 --- a/sh/integration-tests.sh +++ b/sh/integration-tests.sh @@ -50,8 +50,30 @@ if [ $? -ne 0 ]; then exit 1 fi +# +# Delete +# + +helm s3 delete postgresql --version 0.8.3 test-repo +if [ $? -ne 0 ]; then + echo "Failed to delete chart from repo" + exit 1 +fi + +if mc ls -q helm-s3-minio/test-bucket/charts/postgresql-0.8.3.tgz 2>/dev/null ; then + echo "Chart was not actually deleted" + exit 1 +fi + +helm repo update + +if helm search test-repo/postgres | grep -q 0.8.3 ; then + echo "Failed to delete chart from index" + exit 1 +fi + +# Tear down rm postgresql-0.8.3.tgz helm repo remove test-repo - set +x