From 9cc2c62a5b5096f0d6a17d48be1a77cfe930117e Mon Sep 17 00:00:00 2001 From: warjiang <1096409085@qq.com> Date: Wed, 25 Oct 2023 11:32:30 +0800 Subject: [PATCH 1/3] feat: update storage lib and export cmd params Signed-off-by: warjiang <1096409085@qq.com> --- cmd/chartmuseum/main.go | 6 +++++- go.mod | 2 ++ go.sum | 4 ++-- pkg/config/vars.go | 9 +++++++++ 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/cmd/chartmuseum/main.go b/cmd/chartmuseum/main.go index 50a07d2d..86089fa6 100644 --- a/cmd/chartmuseum/main.go +++ b/cmd/chartmuseum/main.go @@ -176,12 +176,16 @@ func amazonBackendFromConfig(conf *config.Config) storage.Backend { conf.Set("storage.amazon.region", "us-east-1") } crashIfConfigMissingVars(conf, []string{"storage.amazon.bucket", "storage.amazon.region"}) - return storage.NewAmazonS3Backend( + forcePathStyle := conf.GetBool("storage.amazon.forcepathstyle") + return storage.NewAmazonS3BackendWithOptions( conf.GetString("storage.amazon.bucket"), conf.GetString("storage.amazon.prefix"), conf.GetString("storage.amazon.region"), conf.GetString("storage.amazon.endpoint"), conf.GetString("storage.amazon.sse"), + &storage.AmazonS3Options{ + S3ForcePathStyle: &forcePathStyle, + }, ) } diff --git a/go.mod b/go.mod index 1cc7317b..275179b0 100644 --- a/go.mod +++ b/go.mod @@ -181,3 +181,5 @@ require ( sigs.k8s.io/kustomize/kyaml v0.14.1 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) + +replace github.com/chartmuseum/storage => github.com/warjiang/storage v0.1.0 diff --git a/go.sum b/go.sum index 83529236..dd727b18 100644 --- a/go.sum +++ b/go.sum @@ -111,8 +111,6 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chartmuseum/auth v0.5.0 h1:ENNmoxvjxcR/JR0HrghAEtGQe7hToMNj16+UoS5CK9Y= github.com/chartmuseum/auth v0.5.0/go.mod h1:BvoSXHyvbsq+/bbhNgVTDQsModM+HERBTNY5o9Vyrig= -github.com/chartmuseum/storage v0.14.0 h1:R/Mp4fRaY5HCtLIEJomrsT415kEi6QumertR74cbHZU= -github.com/chartmuseum/storage v0.14.0/go.mod h1:LB+/k4kZkOu2AW3myaOw/AQt0JtjcNLwL1ktKxoWVug= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= @@ -531,6 +529,8 @@ github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4d github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= +github.com/warjiang/storage v0.1.0 h1:l67S0N6NjB3/VXsjpok/CsxZqVKLwCuCP9G2OVJAQrU= +github.com/warjiang/storage v0.1.0/go.mod h1:UoHcQmlvu0Ef3843TJuaf3EauGqCEuDCo1y60unbTMY= github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/pkg/config/vars.go b/pkg/config/vars.go index f2cf2176..e3cfd0c1 100644 --- a/pkg/config/vars.go +++ b/pkg/config/vars.go @@ -355,6 +355,15 @@ var configVars = map[string]configVar{ EnvVar: "STORAGE_AMAZON_SSE", }, }, + "storage.amazon.forcepathstyle": { + Type: boolType, + Default: true, + CLIFlag: cli.StringFlag{ + Name: "storage-amazon-force-path-style", + Usage: "whether to force path style for amazon storage backend", + EnvVar: "STORAGE_AMAZON_FORCE_PATH_STYLE", + }, + }, "storage.google.bucket": { Type: stringType, Default: "", From 8f984e41eb2a2c08032e398f8d4d69df1e8668da Mon Sep 17 00:00:00 2001 From: warjiang <1096409085@qq.com> Date: Fri, 27 Oct 2023 23:20:27 +0800 Subject: [PATCH 2/3] feat: upgrade chartmuseum/storage to v0.14.1 Signed-off-by: warjiang <1096409085@qq.com> --- go.mod | 4 +--- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 275179b0..a774c3b7 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( github.com/alicebob/miniredis v2.5.0+incompatible github.com/chartmuseum/auth v0.5.0 - github.com/chartmuseum/storage v0.14.0 + github.com/chartmuseum/storage v0.14.1 github.com/gin-contrib/size v0.0.0-20230212012657-e14a14094dc4 github.com/gin-gonic/gin v1.9.1 github.com/go-redis/redis v6.15.9+incompatible @@ -181,5 +181,3 @@ require ( sigs.k8s.io/kustomize/kyaml v0.14.1 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) - -replace github.com/chartmuseum/storage => github.com/warjiang/storage v0.1.0 diff --git a/go.sum b/go.sum index dd727b18..677d2391 100644 --- a/go.sum +++ b/go.sum @@ -111,6 +111,8 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chartmuseum/auth v0.5.0 h1:ENNmoxvjxcR/JR0HrghAEtGQe7hToMNj16+UoS5CK9Y= github.com/chartmuseum/auth v0.5.0/go.mod h1:BvoSXHyvbsq+/bbhNgVTDQsModM+HERBTNY5o9Vyrig= +github.com/chartmuseum/storage v0.14.1 h1:Az+YUopt+GjEg4r4kdq59ZMxuwuytaDbRIeWE6tghbU= +github.com/chartmuseum/storage v0.14.1/go.mod h1:LB+/k4kZkOu2AW3myaOw/AQt0JtjcNLwL1ktKxoWVug= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= @@ -529,8 +531,6 @@ github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4d github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= -github.com/warjiang/storage v0.1.0 h1:l67S0N6NjB3/VXsjpok/CsxZqVKLwCuCP9G2OVJAQrU= -github.com/warjiang/storage v0.1.0/go.mod h1:UoHcQmlvu0Ef3843TJuaf3EauGqCEuDCo1y60unbTMY= github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= From 5d5bdf52f6ce49af8cc36b2160a754aef74148e7 Mon Sep 17 00:00:00 2001 From: warjiang <1096409085@qq.com> Date: Sat, 28 Oct 2023 00:01:27 +0800 Subject: [PATCH 3/3] doc: update usage of force-path-style Signed-off-by: warjiang <1096409085@qq.com> --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 0b5c55ec..d3cd8e3b 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,20 @@ You need at least the following permissions inside your IAM Policy In order to work with AWS service accounts you may need to set `AWS_SDK_LOAD_CONFIG=1` in your environment. For more context, please see [here](https://github.com/helm/chartmuseum/issues/280#issuecomment-592292527). +If you are using S3-Compatible storage, provider of S3 storage has [disabled path-style and force virtual hosted-style](https://aws.amazon.com/cn/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/), you can use specify `storage-amazon-force-path-style` options as following example: +``` +export AWS_ACCESS_KEY_ID="" +export AWS_SECRET_ACCESS_KEY="" +chartmuseum --debug --port=8080 \ + --storage="amazon" \ + --storage-amazon-bucket="my-s3-bucket" \ + --storage-amazon-prefix="" \ + --storage-amazon-region="us-east-1" \ + --storage-amazon-endpoint="my-s3-compatible-service-endpoint" + --storage-amazon-force-path-style=false +``` + + For DigitalOcean, set the credentials using environment variable and pass the `endpoint`. Note below, that the region `us-east-1` needs to be set, since that is how the DigitalOcean cli implementation functions. The actual region of your spaces location is defined by the endpoint. Below we are using Frankfurt as an example. ```bash