From 12997026c736dca5b7408964677973a00f305e6a Mon Sep 17 00:00:00 2001 From: cp-20 Date: Wed, 18 Dec 2024 18:17:31 +0900 Subject: [PATCH] =?UTF-8?q?swift=20=E2=86=92=20s3=20=E3=81=AB=E3=81=97?= =?UTF-8?q?=E3=81=9F=20(=E3=83=AD=E3=83=BC=E3=82=AB=E3=83=AB=E3=81=AE?= =?UTF-8?q?=E3=83=87=E3=83=95=E3=82=A9=E3=83=AB=E3=83=88=E3=82=92MinIO?= =?UTF-8?q?=E3=81=AB)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.yml | 31 ++++++++++++++++- go.mod | 19 +++++++++++ go.sum | 40 ++++++++++++++++++++++ main.go | 17 +++++----- storage/s3.go | 83 ++++++++++++++++++++++++++++++++++++++++++++++ storage/swift.go | 66 ------------------------------------ 6 files changed, 180 insertions(+), 76 deletions(-) create mode 100644 storage/s3.go delete mode 100644 storage/swift.go diff --git a/docker-compose.yml b/docker-compose.yml index 6c98a8f..c16b91c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,28 @@ services: interval: 10s timeout: 5s retries: 3 + s3: + image: minio/minio + restart: unless-stopped + ports: + - '9000:9000' + - '9001:9001' + environment: + MINIO_ROOT_USER: admin + MINIO_ROOT_PASSWORD: password + MINIO_ACCESS_KEY: minio_access_key + MINIO_SECRET_KEY: minio_secret_key + volumes: + - booq-s3:/data + s3-init: + image: minio/mc + depends_on: + - s3 + entrypoint: > + /bin/sh -c " + mc alias set s3-local http://s3:9000 admin password; + mc mb s3-local/booq-v3; + " booq-server: build: . environment: @@ -28,12 +50,18 @@ services: MYSQL_USER: root MYSQL_PASSWORD: password MYSQL_DATABASE: booq-v3 - UPLOAD_DIR: /app/data + S3_BUCKET: booq-v3 + S3_REGION: ap-northeast-1 + S3_ENDPOINT: http://s3:9000 + S3_ACCESS_KEY: minio_access_key + S3_SECRET_KEY: minio_secret_key ports: - '8080:3001' depends_on: db: condition: service_healthy + s3-init: + condition: service_completed_successfully develop: watch: - path: ./Dockerfile @@ -56,3 +84,4 @@ services: volumes: booq-db: + booq-s3: diff --git a/go.mod b/go.mod index 438b9fd..0908881 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,24 @@ require ( github.com/ClickHouse/clickhouse-go/v2 v2.18.0 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/aws/aws-sdk-go-v2 v1.32.6 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect + github.com/aws/aws-sdk-go-v2/config v1.28.6 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.47 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 // indirect + github.com/aws/smithy-go v1.22.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-faster/city v1.0.1 // indirect github.com/go-faster/errors v0.6.1 // indirect @@ -26,6 +44,7 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/compress v1.16.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect diff --git a/go.sum b/go.sum index 886d45d..bc8cc10 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,42 @@ github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer5 github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4= +github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc= +github.com/aws/aws-sdk-go-v2/config v1.28.6 h1:D89IKtGrs/I3QXOLNTH93NJYtDhm8SYa9Q5CsPShmyo= +github.com/aws/aws-sdk-go-v2/config v1.28.6/go.mod h1:GDzxJ5wyyFSCoLkS+UhGB0dArhb9mI+Co4dHtoTxbko= +github.com/aws/aws-sdk-go-v2/credentials v1.17.47 h1:48bA+3/fCdi2yAwVt+3COvmatZ6jUDNkDTIsqDiMUdw= +github.com/aws/aws-sdk-go-v2/credentials v1.17.47/go.mod h1:+KdckOejLW3Ks3b0E3b5rHsr2f9yuORBum0WPnE5o5w= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 h1:AmoU1pziydclFT/xRV+xXE/Vb8fttJCLRPv8oAkprc0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21/go.mod h1:AjUdLYe4Tgs6kpH4Bv7uMZo7pottoyHMn4eTcIcneaY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 h1:s/fF4+yDQDoElYhfIVvSNyeCydfbuTKzhxSXDXCPasU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25/go.mod h1:IgPfDv5jqFIzQSNbUEMoitNooSMXjRSDkhXv8jiROvU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 h1:ZntTCl5EsYnhN/IygQEUugpdwbhdkom9uHcbCftiGgA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25/go.mod h1:DBdPrgeocww+CSl1C8cEV8PN1mHMBhuCDLpXezyvWkE= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25 h1:r67ps7oHCYnflpgDy2LZU0MAQtQbYIOqNNnqGO6xQkE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25/go.mod h1:GrGY+Q4fIokYLtjCVB/aFfCVL6hhGUFl8inD18fDalE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 h1:HCpPsWqmYQieU7SS6E9HXfdAMSud0pteVXieJmcpIRI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6/go.mod h1:ngUiVRCco++u+soRRVBIvBZxSMMvOVMXA4PJ36JLfSw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 h1:50+XsN70RS7dwJ2CkVNXzj7U2L1HKP8nqTd3XWEXBN4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6/go.mod h1:WqgLmwY7so32kG01zD8CPTJWVWM+TzJoOVHwTg4aPug= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 h1:BbGDtTi0T1DYlmjBiCr/le3wzhA37O8QTC5/Ab8+EXk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6/go.mod h1:hLMJt7Q8ePgViKupeymbqI0la+t9/iYFBjxQCFwuAwI= +github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 h1:nyuzXooUNJexRT0Oy0UQY6AhOzxPxhtt4DcBIHyCnmw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0/go.mod h1:sT/iQz8JK3u/5gZkT+Hmr7GzVZehUMkRZpOaAwYXeGY= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 h1:rLnYAfXQ3YAccocshIH5mzNNwZBkBo+bP6EhIxak6Hw= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.7/go.mod h1:ZHtuQJ6t9A/+YDuxOLnbryAmITtr8UysSny3qcyvJTc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 h1:JnhTZR3PiYDNKlXy50/pNeix9aGMo6lLpXwJ1mw8MD4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6/go.mod h1:URronUEGfXZN1VpdktPSD1EkAL9mfrV+2F4sjH38qOY= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 h1:s4074ZO1Hk8qv65GqNXqDjmkf4HSQqJukaLuuW0TpDA= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.2/go.mod h1:mVggCnIWoM09jP71Wh+ea7+5gAp53q+49wDFs1SW5z8= +github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= +github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -58,6 +94,9 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -176,6 +215,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index 18b83d3..081a90b 100644 --- a/main.go +++ b/main.go @@ -40,15 +40,14 @@ func main() { } func setStorage() { - if os.Getenv("OS_CONTAINER") != "" { - // Swiftオブジェクトストレージ - err := storage.SetSwiftStorage( - os.Getenv("OS_CONTAINER"), - os.Getenv("OS_USERNAME"), - os.Getenv("OS_PASSWORD"), - os.Getenv("OS_TENANT_NAME"), - os.Getenv("OS_TENANT_ID"), - os.Getenv("OS_AUTH_URL"), + if os.Getenv("S3_BUCKET") != "" { + // S3 + err := storage.SetS3Storage( + os.Getenv("S3_BUCKET"), + os.Getenv("S3_REGION"), + os.Getenv("S3_ENDPOINT"), + os.Getenv("S3_ACCESS_KEY"), + os.Getenv("S3_SECRET_KEY"), ) if err != nil { log.Fatal(err) diff --git a/storage/s3.go b/storage/s3.go new file mode 100644 index 0000000..8e589a5 --- /dev/null +++ b/storage/s3.go @@ -0,0 +1,83 @@ +package storage + +import ( + "context" + "fmt" + "io" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +type S3Storage struct { + bucket string + client *s3.Client +} + +// S3 (互換) オブジェクトストレージをカレントストレージに設定する +func SetS3Storage(bucket, region, endpoint, apiKey, apiSecret string) error { + cfg, err := config.LoadDefaultConfig(context.Background(), + config.WithRegion(region), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(apiKey, apiSecret, "")), + ) + if err != nil { + return fmt.Errorf("failed to load configuration: %w", err) + } + + client := s3.NewFromConfig(cfg, func(o *s3.Options) { + o.UsePathStyle = true + o.BaseEndpoint = &endpoint + }) + + current = &S3Storage{ + bucket: bucket, + client: client, + } + + return nil +} + +func (s *S3Storage) Save(filename string, src io.Reader) error { + input := &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(filename), + Body: src, + } + + _, err := s.client.PutObject(context.Background(), input) + if err != nil { + return fmt.Errorf("failed to put object: %w", err) + } + + return nil +} + +func (s *S3Storage) Open(filename string) (io.ReadCloser, error) { + input := &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(filename), + } + + output, err := s.client.GetObject(context.Background(), input) + if err != nil { + return nil, fmt.Errorf("failed to get object: %w", err) + } + + return output.Body, nil +} + +func (s *S3Storage) Delete(filename string) error { + input := &s3.DeleteObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(filename), + } + + _, err := s.client.DeleteObject(context.Background(), input) + if err != nil { + return fmt.Errorf("failed to delete object: %w", err) + } + + return nil +} diff --git a/storage/swift.go b/storage/swift.go deleted file mode 100644 index d1469e5..0000000 --- a/storage/swift.go +++ /dev/null @@ -1,66 +0,0 @@ -package storage - -import ( - "github.com/ncw/swift" - "io" -) - -// Swift Swiftオブジェクトストレージ -type Swift struct { - container string - conn *swift.Connection -} - -// SetSwiftStorage Swiftオブジェクトストレージをカレントストレージに設定します -func SetSwiftStorage(container, userName, apiKey, tenant, tenantID, authURL string) error { - conn := &swift.Connection{ - AuthUrl: authURL, - UserName: userName, - ApiKey: apiKey, - Tenant: tenant, - TenantId: tenantID, - } - - // 認証 - if err := conn.Authenticate(); err != nil { - return err - } - - // コンテナの存在を確認 - if _, _, err := conn.Container(container); err != nil { - return err - } - - current = &Swift{ - container: container, - conn: conn, - } - return nil -} - -func (s *Swift) Save(filename string, src io.Reader) error { - _, err := s.conn.ObjectPut(s.container, filename, src, true, "", "", swift.Headers{}) - return err -} - -func (s *Swift) Open(filename string) (io.ReadCloser, error) { - file, _, err := s.conn.ObjectOpen(s.container, filename, true, nil) - if err != nil { - if err == swift.ObjectNotFound { - return nil, ErrFileNotFound - } - return nil, err - } - return file, nil -} - -func (s *Swift) Delete(filename string) error { - err := s.conn.ObjectDelete(s.container, filename) - if err != nil { - if err == swift.ObjectNotFound { - return ErrFileNotFound - } - return err - } - return nil -}