From ca24d85482b3e0f04fe7088705d4f54a5a13e196 Mon Sep 17 00:00:00 2001 From: hijiki51 Date: Sun, 18 Sep 2022 01:43:17 +0900 Subject: [PATCH] add s3 impl --- go.mod | 18 ++++++++++ go.sum | 40 ++++++++++++++++++++++ storage/s3.go | 94 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 152 insertions(+) create mode 100644 storage/s3.go diff --git a/go.mod b/go.mod index 14a18633..0715e0cc 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,10 @@ go 1.19 require ( entgo.io/ent v0.11.3-0.20220824145844-79fe0ebcd627 + github.com/aws/aws-sdk-go-v2 v1.16.15 + github.com/aws/aws-sdk-go-v2/config v1.17.6 + github.com/aws/aws-sdk-go-v2/credentials v1.12.19 + github.com/aws/aws-sdk-go-v2/service/s3 v1.27.10 github.com/go-sql-driver/mysql v1.6.0 github.com/golang/mock v1.6.0 github.com/google/uuid v1.3.0 @@ -19,6 +23,20 @@ require ( ariga.io/atlas v0.7.0 // indirect github.com/agext/levenshtein v1.2.3 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.22 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.16 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.16 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.11.22 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.16.18 // indirect + github.com/aws/smithy-go v1.13.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-openapi/inflect v0.19.0 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect diff --git a/go.sum b/go.sum index 961abc2f..7e39b0d6 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,42 @@ github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7l github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= +github.com/aws/aws-sdk-go-v2 v1.16.15 h1:2sInOWGE4HV54R90Pj8QgqBBw3Qf1I0husqbqjPZzys= +github.com/aws/aws-sdk-go-v2 v1.16.15/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8 h1:tcFliCWne+zOuUfKNRn8JdFBuWPDuISDH08wD2ULkhk= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8/go.mod h1:JTnlBSot91steJeti4ryyu/tLd4Sk84O5W22L7O2EQU= +github.com/aws/aws-sdk-go-v2/config v1.17.6 h1:0xHMch3eQ2C8CByMEi0iJOLF+pTLoAQeHVfhFxN7eyk= +github.com/aws/aws-sdk-go-v2/config v1.17.6/go.mod h1:CrxsoI/AcKUoWyL9Zo0YaDxRlBfSnDZKBYKDdkNYDQ0= +github.com/aws/aws-sdk-go-v2/credentials v1.12.19 h1:fYtSz4Fd0lUavtj4FAtvol9G2k0lh1TK4LfeP1hdnLw= +github.com/aws/aws-sdk-go-v2/credentials v1.12.19/go.mod h1:fRQMbLwSHPr0XRzuez6x7BX5+0nNQs5BzG+oR4bQKyY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.16 h1:LX38v4cqSqrBETHUBnc8B+N6p5YA41GaPQ3jwICjetI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.16/go.mod h1:lnJ8tKos2s7JeBdLVFknwVSlQZAKzkgrFNQmUaTWwRQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.22 h1:pE27/u2A7JlwICjOvONQDob8PToShRTkuiUE74ymVWg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.22/go.mod h1:/vNv5Al0bpiF8YdX2Ov6Xy05VTiXsql94yUqJMYaj0w= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.16 h1:L5LKGHHXOl4t7+5QZMTl38GIzSAq07XUTRtEquiHGMA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.16/go.mod h1:62dsXI0BqTIGomDl8Hpm33dv0OntGaVblri3ZRParVQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.23 h1:Sy266MXyLZZbObFhStGF9dyJm5nFyA8LINTgNm4Q6Ds= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.23/go.mod h1:XtEkQMmxls+Tb5dZLmpa1QAk0OzSIFDAXanC9Jkf81E= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.13 h1:ZrdsZJfzniYAF3Au4ngj7vWxJS1ZHGYCE0YlATcWYUg= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.13/go.mod h1:QyDZ5fgUmZFZztFpVcR7w4HV8vwO0ze1OM9rPy6jkEI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9 h1:Lh1AShsuIJTwMkoxVCAYPJgNG5H+eN6SmoUn8nOZ5wE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9/go.mod h1:a9j48l6yL5XINLHLcOKInjdvknN+vWqPBxqeIDw7ktw= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.17 h1:rf0/i+3BMiQZjcQzhjTz3sAXouSl1dcnCrd8mroC4CE= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.17/go.mod h1:Uo/4yJjc7RDB7R5q9JA7aQqFXasu/lAJke8mulo2dA8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.16 h1:9jysIwpUt7KGdsKOl+zA+0pG+7MpSsi0KQUcbE48n38= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.16/go.mod h1:faBcf/4ZB4FRc17geaXWOxgzktotyJgBcUBZoHqvdfM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.16 h1:jnGshlNJoVF3x8e5EbsFakNKeNUc0Pf3EQvU076bkKU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.16/go.mod h1:tMN4hfJxozc/lHWA8Ug/eXEg4ZSm5c4sPB9WE/mdDuw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.27.10 h1:fR5Z06dU41kkSdmSOtHiROealTOu5aNkwidOQ+lT0KQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.27.10/go.mod h1:B+5EUmLgCYrXHxgQ3nTUu3RUbxnrN1JMa41LSXm7lXw= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.22 h1:LrEyMbp0gMiXVaXpJ67jJkkqKCxivZvOd6wgXem0bWA= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.22/go.mod h1:B2nDzX7lppT8j4EV2/WhT20SnRDp/LdNyqxyGYY46Ow= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.4 h1:d7Wh4xMQVVYfrJ1KHFGQ6jY/O51LjnTCWJgh85RT+TQ= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.4/go.mod h1:mOofcMJCDSJwmtZykUE/i6tWGNwMnkextriwzY1zcbc= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.18 h1:TqEvnK8OceCKNQaDK9d5Ir2bOtC0S0dRQCwSbkV1rz0= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.18/go.mod h1:AE4zMc8qCw1JnDvy0ZrDVb/OXRuuweG3BcT2Nv7Qh3E= +github.com/aws/smithy-go v1.13.3 h1:l7LYxGuzK6/K+NzJ2mC+VvLUbae0sL3bXU//04MkmnA= +github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -20,6 +56,7 @@ github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keL github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -32,6 +69,8 @@ github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7Fsg github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/hashicorp/hcl/v2 v2.14.0 h1:jX6+Q38Ly9zaAJlAjnFVyeNSNCKKW8D0wvyg7vij5Wc= github.com/hashicorp/hcl/v2 v2.14.0/go.mod h1:e4z5nxYlWNPdDSNYX+ph14EvWYMFm3eP0zIUqPc2jr0= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= @@ -116,6 +155,7 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/storage/s3.go b/storage/s3.go new file mode 100644 index 00000000..be562cc1 --- /dev/null +++ b/storage/s3.go @@ -0,0 +1,94 @@ +package storage + +import ( + "context" + "errors" + "io" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" +) + +// S3 S3オブジェクトストレージ +type S3 struct { + bucket string + client *s3.Client +} + +// SetS3Storage S3オブジェクトストレージをカレントストレージに設定します +func SetS3Storage(bucket string, region string, endpoint string, apiKey string, apiSecret string) (*S3, error) { + cfg, err := config.LoadDefaultConfig(context.Background(), + config.WithRegion(region), + config.WithEndpointResolverWithOptions( + aws.EndpointResolverWithOptionsFunc( + func(service string, region string, options ...interface{}) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: endpoint, + SigningRegion: region, + }, nil + }, + ), + ), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(apiKey, apiSecret, "")), + ) + + if err != nil { + return nil, err + } + + client := s3.NewFromConfig(cfg) + + return &S3{ + bucket: bucket, + client: client, + }, nil +} + +func (s *S3) Save(filename string, src io.Reader) error { + + input := s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(filename), + Body: src, + } + + _, err := s.client.PutObject(context.Background(), &input) + return err +} + +func (s *S3) Open(filename string) (io.ReadCloser, error) { + input := s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(filename), + } + + res, err := s.client.GetObject(context.Background(), &input) + if err != nil { + var nf *types.NoSuchKey + if errors.As(err, &nf) { + return nil, ErrFileNotFound + } + return nil, err + } + return res.Body, nil +} + +func (s *S3) Delete(filename string) error { + input := s3.DeleteObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(filename), + } + + _, err := s.client.DeleteObject(context.Background(), &input) + if err != nil { + var nf *types.NoSuchKey + if errors.As(err, &nf) { + return ErrFileNotFound + } + return err + } + return nil +}