diff --git a/go.mod b/go.mod index 150f0309..1c95412a 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,12 @@ go 1.21.6 require ( github.com/aws/aws-sdk-go v1.54.20 + github.com/aws/aws-sdk-go-v2 v1.32.2 + github.com/aws/aws-sdk-go-v2/config v1.27.43 + github.com/aws/aws-sdk-go-v2/credentials v1.17.41 + github.com/aws/aws-sdk-go-v2/service/s3 v1.65.2 + github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 + github.com/aws/smithy-go v1.22.0 github.com/ekristen/libnuke v0.19.1 github.com/fatih/color v1.17.0 github.com/golang/mock v1.6.0 @@ -19,6 +25,18 @@ require ( ) require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 // indirect github.com/benbjohnson/clock v1.3.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -32,10 +50,8 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/stevenle/topsort v0.2.0 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect - golang.org/x/mod v0.17.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.20.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index bfcc3707..6187ba5c 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,41 @@ github.com/aws/aws-sdk-go v1.54.20 h1:FZ2UcXya7bUkvkpf7TaPmiL7EubK0go1nlXGLRwEsoo= github.com/aws/aws-sdk-go v1.54.20/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcTduI= +github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA= +github.com/aws/aws-sdk-go-v2/config v1.27.43 h1:p33fDDihFC390dhhuv8nOmX419wjOSDQRb+USt20RrU= +github.com/aws/aws-sdk-go-v2/config v1.27.43/go.mod h1:pYhbtvg1siOOg8h5an77rXle9tVG8T+BWLWAo7cOukc= +github.com/aws/aws-sdk-go-v2/credentials v1.17.41 h1:7gXo+Axmp+R4Z+AK8YFQO0ZV3L0gizGINCOWxSLY9W8= +github.com/aws/aws-sdk-go-v2/credentials v1.17.41/go.mod h1:u4Eb8d3394YLubphT4jLEwN1rLNq2wFOlT6OuxFwPzU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 h1:TMH3f/SCAWdNtXXVPPu5D6wrr4G5hI1rAxbcocKfC7Q= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17/go.mod h1:1ZRXLdTpzdJb9fwTMXiLipENRxkGMTn1sfKexGllQCw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 h1:UAsR3xA31QGf79WzpG/ixT9FZvQlh5HY1NRqSHBNOCk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21/go.mod h1:JNr43NFf5L9YaG3eKTm7HQzls9J+A9YYcGI5Quh1r2Y= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYEQfC5fAQmlo80CeL5psbno6r0s= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 h1:7edmS3VOBDhK00b/MwGtGglCm7hhwNYnjJs/PgFdMQE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21/go.mod h1:Q9o5h4HoIWG8XfzxqiuK/CGUbepCJ8uTlaE3bAbxytQ= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 h1:4FMHqLfk0efmTqhXVRL5xYRqlEBNBiRI7N6w4jsEdd4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2/go.mod h1:LWoqeWlK9OZeJxsROW2RqrSPvQHKTpp69r/iDjwsSaw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 h1:s7NA1SOw8q/5c0wr8477yOPp0z+uBaXBnLE0XYb0POA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2/go.mod h1:fnjjWyAW/Pj5HYOxl9LJqWtEwS7W2qgcRLWP+uWbss0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 h1:t7iUP9+4wdc5lt3E41huP+GvQZJD38WLsgVp4iOtAjg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2/go.mod h1:/niFCtmuQNxqx9v8WAPq5qh7EH25U4BF6tjoyq9bObM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.65.2 h1:yi8m+jepdp6foK14xXLGkYBenxnlcfJ45ka4Pg7fDSQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.65.2/go.mod h1:cB6oAuus7YXRZhWCc1wIwPywwZ1XwweNp2TVAEGYeB8= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 h1:bSYXVyUzoTHoKalBmwaZxs97HU9DWWI3ehHSAMa7xOk= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.2/go.mod h1:skMqY7JElusiOUjMJMOv1jJsP7YUg7DrhgqZZWuzu1U= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 h1:AhmO1fHINP9vFYUE0LHzCWg/LfUWUF+zFPEcY9QXb7o= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2/go.mod h1:o8aQygT2+MVP0NaV6kbdE1YnnIM8RRVQzoeUH45GOdI= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 h1:CiS7i0+FUe+/YY1GvIBLLrR/XNGZ4CtM1Ll0XavNuVo= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.2/go.mod h1:HtaiBI8CjYoNVde8arShXb94UbQQi9L4EMr6D+xGBwo= +github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= +github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= @@ -8,8 +44,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/ekristen/libnuke v0.19.0 h1:pXVxPlbKfYbP1iSwsNu67MQ8HNvZPEZIeKiyw/k8FWg= -github.com/ekristen/libnuke v0.19.0/go.mod h1:riI1tjCf6r+et/9oUBd1vQeFmn2Sn6UeFUR0nWkMeYw= github.com/ekristen/libnuke v0.19.1 h1:n52PMccQjs4MsaYPtulavxmKyHFq4xz3KCy6mpjoX/I= github.com/ekristen/libnuke v0.19.1/go.mod h1:riI1tjCf6r+et/9oUBd1vQeFmn2Sn6UeFUR0nWkMeYw= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= @@ -68,8 +102,6 @@ go.uber.org/ratelimit v0.3.1/go.mod h1:6euWsTB6U/Nb3X++xEUXA8ciPJvr19Q/0h1+oDcJh golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= @@ -95,8 +127,6 @@ golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/awsmod/batch.go b/pkg/awsmod/batch.go index 6b57f853..4b43fa47 100644 --- a/pkg/awsmod/batch.go +++ b/pkg/awsmod/batch.go @@ -2,14 +2,13 @@ package awsmod import ( "bytes" + "context" "fmt" - "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/client" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3iface" ) const ( @@ -64,8 +63,8 @@ func (err *Error) Error() string { origErr = ":\n" + err.OrigErr.Error() } return fmt.Sprintf("failed to perform batch operation on %q to %q%s", - aws.StringValue(err.Key), - aws.StringValue(err.Bucket), + aws.ToString(err.Key), + aws.ToString(err.Bucket), origErr, ) } @@ -137,25 +136,16 @@ type BatchDeleteIterator interface { // } type DeleteListIterator struct { Bucket *string - Paginator request.Pagination - objects []*s3.Object + Paginator *s3.ListObjectsV2Paginator + objects []s3types.Object + err error } // NewDeleteListIterator will return a new DeleteListIterator. -func NewDeleteListIterator(svc s3iface.S3API, input *s3.ListObjectsInput, opts ...func(*DeleteListIterator)) BatchDeleteIterator { +func NewDeleteListIterator(svc s3.ListObjectsV2APIClient, input *s3.ListObjectsV2Input, opts ...func(*DeleteListIterator)) BatchDeleteIterator { iter := &DeleteListIterator{ - Bucket: input.Bucket, - Paginator: request.Pagination{ - NewRequest: func() (*request.Request, error) { - var inCpy *s3.ListObjectsInput - if input != nil { - tmp := *input - inCpy = &tmp - } - req, _ := svc.ListObjectsRequest(inCpy) - return req, nil - }, - }, + Bucket: input.Bucket, + Paginator: s3.NewListObjectsV2Paginator(svc, input), } for _, opt := range opts { @@ -168,18 +158,28 @@ func NewDeleteListIterator(svc s3iface.S3API, input *s3.ListObjectsInput, opts . func (iter *DeleteListIterator) Next() bool { if len(iter.objects) > 0 { iter.objects = iter.objects[1:] + if len(iter.objects) > 0 { + return true + } } - if len(iter.objects) == 0 && iter.Paginator.Next() { - iter.objects = iter.Paginator.Page().(*s3.ListObjectsOutput).Contents + if !iter.Paginator.HasMorePages() { + return false } + page, err := iter.Paginator.NextPage(context.TODO()) + if err != nil { + iter.err = err + return false + } + + iter.objects = page.Contents return len(iter.objects) > 0 } // Err will return the last known error from Next. func (iter *DeleteListIterator) Err() error { - return iter.Paginator.Err() + return iter.err } // DeleteObject will return the current object to be deleted. @@ -192,10 +192,15 @@ func (iter *DeleteListIterator) DeleteObject() BatchDeleteObject { } } +// DeleteObjectsAPIClient implements the S3.DeleteObjects operation. +type DeleteObjectsAPIClient interface { + DeleteObjects(context.Context, *s3.DeleteObjectsInput, ...func(*s3.Options)) (*s3.DeleteObjectsOutput, error) +} + // BatchDelete will use the s3 package's service client to perform a batch // delete. type BatchDelete struct { - Client s3iface.S3API + Client DeleteObjectsAPIClient BatchSize int } @@ -220,7 +225,7 @@ type BatchDelete struct { // }); err != nil { // return err // } -func NewBatchDeleteWithClient(s3client s3iface.S3API, options ...func(*BatchDelete)) *BatchDelete { +func NewBatchDeleteWithClient(s3client DeleteObjectsAPIClient, options ...func(*BatchDelete)) *BatchDelete { svc := &BatchDelete{ Client: s3client, BatchSize: DefaultBatchSize, @@ -254,8 +259,8 @@ func NewBatchDeleteWithClient(s3client s3iface.S3API, options ...func(*BatchDele // }); err != nil { // return err // } -func NewBatchDelete(c client.ConfigProvider, options ...func(*BatchDelete)) *BatchDelete { - s3client := s3.New(c) +func NewBatchDelete(c aws.Config, options ...func(*BatchDelete)) *BatchDelete { + s3client := s3.NewFromConfig(c) return NewBatchDeleteWithClient(s3client, options...) } @@ -300,7 +305,7 @@ func (iter *DeleteObjectsIterator) DeleteObject() BatchDeleteObject { // Delete will use the iterator to queue up objects that need to be deleted. // Once the batch size is met, this will call the deleteBatch function. -func (d *BatchDelete) Delete(ctx aws.Context, iter BatchDeleteIterator, opts ...func(input *s3.DeleteObjectsInput)) error { +func (d *BatchDelete) Delete(ctx context.Context, iter BatchDeleteIterator, opts ...func(input *s3.DeleteObjectsInput)) error { var errs []Error var objects []BatchDeleteObject var input *s3.DeleteObjectsInput @@ -318,7 +323,7 @@ func (d *BatchDelete) Delete(ctx aws.Context, iter BatchDeleteIterator, opts ... parity := hasParity(input, o) if parity { - input.Delete.Objects = append(input.Delete.Objects, &s3.ObjectIdentifier{ + input.Delete.Objects = append(input.Delete.Objects, s3types.ObjectIdentifier{ Key: o.Object.Key, VersionId: o.Object.VersionId, }) @@ -341,7 +346,7 @@ func (d *BatchDelete) Delete(ctx aws.Context, iter BatchDeleteIterator, opts ... opt(input) } - input.Delete.Objects = append(input.Delete.Objects, &s3.ObjectIdentifier{ + input.Delete.Objects = append(input.Delete.Objects, s3types.ObjectIdentifier{ Key: o.Object.Key, VersionId: o.Object.VersionId, }) @@ -371,7 +376,7 @@ func initDeleteObjectsInput(o *s3.DeleteObjectInput) *s3.DeleteObjectsInput { Bucket: o.Bucket, MFA: o.MFA, RequestPayer: o.RequestPayer, - Delete: &s3.Delete{}, + Delete: &s3types.Delete{}, } } @@ -383,10 +388,10 @@ const ( ) // deleteBatch will delete a batch of items in the objects parameters. -func deleteBatch(ctx aws.Context, d *BatchDelete, input *s3.DeleteObjectsInput, objects []BatchDeleteObject) []Error { +func deleteBatch(ctx context.Context, d *BatchDelete, input *s3.DeleteObjectsInput, objects []BatchDeleteObject) []Error { var errs []Error - if result, err := d.Client.DeleteObjectsWithContext(ctx, input); err != nil { + if result, err := d.Client.DeleteObjects(ctx, input); err != nil { for i := 0; i < len(input.Delete.Objects); i++ { errs = append(errs, newError(err, input.Bucket, input.Delete.Objects[i].Key)) } @@ -433,8 +438,8 @@ func hasParity(o1 *s3.DeleteObjectsInput, o2 BatchDeleteObject) bool { return false } - if o1.RequestPayer != nil && o2.Object.RequestPayer != nil { - if *o1.RequestPayer != *o2.Object.RequestPayer { + if o1.RequestPayer != "" && o2.Object.RequestPayer != "" { + if o1.RequestPayer != o2.Object.RequestPayer { return false } } else if o1.RequestPayer != o2.Object.RequestPayer { diff --git a/pkg/awsutil/config.go b/pkg/awsutil/config.go new file mode 100644 index 00000000..6213c85c --- /dev/null +++ b/pkg/awsutil/config.go @@ -0,0 +1,199 @@ +package awsutil + +import ( + "context" + "crypto/tls" + "fmt" + "net/http" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/aws/smithy-go/middleware" + smithyhttp "github.com/aws/smithy-go/transport/http" + liberrors "github.com/ekristen/libnuke/pkg/errors" + log "github.com/sirupsen/logrus" +) + +func (c *Credentials) NewConfig(ctx context.Context, region, serviceType string) (*aws.Config, error) { + log.Debugf("creating new config in %s for %s", region, serviceType) + + if region == GlobalRegionID { + region = "us-east-1" + } + + var cfg *aws.Config + if customRegion := c.CustomEndpoints.GetRegion(region); customRegion != nil { + var opts []func(*config.LoadOptions) error + + customService := customRegion.Services.GetService(serviceType) + if customService == nil { + return nil, liberrors.ErrSkipRequest(fmt.Sprintf( + ".service '%s' is not available in region '%s'", + serviceType, region)) + } + + opts = append(opts, config.WithRegion(region)) + opts = append(opts, config.WithCredentialsProvider(c.awsNewStaticCredentialsV2())) + // FUTURE: SDK v2 does not support sideloading of a base endpoint URL + // + // the only way to force an endpoint from config is to use the + // deprecated "v1" endpoint resolver (in the v2 SDK) + // + // https://github.com/aws/aws-sdk-go-v2/issues/2422 + opts = append(opts, config.WithEndpointResolver(baseEndpoint{customService.URL})) + + if customService.TLSInsecureSkipVerify { + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec + }, + } + opts = append(opts, config.WithHTTPClient(client)) + } + + cfgv, err := config.LoadDefaultConfig(ctx, opts...) + if err != nil { + return nil, err + } + + cfg = &cfgv + } + + if cfg == nil { + root, err := c.rootConfig(ctx) + if err != nil { + return nil, err + } + + cfgCopy := root.Copy() + cfgCopy.Region = region + cfg = &cfgCopy + } + + return cfg, nil +} + +func (c *Credentials) rootConfig(ctx context.Context) (*aws.Config, error) { + if c.cfg != nil { + return c.cfg, nil + } + + var opts []func(*config.LoadOptions) error + opts = append(opts, config.WithAPIOptions([]func(*middleware.Stack) error{ + func(stack *middleware.Stack) error { + if err := stack.Finalize.Add(traceRequest{}, middleware.After); err != nil { + return err + } + return stack.Deserialize.Add(traceResponse{}, middleware.After) + }, + })) + + region := DefaultRegionID + log.Debugf("creating new root session in %s", region) + + switch { + case c.HasAwsCredentials(): // adapts from v1 credentials provider + creds, err := c.Credentials.GetWithContext(ctx) + if err != nil { + return nil, err + } + + static := credentials.NewStaticCredentialsProvider( + creds.AccessKeyID, + creds.SecretAccessKey, + creds.SessionToken, + ) + opts = append(opts, config.WithCredentialsProvider(static)) + + case c.HasProfile() && c.HasKeys(): + return nil, fmt.Errorf("you have to specify a profile or credentials for at least one region") + + case c.HasKeys(): + static := credentials.NewStaticCredentialsProvider( + strings.TrimSpace(c.AccessKeyID), + strings.TrimSpace(c.SecretAccessKey), + strings.TrimSpace(c.SessionToken), + ) + opts = append(opts, config.WithCredentialsProvider(static)) + + case c.HasProfile(): + fallthrough //nolint:gocritic + + default: + opts = append(opts, config.WithSharedConfigProfile(c.Profile)) + } + + opts = append(opts, config.WithRegion(region)) + cfg, err := config.LoadDefaultConfig(ctx, opts...) + if err != nil { + return nil, err + } + + // if given a role to assume, overwrite the session credentials with assume role credentials + if c.AssumeRoleArn != "" { + cfg.Credentials = stscreds.NewAssumeRoleProvider(sts.NewFromConfig(cfg), c.AssumeRoleArn, func(p *stscreds.AssumeRoleOptions) { + if c.RoleSessionName != "" { + p.RoleSessionName = c.RoleSessionName + } + + if c.ExternalID != "" { + p.ExternalID = aws.String(c.ExternalID) + } + }) + } + + c.cfg = &cfg + return c.cfg, nil +} + +type traceRequest struct{} + +func (traceRequest) ID() string { + return "aws-nuke::traceRequest" +} + +func (traceRequest) HandleFinalize( + ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler, +) ( + middleware.FinalizeOutput, middleware.Metadata, error, +) { + req, ok := in.Request.(*smithyhttp.Request) + if !ok { + return next.HandleFinalize(ctx, in) + } + + log.Tracef("sending AWS request:\n%s", DumpRequest(req.Request)) + return next.HandleFinalize(ctx, in) +} + +type traceResponse struct{} + +func (traceResponse) ID() string { + return "aws-nuke::traceResponse" +} + +func (traceResponse) HandleDeserialize( + ctx context.Context, in middleware.DeserializeInput, next middleware.DeserializeHandler, +) ( + middleware.DeserializeOutput, middleware.Metadata, error, +) { + out, md, err := next.HandleDeserialize(ctx, in) + + resp, ok := out.RawResponse.(*smithyhttp.Response) + if ok { + log.Tracef("sending AWS request:\n%s", DumpResponse(resp.Response)) + } + return out, md, err +} + +type baseEndpoint struct { + url string +} + +func (v baseEndpoint) ResolveEndpoint(service, region string) (aws.Endpoint, error) { + return aws.Endpoint{URL: v.url}, nil +} diff --git a/pkg/awsutil/session.go b/pkg/awsutil/session.go index d44421af..ee87ed0a 100644 --- a/pkg/awsutil/session.go +++ b/pkg/awsutil/session.go @@ -1,14 +1,18 @@ package awsutil import ( + "context" "crypto/tls" "fmt" "net" "net/http" + "os" "strings" log "github.com/sirupsen/logrus" + awsv2 "github.com/aws/aws-sdk-go-v2/aws" + credentialsv2 "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/credentials/stscreds" @@ -27,10 +31,10 @@ const ( var ( // DefaultRegionID The default region. Can be customized for non AWS implementations - DefaultRegionID = endpoints.UsEast1RegionID + DefaultRegionID = "us-east-1" // DefaultAWSPartitionID The default aws partition. Can be customized for non AWS implementations - DefaultAWSPartitionID = endpoints.AwsPartitionID + DefaultAWSPartitionID = "aws" ) type Credentials struct { @@ -47,6 +51,7 @@ type Credentials struct { CustomEndpoints config.CustomEndpoints session *session.Session + cfg *awsv2.Config } func (c *Credentials) HasProfile() bool { @@ -73,6 +78,8 @@ func (c *Credentials) Validate() error { return nil } +// FUTURE(187): when all services are migrated to SDK v2, remove usage of +// session.Session throughout func (c *Credentials) rootSession() (*session.Session, error) { if c.session == nil { var opts session.Options @@ -146,6 +153,17 @@ func (c *Credentials) awsNewStaticCredentials() *credentials.Credentials { ) } +func (c *Credentials) awsNewStaticCredentialsV2() awsv2.CredentialsProvider { + if !c.HasKeys() { + return envCredentialsProviderV2{} + } + return credentialsv2.NewStaticCredentialsProvider( + strings.TrimSpace(c.AccessKeyID), + strings.TrimSpace(c.SecretAccessKey), + strings.TrimSpace(c.SessionToken), + ) +} + func (c *Credentials) NewSession(region, serviceType string) (*session.Session, error) { log.Debugf("creating new session in %s for %s", region, serviceType) @@ -274,3 +292,36 @@ func skipGlobalHandler(global bool) func(r *request.Request) { } } } + +// SDK v2 does not directly expose an environment creds provider since that +// functionality has been opaquely rolled into LoadDefaultConfig +// +// this provider recreates the behavior that v1 had (including support for +// extra nonstandard envs) +type envCredentialsProviderV2 struct{} + +func (envCredentialsProviderV2) Retrieve(ctx context.Context) (awsv2.Credentials, error) { + id := os.Getenv("AWS_ACCESS_KEY_ID") + if id == "" { + id = os.Getenv("AWS_ACCESS_KEY") + } + + secret := os.Getenv("AWS_SECRET_ACCESS_KEY") + if secret == "" { + secret = os.Getenv("AWS_SECRET_KEY") + } + + if id == "" { + return awsv2.Credentials{}, fmt.Errorf("AWS access key unset") + } + + if secret == "" { + return awsv2.Credentials{}, fmt.Errorf("AWS secret key unset") + } + + return awsv2.Credentials{ + AccessKeyID: id, + SecretAccessKey: secret, + SessionToken: os.Getenv("AWS_SESSION_TOKEN"), + }, nil +} diff --git a/pkg/commands/nuke/nuke.go b/pkg/commands/nuke/nuke.go index 8f1b7190..72008b14 100644 --- a/pkg/commands/nuke/nuke.go +++ b/pkg/commands/nuke/nuke.go @@ -181,7 +181,7 @@ func execute(c *cli.Context) error { //nolint:funlen,gocyclo // Register the scanners for each region that is defined in the configuration. for _, regionName := range parsedConfig.Regions { // Step 1 - Create the region object - region := nuke.NewRegion(regionName, account.ResourceTypeToServiceType, account.NewSession) + region := nuke.NewRegion(regionName, account.ResourceTypeToServiceType, account.NewSession, account.NewConfig) // Step 2 - Create the scannerActual object scannerActual := scanner.New(regionName, resourceTypes, &nuke.ListerOpts{ diff --git a/pkg/nuke/region.go b/pkg/nuke/region.go index 730ad6aa..357631de 100644 --- a/pkg/nuke/region.go +++ b/pkg/nuke/region.go @@ -1,9 +1,11 @@ package nuke import ( + "context" "fmt" "sync" + awsv2 "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go/aws/session" liberrors "github.com/ekristen/libnuke/pkg/errors" @@ -12,27 +14,34 @@ import ( // SessionFactory support for custom endpoints type SessionFactory func(regionName, svcType string) (*session.Session, error) +// ConfigFactory is the SDK v2 equivalent to SessionFactory. +type ConfigFactory func(ctx context.Context, regionName, svcType string) (*awsv2.Config, error) + // ResourceTypeResolver returns the service type from the resourceType type ResourceTypeResolver func(regionName, resourceType string) string // Region is an AWS Region with an attached SessionFactory type Region struct { Name string - NewSession SessionFactory + NewSession SessionFactory // SDK v1 + NewConfig ConfigFactory // SDK v2 ResTypeResolver ResourceTypeResolver - cache map[string]*session.Session - lock *sync.RWMutex + cache map[string]*session.Session + cfgCache map[string]*awsv2.Config + lock *sync.RWMutex } // NewRegion creates a new Region and returns it. -func NewRegion(name string, typeResolver ResourceTypeResolver, sessionFactory SessionFactory) *Region { +func NewRegion(name string, typeResolver ResourceTypeResolver, sessionFactory SessionFactory, cfgFactory ConfigFactory) *Region { return &Region{ Name: name, NewSession: sessionFactory, + NewConfig: cfgFactory, ResTypeResolver: typeResolver, lock: &sync.RWMutex{}, cache: make(map[string]*session.Session), + cfgCache: make(map[string]*awsv2.Config), } } @@ -64,3 +73,33 @@ func (region *Region) Session(resourceType string) (*session.Session, error) { region.lock.Unlock() return sess, nil } + +// Config returns an SDK v2 config for a given resource type for the region +// it's associated to. +func (region *Region) Config(resourceType string) (*awsv2.Config, error) { + svcType := region.ResTypeResolver(region.Name, resourceType) + if svcType == "" { + return nil, liberrors.ErrSkipRequest(fmt.Sprintf( + "No service available in region '%s' to handle '%s'", + region.Name, resourceType)) + } + + // Need to read + region.lock.RLock() + cfg := region.cfgCache[svcType] + region.lock.RUnlock() + if cfg != nil { + return cfg, nil + } + + // Need to write: + region.lock.Lock() + cfg, err := region.NewConfig(context.TODO(), region.Name, svcType) + if err != nil { + region.lock.Unlock() + return nil, err + } + region.cfgCache[svcType] = cfg + region.lock.Unlock() + return cfg, nil +} diff --git a/pkg/nuke/resource.go b/pkg/nuke/resource.go index b6017ca5..71c1d7dc 100644 --- a/pkg/nuke/resource.go +++ b/pkg/nuke/resource.go @@ -3,6 +3,7 @@ package nuke import ( "github.com/sirupsen/logrus" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/ekristen/libnuke/pkg/registry" @@ -16,7 +17,8 @@ const Account registry.Scope = "account" // the interface{} to get the options it needs. type ListerOpts struct { Region *Region - Session *session.Session + Session *session.Session // SDK v1 + Config *aws.Config // SDK v2 AccountID *string Logger *logrus.Entry } @@ -34,6 +36,13 @@ var MutateOpts = func(opts interface{}, resourceType string) interface{} { o.Session = session + cfg, err := o.Region.Config(resourceType) + if err != nil { + panic(err) + } + + o.Config = cfg + if o.Logger != nil { o.Logger = o.Logger.WithField("resource", resourceType) } else { diff --git a/resources/s3-bucket.go b/resources/s3-bucket.go index b1d44570..57251885 100644 --- a/resources/s3-bucket.go +++ b/resources/s3-bucket.go @@ -9,12 +9,10 @@ import ( "github.com/gotidy/ptr" "github.com/sirupsen/logrus" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/endpoints" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3iface" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" "github.com/ekristen/libnuke/pkg/registry" "github.com/ekristen/libnuke/pkg/resource" @@ -45,11 +43,11 @@ func init() { type S3BucketLister struct{} -func (l *S3BucketLister) List(_ context.Context, o interface{}) ([]resource.Resource, error) { +func (l *S3BucketLister) List(ctx context.Context, o interface{}) ([]resource.Resource, error) { opts := o.(*nuke.ListerOpts) - svc := s3.New(opts.Session) + svc := s3.NewFromConfig(*opts.Config) - buckets, err := DescribeS3Buckets(svc) + buckets, err := DescribeS3Buckets(ctx, svc) if err != nil { return nil, err } @@ -58,19 +56,19 @@ func (l *S3BucketLister) List(_ context.Context, o interface{}) ([]resource.Reso for _, bucket := range buckets { newBucket := &S3Bucket{ svc: svc, - name: aws.StringValue(bucket.Name), - creationDate: aws.TimeValue(bucket.CreationDate), - tags: make([]*s3.Tag, 0), + name: aws.ToString(bucket.Name), + creationDate: aws.ToTime(bucket.CreationDate), + tags: make([]s3types.Tag, 0), } - lockCfg, err := svc.GetObjectLockConfiguration(&s3.GetObjectLockConfigurationInput{ + lockCfg, err := svc.GetObjectLockConfiguration(ctx, &s3.GetObjectLockConfigurationInput{ Bucket: &newBucket.name, }) if err != nil { // check if aws error is NoSuchObjectLockConfiguration - var aerr awserr.Error + var aerr smithy.APIError if errors.As(err, &aerr) { - if aerr.Code() != "ObjectLockConfigurationNotFoundError" { + if aerr.ErrorCode() != "ObjectLockConfigurationNotFoundError" { logrus.WithError(err).Warn("unknown failure during get object lock configuration") } } @@ -80,13 +78,13 @@ func (l *S3BucketLister) List(_ context.Context, o interface{}) ([]resource.Reso newBucket.ObjectLock = lockCfg.ObjectLockConfiguration.ObjectLockEnabled } - tags, err := svc.GetBucketTagging(&s3.GetBucketTaggingInput{ + tags, err := svc.GetBucketTagging(ctx, &s3.GetBucketTaggingInput{ Bucket: bucket.Name, }) if err != nil { - var aerr awserr.Error + var aerr smithy.APIError if errors.As(err, &aerr) { - if aerr.Code() == "NoSuchTagSet" { + if aerr.ErrorCode() == "NoSuchTagSet" { resources = append(resources, newBucket) } } @@ -100,31 +98,37 @@ func (l *S3BucketLister) List(_ context.Context, o interface{}) ([]resource.Reso return resources, nil } -func DescribeS3Buckets(svc *s3.S3) ([]s3.Bucket, error) { - resp, err := svc.ListBuckets(nil) +type DescribeS3BucketsAPIClient interface { + Options() s3.Options + ListBuckets(context.Context, *s3.ListBucketsInput, ...func(*s3.Options)) (*s3.ListBucketsOutput, error) + GetBucketLocation(context.Context, *s3.GetBucketLocationInput, ...func(*s3.Options)) (*s3.GetBucketLocationOutput, error) +} + +func DescribeS3Buckets(ctx context.Context, svc DescribeS3BucketsAPIClient) ([]s3types.Bucket, error) { + resp, err := svc.ListBuckets(ctx, nil) if err != nil { return nil, err } - buckets := make([]s3.Bucket, 0) + buckets := make([]s3types.Bucket, 0) for _, out := range resp.Buckets { - bucketLocationResponse, err := svc.GetBucketLocation(&s3.GetBucketLocationInput{Bucket: out.Name}) + bucketLocationResponse, err := svc.GetBucketLocation(ctx, &s3.GetBucketLocationInput{Bucket: out.Name}) if err != nil { continue } - location := ptr.ToString(bucketLocationResponse.LocationConstraint) + location := string(bucketLocationResponse.LocationConstraint) if location == "" { - location = endpoints.UsEast1RegionID + location = "us-east-1" } - region := ptr.ToString(svc.Config.Region) + region := svc.Options().Region if region == "" { - region = endpoints.UsEast1RegionID + region = "us-east-1" } if location == region { - buckets = append(buckets, *out) + buckets = append(buckets, out) } } @@ -132,25 +136,25 @@ func DescribeS3Buckets(svc *s3.S3) ([]s3.Bucket, error) { } type S3Bucket struct { - svc *s3.S3 + svc *s3.Client settings *libsettings.Setting name string creationDate time.Time - tags []*s3.Tag - ObjectLock *string + tags []s3types.Tag + ObjectLock s3types.ObjectLockEnabled } func (r *S3Bucket) Remove(ctx context.Context) error { - _, err := r.svc.DeleteBucketPolicy(&s3.DeleteBucketPolicyInput{ + _, err := r.svc.DeleteBucketPolicy(ctx, &s3.DeleteBucketPolicyInput{ Bucket: &r.name, }) if err != nil { return err } - _, err = r.svc.PutBucketLogging(&s3.PutBucketLoggingInput{ + _, err = r.svc.PutBucketLogging(ctx, &s3.PutBucketLoggingInput{ Bucket: &r.name, - BucketLoggingStatus: &s3.BucketLoggingStatus{}, + BucketLoggingStatus: &s3types.BucketLoggingStatus{}, }) if err != nil { return err @@ -171,19 +175,19 @@ func (r *S3Bucket) Remove(ctx context.Context) error { return err } - _, err = r.svc.DeleteBucket(&s3.DeleteBucketInput{ + _, err = r.svc.DeleteBucket(ctx, &s3.DeleteBucketInput{ Bucket: &r.name, }) return err } -func (r *S3Bucket) RemoveAllLegalHolds(_ context.Context) error { +func (r *S3Bucket) RemoveAllLegalHolds(ctx context.Context) error { if !r.settings.GetBool("RemoveObjectLegalHold") { return nil } - if r.ObjectLock == nil || ptr.ToString(r.ObjectLock) != "Enabled" { + if r.ObjectLock == "" || r.ObjectLock != s3types.ObjectLockEnabledEnabled { return nil } @@ -192,7 +196,7 @@ func (r *S3Bucket) RemoveAllLegalHolds(_ context.Context) error { } for { - res, err := r.svc.ListObjectsV2(params) + res, err := r.svc.ListObjectsV2(ctx, params) if err != nil { return err } @@ -200,10 +204,10 @@ func (r *S3Bucket) RemoveAllLegalHolds(_ context.Context) error { params.ContinuationToken = res.NextContinuationToken for _, obj := range res.Contents { - _, err := r.svc.PutObjectLegalHold(&s3.PutObjectLegalHoldInput{ + _, err := r.svc.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ Bucket: &r.name, Key: obj.Key, - LegalHold: &s3.ObjectLockLegalHold{Status: aws.String("OFF")}, + LegalHold: &s3types.ObjectLockLegalHold{Status: s3types.ObjectLockLegalHoldStatusOff}, }) if err != nil { return err @@ -225,7 +229,7 @@ func (r *S3Bucket) RemoveAllVersions(ctx context.Context) error { var setBypass bool var opts []func(input *s3.DeleteObjectsInput) - if ptr.ToString(r.ObjectLock) == s3.ObjectLockEnabledEnabled && + if r.ObjectLock == s3types.ObjectLockEnabledEnabled && r.settings.GetBool("BypassGovernanceRetention") { setBypass = true opts = append(opts, bypassGovernanceRetention) @@ -236,13 +240,13 @@ func (r *S3Bucket) RemoveAllVersions(ctx context.Context) error { } func (r *S3Bucket) RemoveAllObjects(ctx context.Context) error { - params := &s3.ListObjectsInput{ + params := &s3.ListObjectsV2Input{ Bucket: &r.name, } var setBypass bool var opts []func(input *s3.DeleteObjectsInput) - if ptr.ToString(r.ObjectLock) == s3.ObjectLockEnabledEnabled && + if r.ObjectLock == s3types.ObjectLockEnabledEnabled && r.settings.GetBool("BypassGovernanceRetention") { setBypass = true opts = append(opts, bypassGovernanceRetention) @@ -279,30 +283,21 @@ func bypassGovernanceRetention(input *s3.DeleteObjectsInput) { type s3DeleteVersionListIterator struct { Bucket *string - Paginator request.Pagination - objects []*s3.ObjectVersion + Paginator *s3.ListObjectVersionsPaginator + objects []s3types.ObjectVersion lastNotify time.Time BypassGovernanceRetention *bool + err error } func newS3DeleteVersionListIterator( - svc s3iface.S3API, + svc *s3.Client, input *s3.ListObjectVersionsInput, bypass bool, opts ...func(*s3DeleteVersionListIterator)) awsmod.BatchDeleteIterator { iter := &s3DeleteVersionListIterator{ - Bucket: input.Bucket, - Paginator: request.Pagination{ - NewRequest: func() (*request.Request, error) { - var inCpy *s3.ListObjectVersionsInput - if input != nil { - tmp := *input - inCpy = &tmp - } - req, _ := svc.ListObjectVersionsRequest(inCpy) - return req, nil - }, - }, + Bucket: input.Bucket, + Paginator: s3.NewListObjectVersionsPaginator(svc, input), BypassGovernanceRetention: ptr.Bool(bypass), } @@ -317,18 +312,27 @@ func newS3DeleteVersionListIterator( func (iter *s3DeleteVersionListIterator) Next() bool { if len(iter.objects) > 0 { iter.objects = iter.objects[1:] + if len(iter.objects) > 0 { + return true + } } - if len(iter.objects) == 0 && iter.Paginator.Next() { - output := iter.Paginator.Page().(*s3.ListObjectVersionsOutput) - iter.objects = output.Versions + if !iter.Paginator.HasMorePages() { + return false + } - for _, entry := range output.DeleteMarkers { - iter.objects = append(iter.objects, &s3.ObjectVersion{ - Key: entry.Key, - VersionId: entry.VersionId, - }) - } + page, err := iter.Paginator.NextPage(context.TODO()) + if err != nil { + iter.err = err + return false + } + + iter.objects = page.Versions + for _, entry := range page.DeleteMarkers { + iter.objects = append(iter.objects, s3types.ObjectVersion{ + Key: entry.Key, + VersionId: entry.VersionId, + }) } if len(iter.objects) > 500 && (iter.lastNotify.IsZero() || time.Since(iter.lastNotify) > 120*time.Second) { @@ -343,7 +347,7 @@ func (iter *s3DeleteVersionListIterator) Next() bool { // Err will return the last known error from Next. func (iter *s3DeleteVersionListIterator) Err() error { - return iter.Paginator.Err() + return iter.err } // DeleteObject will return the current object to be deleted. @@ -360,30 +364,21 @@ func (iter *s3DeleteVersionListIterator) DeleteObject() awsmod.BatchDeleteObject type s3ObjectDeleteListIterator struct { Bucket *string - Paginator request.Pagination - objects []*s3.Object + Paginator *s3.ListObjectsV2Paginator + objects []s3types.Object lastNotify time.Time BypassGovernanceRetention bool + err error } func newS3ObjectDeleteListIterator( - svc s3iface.S3API, - input *s3.ListObjectsInput, + svc *s3.Client, + input *s3.ListObjectsV2Input, bypass bool, opts ...func(*s3ObjectDeleteListIterator)) awsmod.BatchDeleteIterator { iter := &s3ObjectDeleteListIterator{ - Bucket: input.Bucket, - Paginator: request.Pagination{ - NewRequest: func() (*request.Request, error) { - var inCpy *s3.ListObjectsInput - if input != nil { - tmp := *input - inCpy = &tmp - } - req, _ := svc.ListObjectsRequest(inCpy) - return req, nil - }, - }, + Bucket: input.Bucket, + Paginator: s3.NewListObjectsV2Paginator(svc, input), BypassGovernanceRetention: bypass, } @@ -397,12 +392,23 @@ func newS3ObjectDeleteListIterator( func (iter *s3ObjectDeleteListIterator) Next() bool { if len(iter.objects) > 0 { iter.objects = iter.objects[1:] + if len(iter.objects) > 0 { + return true + } } - if len(iter.objects) == 0 && iter.Paginator.Next() { - iter.objects = iter.Paginator.Page().(*s3.ListObjectsOutput).Contents + if !iter.Paginator.HasMorePages() { + return false } + page, err := iter.Paginator.NextPage(context.TODO()) + if err != nil { + iter.err = err + return false + } + + iter.objects = page.Contents + if len(iter.objects) > 500 && (iter.lastNotify.IsZero() || time.Since(iter.lastNotify) > 120*time.Second) { logrus.Infof( "S3Bucket: %s - empty bucket operation in progress, this could take a while, please be patient", @@ -415,7 +421,7 @@ func (iter *s3ObjectDeleteListIterator) Next() bool { // Err will return the last known error from Next. func (iter *s3ObjectDeleteListIterator) Err() error { - return iter.Paginator.Err() + return iter.err } // DeleteObject will return the current object to be deleted. diff --git a/resources/s3-bucket_test.go b/resources/s3-bucket_test.go index e22f7aa9..b153a583 100644 --- a/resources/s3-bucket_test.go +++ b/resources/s3-bucket_test.go @@ -5,6 +5,7 @@ package resources import ( "context" "fmt" + "io" "strings" "testing" "time" @@ -12,19 +13,24 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" libsettings "github.com/ekristen/libnuke/pkg/settings" "github.com/ekristen/aws-nuke/v3/pkg/awsmod" ) +type readSeekCloser struct{ io.ReadSeeker } + +func (readSeekCloser) Close() error { return nil } + type TestS3BucketObjectLockSuite struct { suite.Suite bucket string - svc *s3.S3 + svc *s3.Client } func (suite *TestS3BucketObjectLockSuite) SetupSuite() { @@ -32,29 +38,32 @@ func (suite *TestS3BucketObjectLockSuite) SetupSuite() { suite.bucket = fmt.Sprintf("aws-nuke-testing-bucket-%d", time.Now().UnixNano()) - sess, err := session.NewSession(&aws.Config{ - Region: aws.String("us-west-2")}, - ) + ctx := context.TODO() + + cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion("us-west-2")) if err != nil { suite.T().Fatalf("failed to create session, %v", err) } // Create S3 service client - suite.svc = s3.New(sess) + suite.svc = s3.NewFromConfig(cfg) // Create the bucket - _, err = suite.svc.CreateBucket(&s3.CreateBucketInput{ + _, err = suite.svc.CreateBucket(ctx, &s3.CreateBucketInput{ Bucket: aws.String(suite.bucket), + CreateBucketConfiguration: &s3types.CreateBucketConfiguration{ + LocationConstraint: s3types.BucketLocationConstraint("us-west-2"), + }, }) if err != nil { suite.T().Fatalf("failed to create bucket, %v", err) } // enable versioning - _, err = suite.svc.PutBucketVersioning(&s3.PutBucketVersioningInput{ + _, err = suite.svc.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{ Bucket: aws.String(suite.bucket), - VersioningConfiguration: &s3.VersioningConfiguration{ - Status: aws.String("Enabled"), + VersioningConfiguration: &s3types.VersioningConfiguration{ + Status: s3types.BucketVersioningStatusEnabled, }, }) if err != nil { @@ -62,14 +71,14 @@ func (suite *TestS3BucketObjectLockSuite) SetupSuite() { } // Set the object lock configuration to governance mode - _, err = suite.svc.PutObjectLockConfiguration(&s3.PutObjectLockConfigurationInput{ + _, err = suite.svc.PutObjectLockConfiguration(ctx, &s3.PutObjectLockConfigurationInput{ Bucket: aws.String(suite.bucket), - ObjectLockConfiguration: &s3.ObjectLockConfiguration{ - ObjectLockEnabled: aws.String("Enabled"), - Rule: &s3.ObjectLockRule{ - DefaultRetention: &s3.DefaultRetention{ - Mode: aws.String("GOVERNANCE"), - Days: aws.Int64(1), + ObjectLockConfiguration: &s3types.ObjectLockConfiguration{ + ObjectLockEnabled: s3types.ObjectLockEnabledEnabled, + Rule: &s3types.ObjectLockRule{ + DefaultRetention: &s3types.DefaultRetention{ + Mode: s3types.ObjectLockRetentionModeGovernance, + Days: aws.Int32(1), }, }, }, @@ -79,10 +88,11 @@ func (suite *TestS3BucketObjectLockSuite) SetupSuite() { } // Create an object in the bucket - _, err = suite.svc.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(suite.bucket), - Key: aws.String("test-object"), - Body: aws.ReadSeekCloser(strings.NewReader("test content")), + _, err = suite.svc.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(suite.bucket), + Key: aws.String("test-object"), + Body: readSeekCloser{strings.NewReader("test content")}, + ChecksumAlgorithm: s3types.ChecksumAlgorithmCrc32, }) if err != nil { suite.T().Fatalf("failed to create object, %v", err) @@ -99,7 +109,7 @@ func (suite *TestS3BucketObjectLockSuite) TearDownSuite() { } } - iterator2 := newS3ObjectDeleteListIterator(suite.svc, &s3.ListObjectsInput{ + iterator2 := newS3ObjectDeleteListIterator(suite.svc, &s3.ListObjectsV2Input{ Bucket: &suite.bucket, }, true) if err := awsmod.NewBatchDeleteWithClient(suite.svc).Delete(context.TODO(), iterator2, bypassGovernanceRetention); err != nil { @@ -108,7 +118,7 @@ func (suite *TestS3BucketObjectLockSuite) TearDownSuite() { } } - _, err := suite.svc.DeleteBucket(&s3.DeleteBucketInput{ + _, err := suite.svc.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{ Bucket: aws.String(suite.bucket), }) if err != nil { @@ -120,16 +130,16 @@ func (suite *TestS3BucketObjectLockSuite) TearDownSuite() { func (suite *TestS3BucketObjectLockSuite) TestS3BucketObjectLock() { // Verify the object lock configuration - result, err := suite.svc.GetObjectLockConfiguration(&s3.GetObjectLockConfigurationInput{ + result, err := suite.svc.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{ Bucket: aws.String(suite.bucket), }) if err != nil { suite.T().Fatalf("failed to get object lock configuration, %v", err) } - assert.Equal(suite.T(), "Enabled", *result.ObjectLockConfiguration.ObjectLockEnabled) - assert.Equal(suite.T(), "GOVERNANCE", *result.ObjectLockConfiguration.Rule.DefaultRetention.Mode) - assert.Equal(suite.T(), int64(1), *result.ObjectLockConfiguration.Rule.DefaultRetention.Days) + assert.Equal(suite.T(), s3types.ObjectLockEnabledEnabled, result.ObjectLockConfiguration.ObjectLockEnabled) + assert.Equal(suite.T(), s3types.ObjectLockRetentionModeGovernance, result.ObjectLockConfiguration.Rule.DefaultRetention.Mode) + assert.Equal(suite.T(), int32(1), *result.ObjectLockConfiguration.Rule.DefaultRetention.Days) } func (suite *TestS3BucketObjectLockSuite) TestS3BucketRemove() { diff --git a/resources/s3-multipart-uploads.go b/resources/s3-multipart-uploads.go index d44f8a53..0d413ff9 100644 --- a/resources/s3-multipart-uploads.go +++ b/resources/s3-multipart-uploads.go @@ -5,8 +5,8 @@ import ( "fmt" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/ekristen/libnuke/pkg/registry" "github.com/ekristen/libnuke/pkg/resource" @@ -27,13 +27,13 @@ func init() { type S3MultipartUploadLister struct{} -func (l *S3MultipartUploadLister) List(_ context.Context, o interface{}) ([]resource.Resource, error) { +func (l *S3MultipartUploadLister) List(ctx context.Context, o interface{}) ([]resource.Resource, error) { opts := o.(*nuke.ListerOpts) - svc := s3.New(opts.Session) + svc := s3.NewFromConfig(*opts.Config) resources := make([]resource.Resource, 0) - buckets, err := DescribeS3Buckets(svc) + buckets, err := DescribeS3Buckets(ctx, svc) if err != nil { return nil, err } @@ -44,7 +44,7 @@ func (l *S3MultipartUploadLister) List(_ context.Context, o interface{}) ([]reso } for { - resp, err := svc.ListMultipartUploads(params) + resp, err := svc.ListMultipartUploads(ctx, params) if err != nil { return nil, err } @@ -56,7 +56,7 @@ func (l *S3MultipartUploadLister) List(_ context.Context, o interface{}) ([]reso resources = append(resources, &S3MultipartUpload{ svc: svc, - bucket: aws.StringValue(bucket.Name), + bucket: aws.ToString(bucket.Name), key: *upload.Key, uploadID: *upload.UploadId, }) @@ -75,20 +75,20 @@ func (l *S3MultipartUploadLister) List(_ context.Context, o interface{}) ([]reso } type S3MultipartUpload struct { - svc *s3.S3 + svc *s3.Client bucket string key string uploadID string } -func (e *S3MultipartUpload) Remove(_ context.Context) error { +func (e *S3MultipartUpload) Remove(ctx context.Context) error { params := &s3.AbortMultipartUploadInput{ Bucket: &e.bucket, Key: &e.key, UploadId: &e.uploadID, } - _, err := e.svc.AbortMultipartUpload(params) + _, err := e.svc.AbortMultipartUpload(ctx, params) if err != nil { return err } diff --git a/resources/s3-objects.go b/resources/s3-objects.go index 1817f821..241d09a3 100644 --- a/resources/s3-objects.go +++ b/resources/s3-objects.go @@ -8,8 +8,8 @@ import ( "github.com/gotidy/ptr" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/ekristen/libnuke/pkg/registry" "github.com/ekristen/libnuke/pkg/resource" @@ -30,13 +30,13 @@ func init() { type S3ObjectLister struct{} -func (l *S3ObjectLister) List(_ context.Context, o interface{}) ([]resource.Resource, error) { +func (l *S3ObjectLister) List(ctx context.Context, o interface{}) ([]resource.Resource, error) { opts := o.(*nuke.ListerOpts) - svc := s3.New(opts.Session) + svc := s3.NewFromConfig(*opts.Config) resources := make([]resource.Resource, 0) - buckets, err := DescribeS3Buckets(svc) + buckets, err := DescribeS3Buckets(ctx, svc) if err != nil { return nil, err } @@ -47,7 +47,7 @@ func (l *S3ObjectLister) List(_ context.Context, o interface{}) ([]resource.Reso } for { - resp, err := svc.ListObjectVersions(params) + resp, err := svc.ListObjectVersions(ctx, params) if err != nil { return nil, err } @@ -59,8 +59,8 @@ func (l *S3ObjectLister) List(_ context.Context, o interface{}) ([]resource.Reso resources = append(resources, &S3Object{ svc: svc, - bucket: aws.StringValue(bucket.Name), - creationDate: aws.TimeValue(bucket.CreationDate), + bucket: aws.ToString(bucket.Name), + creationDate: aws.ToTime(bucket.CreationDate), key: *out.Key, versionID: out.VersionId, latest: ptr.ToBool(out.IsLatest), @@ -74,8 +74,8 @@ func (l *S3ObjectLister) List(_ context.Context, o interface{}) ([]resource.Reso resources = append(resources, &S3Object{ svc: svc, - bucket: aws.StringValue(bucket.Name), - creationDate: aws.TimeValue(bucket.CreationDate), + bucket: aws.ToString(bucket.Name), + creationDate: aws.ToTime(bucket.CreationDate), key: *out.Key, versionID: out.VersionId, latest: ptr.ToBool(out.IsLatest), @@ -96,7 +96,7 @@ func (l *S3ObjectLister) List(_ context.Context, o interface{}) ([]resource.Reso } type S3Object struct { - svc *s3.S3 + svc *s3.Client bucket string creationDate time.Time key string @@ -104,14 +104,14 @@ type S3Object struct { latest bool } -func (e *S3Object) Remove(_ context.Context) error { +func (e *S3Object) Remove(ctx context.Context) error { params := &s3.DeleteObjectInput{ Bucket: &e.bucket, Key: &e.key, VersionId: e.versionID, } - _, err := e.svc.DeleteObject(params) + _, err := e.svc.DeleteObject(ctx, params) if err != nil { return err }