Merge pull request #161 from nabbar/aws-mpu-rework

Rework Aws/MPU

Package AWS:
- rework MultipartUpload process & helper
- update test to use lib size
- update object multipart to use new helper
    
Package IO Utils :
- add truncate & sync to FileProgress
- fix error on open file mode for FileProgress
    
Package Console :
- fix interface used for color buffer
    
Package Cobra :
- add function to print message on write config to use custom message instead of internal message. If the function is not set, the default message will be print.
    
Other:
- fix golangci-lint config to remove crazy linter (use only golang group compliance linter)
- bump dependencies
- fix issue #160

.golangci.yml

@@ -82,20 +82,20 @@ linters:
   disable:
 #    - bodyclose
 #    - contextcheck
-#    - cyclop
+    - cyclop
     - deadcode
 #    - errname
 #    - errorlint
     - exhaustive
     - exhaustivestruct
     - exhaustruct
 #    - forbidigo
-#    - funlen
+    - funlen
 #    - gci
 #    - gochecknoglobals
 #    - gochecknoinits
 #    - gocognit
-#    - gocritic
+    - gocritic
 #    - gocyclo
 #    - godot
 #    - godox
@@ -106,16 +106,16 @@ linters:
     - ifshort
 #    - interfacebloat
     - interfacer
-#    - ireturn
+    - ireturn
 #    - lll
     - maligned
 #    - nakedret
-#    - nestif
+    - nestif
 #    - nilerr
 #    - nlreturn
 #    - noctx
-#    - nolintlint
-#    - nonamedreturns
+    - nolintlint
+    - nonamedreturns
     - nosnakecase
 #    - revive
 #    - rowserrcheck
@@ -129,5 +129,5 @@ linters:
     - varnamelen
 #    - wastedassign
 #    - whitespace
-#    - wrapcheck
-#    - wsl
+    - wrapcheck
+    - wsl

aws/aws_suite_test.go

@@ -31,6 +31,7 @@ import (
 	"crypto/rand"
 	"errors"
 	"fmt"
+	libsiz "github.com/nabbar/golib/size"
 	"io/ioutil"
 	"net"
 	"net/http"
@@ -240,8 +241,8 @@ func WaitMinio(host string) bool {
 	return err == nil
 }
 
-func randContent(size int) *bytes.Buffer {
-	p := make([]byte, size)
+func randContent(size libsiz.Size) *bytes.Buffer {
+	p := make([]byte, size.Int64())
 
 	_, err := rand.Read(p)
 

aws/bucket_test.go

@@ -26,6 +26,7 @@
 package aws_test
 
 import (
+	libsiz "github.com/nabbar/golib/size"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
@@ -52,7 +53,7 @@ var _ = Describe("Bucket", func() {
 			It("Must succeed", func() {
 				var (
 					err error
-					rnd = randContent(64 * 1024)
+					rnd = randContent(10 * libsiz.SizeMega)
 				)
 
 				err = cli.Object().MultipartPut("object", rnd)

aws/multipart/errors.go

@@ -0,0 +1,37 @@
+/*
+ *  MIT License
+ *
+ *  Copyright (c) 2020 Nicolas JUHEL
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy
+ *  of this software and associated documentation files (the "Software"), to deal
+ *  in the Software without restriction, including without limitation the rights
+ *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *  copies of the Software, and to permit persons to whom the Software is
+ *  furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in all
+ *  copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *  SOFTWARE.
+ *
+ */
+
+package multipart
+
+import "fmt"
+
+var (
+	ErrInvalidInstance           = fmt.Errorf("invalid instance")
+	ErrInvalidClient             = fmt.Errorf("invalid aws S3 client")
+	ErrInvalidResponse           = fmt.Errorf("invalid aws S3 response")
+	ErrInvalidUploadID           = fmt.Errorf("invalid aws s3 MPU Upload ID")
+	ErrInvalidTMPFile            = fmt.Errorf("invalid working or temporary file")
+	ErrWorkingPartFileExceedSize = fmt.Errorf("working or temporary file used exceed the aws S3 size limits")
+)

aws/multipart/interface.go

@@ -0,0 +1,114 @@
+/*
+ *  MIT License
+ *
+ *  Copyright (c) 2020 Nicolas JUHEL
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy
+ *  of this software and associated documentation files (the "Software"), to deal
+ *  in the Software without restriction, including without limitation the rights
+ *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *  copies of the Software, and to permit persons to whom the Software is
+ *  furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in all
+ *  copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *  SOFTWARE.
+ *
+ */
+
+package multipart
+
+import (
+	"fmt"
+	"io"
+	"math"
+	"sync"
+
+	sdksss "github.com/aws/aws-sdk-go-v2/service/s3"
+	libctx "github.com/nabbar/golib/context"
+	libsiz "github.com/nabbar/golib/size"
+)
+
+const (
+	DefaultPartSize       = 5 * libsiz.SizeMega
+	MaxPartSize           = 5 * libsiz.SizeGiga
+	MaxObjectSize         = 5 * libsiz.SizeTera
+	MaxNumberPart   int32 = 10000
+)
+
+type FuncClientS3 func() *sdksss.Client
+
+type MultiPart interface {
+	io.WriteCloser
+
+	RegisterContext(fct libctx.FuncContext)
+	RegisterClientS3(fct FuncClientS3)
+	RegisterMultipartID(id string)
+	RegisterWorkingFile(file string, truncate bool) error
+	RegisterFuncOnPushPart(fct func(eTag string, e error))
+	RegisterFuncOnAbort(fct func(nPart int, obj string, e error))
+	RegisterFuncOnComplete(fct func(nPart int, obj string, e error))
+
+	StartMPU() error
+	StopMPU(abort bool) error
+
+	AddPart(r io.Reader) (n int64, e error)
+	SendPart() error
+	CurrentSizePart() int64
+	AddToPart(p []byte) (n int, e error)
+	RegisterPart(etag string)
+
+	IsStarted() bool
+	Counter() int32
+	CounterLeft() int32
+}
+
+func New(partSize libsiz.Size, object string, bucket string) MultiPart {
+	return &mpu{
+		m: sync.RWMutex{},
+		c: nil,
+		s: partSize,
+		i: "",
+		o: object,
+		b: bucket,
+		n: 0,
+	}
+}
+
+func GetOptimalPartSize(objectSize, partSize libsiz.Size) (libsiz.Size, error) {
+	var (
+		lim = math.MaxFloat64
+		nbr int64
+		prt int64
+		obj int64
+	)
+
+	if partSize <= DefaultPartSize {
+		prt = DefaultPartSize.Int64()
+	} else {
+		prt = partSize.Int64()
+	}
+
+	obj = objectSize.Int64()
+
+	if obj > (int64(MaxNumberPart) * MaxPartSize.Int64()) {
+		return 0, fmt.Errorf("object size need exceed the maximum number of part with the maximum size of part")
+	} else if objectSize > MaxObjectSize {
+		return 0, fmt.Errorf("object size is over allowed maximum size of object")
+	} else if uint64(obj) > uint64(lim) || uint64(prt) > uint64(lim) || uint64(obj/prt) > uint64(lim) {
+		return GetOptimalPartSize(objectSize, libsiz.SizeFromInt64(prt*2))
+	} else if nbr = int64(math.Ceil(float64(obj) / float64(prt))); nbr > int64(MaxNumberPart) {
+		return GetOptimalPartSize(objectSize, libsiz.SizeFromInt64(prt*2))
+	} else if prt > MaxPartSize.Int64() {
+		return GetOptimalPartSize(objectSize, libsiz.SizeFromInt64((prt/4)*3))
+	}
+
+	return libsiz.SizeFromInt64(prt), nil
+}

aws/multipart/io.go

@@ -0,0 +1,34 @@
+/*
+ *  MIT License
+ *
+ *  Copyright (c) 2020 Nicolas JUHEL
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy
+ *  of this software and associated documentation files (the "Software"), to deal
+ *  in the Software without restriction, including without limitation the rights
+ *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *  copies of the Software, and to permit persons to whom the Software is
+ *  furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in all
+ *  copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *  SOFTWARE.
+ *
+ */
+
+package multipart
+
+func (m *mpu) Write(p []byte) (n int, err error) {
+	return m.AddToPart(p)
+}
+
+func (m *mpu) Close() error {
+	return m.StopMPU(false)
+}