Merge pull request #740 from rodrigargar/respect-s3-max-parts-count

respect max_parts_count
Altinity · Sep 7, 2023 · 76e1e08 · 76e1e08
2 parents 94b0b42 + 0504c8f
commit 76e1e08
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 1 deletion.
diff --git a/pkg/storage/general.go b/pkg/storage/general.go
@@ -633,6 +633,9 @@ func NewBackupDestination(ctx context.Context, cfg *config.Config, ch *clickhous
 		// https://github.com/Altinity/clickhouse-backup/issues/317
 		if bufferSize <= 0 {
 			bufferSize = int(cfg.General.MaxFileSize) / cfg.AzureBlob.MaxPartsCount
+			if int(cfg.General.MaxFileSize) % cfg.AzureBlob.MaxPartsCount > 0 {
+				bufferSize++
+			}
 			if bufferSize < 2*1024*1024 {
 				bufferSize = 2 * 1024 * 1024
 			}
@@ -652,6 +655,9 @@ func NewBackupDestination(ctx context.Context, cfg *config.Config, ch *clickhous
 		partSize := cfg.S3.PartSize
 		if cfg.S3.PartSize <= 0 {
 			partSize = cfg.General.MaxFileSize / cfg.S3.MaxPartsCount
+			if cfg.General.MaxFileSize % cfg.S3.MaxPartsCount > 0 {
+				partSize++
+			}
 			if partSize < 5*1024*1024 {
 				partSize = 5 * 1024 * 1024
 			}

diff --git a/pkg/storage/s3.go b/pkg/storage/s3.go
@@ -531,8 +531,11 @@ func (s *S3) CopyObject(ctx context.Context, srcBucket, srcKey, dstKey string) (
 
 	// Set the part size (e.g., 5 MB)
 	partSize := srcSize / s.Config.MaxPartsCount
+	if srcSize % s.Config.MaxPartsCount > 0 {
+		partSize++
+	}
 	if partSize < 5*1024*1024 {
-		partSize = 5 * 1014 * 1024
+		partSize = 5 * 1024 * 1024
 	}
 
 	// Calculate the number of parts