[DO NOT MERGE] Feat flex checksum

aws/middleware/user_agent.go

@@ -76,19 +76,28 @@ type UserAgentFeature string
 
 // Enumerates UserAgentFeature.
 const (
-	UserAgentFeatureResourceModel          UserAgentFeature = "A" // n/a (we don't generate separate resource types)
-	UserAgentFeatureWaiter                                  = "B"
-	UserAgentFeaturePaginator                               = "C"
-	UserAgentFeatureRetryModeLegacy                         = "D" // n/a (equivalent to standard)
-	UserAgentFeatureRetryModeStandard                       = "E"
-	UserAgentFeatureRetryModeAdaptive                       = "F"
-	UserAgentFeatureS3Transfer                              = "G"
-	UserAgentFeatureS3CryptoV1N                             = "H" // n/a (crypto client is external)
-	UserAgentFeatureS3CryptoV2                              = "I" // n/a
-	UserAgentFeatureS3ExpressBucket                         = "J"
-	UserAgentFeatureS3AccessGrants                          = "K" // not yet implemented
-	UserAgentFeatureGZIPRequestCompression                  = "L"
-	UserAgentFeatureProtocolRPCV2CBOR                       = "M"
+	UserAgentFeatureResourceModel                 UserAgentFeature = "A" // n/a (we don't generate separate resource types)
+	UserAgentFeatureWaiter                                         = "B"
+	UserAgentFeaturePaginator                                      = "C"
+	UserAgentFeatureRetryModeLegacy                                = "D" // n/a (equivalent to standard)
+	UserAgentFeatureRetryModeStandard                              = "E"
+	UserAgentFeatureRetryModeAdaptive                              = "F"
+	UserAgentFeatureS3Transfer                                     = "G"
+	UserAgentFeatureS3CryptoV1N                                    = "H" // n/a (crypto client is external)
+	UserAgentFeatureS3CryptoV2                                     = "I" // n/a
+	UserAgentFeatureS3ExpressBucket                                = "J"
+	UserAgentFeatureS3AccessGrants                                 = "K" // not yet implemented
+	UserAgentFeatureGZIPRequestCompression                         = "L"
+	UserAgentFeatureProtocolRPCV2CBOR                              = "M"
+	UserAgentFeatureRequestChecksumWhenSupported                   = "Z"
+	UserAgentFeatureRequestChecksumWhenRequired                    = "a"
+	UserAgentFeatureResponseChecksumWhenSupported                  = "b"
+	UserAgentFeatureResponseChecksumWhenRequired                   = "c"
+	UserAgentFeatureRequestChecksumCRC32                           = "U"
+	UserAgentFeatureRequestChecksumCRC32C                          = "V"
+	UserAgentFeatureRequestChecksumCRC64                           = "W"
+	UserAgentFeatureRequestChecksumSHA1                            = "X"
+	UserAgentFeatureRequestChecksumSHA256                          = "Y"
 )
 
 // RequestUserAgent is a build middleware that set the User-Agent for the request.

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java

@@ -23,6 +23,7 @@
 import java.util.TreeSet;
 import software.amazon.smithy.aws.go.codegen.customization.AwsCustomGoDependency;
 import software.amazon.smithy.aws.go.codegen.customization.PresignURLAutoFill;
+import software.amazon.smithy.aws.traits.HttpChecksumTrait;
 import software.amazon.smithy.aws.traits.ServiceTrait;
 import software.amazon.smithy.aws.traits.protocols.AwsQueryTrait;
 import software.amazon.smithy.aws.traits.protocols.Ec2QueryTrait;
@@ -67,7 +68,7 @@ public class AwsHttpPresignURLClientGenerator implements GoIntegration {
     private static final String CONVERT_TO_PRESIGN_MIDDLEWARE_NAME = "convertToPresignMiddleware";
     private static final String CONVERT_TO_PRESIGN_TYPE_NAME = "presignConverter";
     private static final String NOP_HTTP_CLIENT_OPTION_FUNC_NAME = "withNopHTTPClientAPIOption";
-
+    private static final String NO_DEFAULT_CHECKSUM_OPTION_FUNC_NAME = "withNoDefaultChecksumAPIOption";
     private static final String PRESIGN_CLIENT = "PresignClient";
     private static final Symbol presignClientSymbol = buildSymbol(PRESIGN_CLIENT, true);
 
@@ -218,7 +219,11 @@ public void writeAdditionalFiles(
             writeConvertToPresignMiddleware(writer, model, symbolProvider, serviceShape);
         });
 
+        boolean supportsComputeInputChecksumsWorkflow = false;
         for (OperationShape operationShape : TopDownIndex.of(model).getContainedOperations(serviceShape)) {
+            if (hasInputChecksumTrait(operationShape)) {
+                supportsComputeInputChecksumsWorkflow = true;
+            }
             if (!validOperations.contains(operationShape.getId())) {
                 continue;
             }
@@ -231,6 +236,10 @@ public void writeAdditionalFiles(
                 writeS3AddAsUnsignedPayloadHelper(writer, model, symbolProvider, serviceShape, operationShape);
             });
         }
+
+        if (supportsComputeInputChecksumsWorkflow) {
+            writePresignRequestChecksumConfigHelpers(settings, goDelegator);
+        }
     }
 
     private void writePresignOperationFunction(
@@ -263,6 +272,10 @@ private void writePresignOperationFunction(
 
                     writer.write("clientOptFns := append(options.ClientOptions, $L)", NOP_HTTP_CLIENT_OPTION_FUNC_NAME);
                     writer.write("");
+                    if (hasInputChecksumTrait(operationShape)) {
+                        writer.write("clientOptFns = append(options.ClientOptions, $L)", NO_DEFAULT_CHECKSUM_OPTION_FUNC_NAME);
+                        writer.write("");
+                    }
 
                     writer.openBlock("result, _, err := c.client.invokeOperation(ctx, $S, params, clientOptFns,", ")",
                             operationSymbol.getName(), () -> {
@@ -572,6 +585,29 @@ private void writePresignClientHelpers(
         writer.write("");
     }
 
+    private void writePresignRequestChecksumConfigHelpers(
+            GoSettings settings,
+            GoDelegator goDelegator
+    ) {
+        goDelegator.useFileWriter("api_client.go", settings.getModuleName(), goTemplate("""
+            func $fn:L(options *Options) {
+                options.RequestChecksumCalculation = $requestChecksumCalculationWhenRequired:T
+            }""",
+                Map.of(
+                        "fn", NO_DEFAULT_CHECKSUM_OPTION_FUNC_NAME,
+                        "requestChecksumCalculationWhenRequired",
+                        AwsGoDependency.AWS_CORE.valueSymbol("RequestChecksumCalculationWhenRequired")
+                    )));
+    }
+
+    private static boolean hasInputChecksumTrait(OperationShape operation) {
+        if (!operation.hasTrait(HttpChecksumTrait.class)) {
+            return false;
+        }
+        HttpChecksumTrait trait = operation.expectTrait(HttpChecksumTrait.class);
+        return trait.isRequestChecksumRequired() || trait.getRequestAlgorithmMember().isPresent();
+    }
+
     /**
      * Writes the presigner interface used by the presign url client
      */

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/ChecksumMetricsTracking.java

@@ -0,0 +1,139 @@
+package software.amazon.smithy.aws.go.codegen.customization;
+
+import software.amazon.smithy.aws.go.codegen.AwsGoDependency;
+import software.amazon.smithy.aws.traits.HttpChecksumTrait;
+import software.amazon.smithy.codegen.core.SymbolProvider;
+import software.amazon.smithy.go.codegen.GoDelegator;
+import software.amazon.smithy.go.codegen.GoSettings;
+import software.amazon.smithy.go.codegen.SmithyGoDependency;
+import software.amazon.smithy.go.codegen.integration.GoIntegration;
+import software.amazon.smithy.go.codegen.integration.MiddlewareRegistrar;
+import software.amazon.smithy.go.codegen.integration.RuntimeClientPlugin;
+import software.amazon.smithy.model.Model;
+import software.amazon.smithy.model.knowledge.TopDownIndex;
+import software.amazon.smithy.model.shapes.OperationShape;
+import software.amazon.smithy.model.shapes.ServiceShape;
+
+import java.util.List;
+import java.util.Map;
+
+import static software.amazon.smithy.go.codegen.GoWriter.goTemplate;
+import static software.amazon.smithy.go.codegen.SymbolUtils.buildPackageSymbol;
+
+public class ChecksumMetricsTracking implements GoIntegration {
+    private static final MiddlewareRegistrar RequestMIDDLEWARE = MiddlewareRegistrar.builder()
+            .resolvedFunction(buildPackageSymbol("addRequestChecksumMetricsTracking"))
+            .useClientOptions()
+            .build();
+    private static final MiddlewareRegistrar ResponseMIDDLEWARE = MiddlewareRegistrar.builder()
+            .resolvedFunction(buildPackageSymbol("addResponseChecksumMetricsTracking"))
+            .useClientOptions()
+            .build();
+
+    @Override
+    public List<RuntimeClientPlugin> getClientPlugins() {
+        return List.of(
+                RuntimeClientPlugin.builder()
+                        .operationPredicate((m, s, o) -> {
+                            if (!hasInputChecksumTrait(m, s, o)) {
+                                return false;
+                            }
+                        return true;
+                        })
+                        .registerMiddleware(RequestMIDDLEWARE)
+                        .build(),
+                RuntimeClientPlugin.builder()
+                        .operationPredicate((m, s, o) -> {
+                            if (!hasOutputChecksumTrait(m, s, o)) {
+                                return false;
+                            }
+                            return true;
+                        })
+                        .registerMiddleware(ResponseMIDDLEWARE)
+                        .build()
+        );
+    }
+
+    private static boolean hasChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
+                            return operation.hasTrait(HttpChecksumTrait.class);
+    }
+
+    private static boolean hasInputChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
+        if (!hasChecksumTrait(model, service, operation)) {
+            return false;
+        }
+        HttpChecksumTrait trait = operation.expectTrait(HttpChecksumTrait.class);
+        return trait.isRequestChecksumRequired() || trait.getRequestAlgorithmMember().isPresent();
+    }
+
+    private static boolean hasOutputChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
+        if (!hasChecksumTrait(model, service, operation)) {
+            return false;
+        }
+        HttpChecksumTrait trait = operation.expectTrait(HttpChecksumTrait.class);
+        return trait.getRequestValidationModeMember().isPresent() && !trait.getResponseAlgorithms().isEmpty();
+    }
+
+    @Override
+    public void writeAdditionalFiles(GoSettings settings, Model model, SymbolProvider symbolProvider, GoDelegator goDelegator) {
+        ServiceShape service = settings.getService(model);
+        boolean supportsComputeInputChecksumsWorkflow = false;
+        boolean supportsChecksumValidationWorkflow = false;
+
+        for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) {
+            if (!hasChecksumTrait(model, service, operation)) {
+                continue;
+            }
+
+            if (hasInputChecksumTrait(model, service, operation)) {
+                supportsComputeInputChecksumsWorkflow = true;
+            }
+
+            if (hasOutputChecksumTrait(model, service, operation)) {
+                supportsChecksumValidationWorkflow = true;
+            }
+        }
+
+        if (supportsComputeInputChecksumsWorkflow) {
+            goDelegator.useFileWriter("api_client.go", settings.getModuleName(), goTemplate("""
+                func addRequestChecksumMetricsTracking(stack $stack:P, options Options) error {
+                    ua, err := getOrAddRequestUserAgent(stack)
+                    if err != nil {
+                        return err
+                    }
+
+                    return stack.Build.Insert(&$requestMetricsTracking:P{
+                        RequestChecksumCalculation: options.RequestChecksumCalculation,
+                        UserAgent: ua,
+                    }, "UserAgent", $before:T)
+                }""",
+                    Map.of(
+                            "stack", SmithyGoDependency.SMITHY_MIDDLEWARE.struct("Stack"),
+                            "requestMetricsTracking", AwsGoDependency.SERVICE_INTERNAL_CHECKSUM.valueSymbol("RequestChecksumMetricsTracking"),
+                            "before", SmithyGoDependency.SMITHY_MIDDLEWARE
+                                    .valueSymbol("Before")
+                    )));
+        }
+
+        if (supportsChecksumValidationWorkflow) {
+            goDelegator.useFileWriter("api_client.go", settings.getModuleName(), goTemplate("""
+                func addResponseChecksumMetricsTracking(stack $stack:P, options Options) error {
+                    ua, err := getOrAddRequestUserAgent(stack)
+                    if err != nil {
+                        return err
+                    }
+
+                    return stack.Build.Insert(&$responseMetricsTracking:P{
+                        ResponseChecksumValidation: options.ResponseChecksumValidation,
+                        UserAgent: ua,
+                    }, "UserAgent", $before:T)
+                }""",
+                    Map.of(
+                            "stack", SmithyGoDependency.SMITHY_MIDDLEWARE.struct("Stack"),
+                            "responseMetricsTracking", AwsGoDependency.SERVICE_INTERNAL_CHECKSUM.valueSymbol("ResponseChecksumMetricsTracking"),
+                            "before", SmithyGoDependency.SMITHY_MIDDLEWARE
+                                    .valueSymbol("Before")
+                    )));
+        }
+    }
+}

codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration

@@ -88,4 +88,4 @@ software.amazon.smithy.aws.go.codegen.customization.s3.ExpressUserAgent
 software.amazon.smithy.aws.go.codegen.customization.BackfillRequiredTrait
 software.amazon.smithy.aws.go.codegen.customization.DeprecateService
 software.amazon.smithy.aws.go.codegen.customization.BasicUserAgentFeatures
-
+software.amazon.smithy.aws.go.codegen.customization.ChecksumMetricsTracking

feature/s3/manager/integ_upload_test.go

@@ -23,7 +23,6 @@ import (
 	"testing"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
-	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
 	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
@@ -223,6 +222,28 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
 			},
 		},
 		"auto multipart part": {
+			"no checksum": {
+				payload: bytes.NewReader(multiPartBytes),
+				expectParts: []s3types.CompletedPart{
+					{
+						ChecksumCRC32: aws.String(singlePartCRC32),
+						ETag:          aws.String(singlePartETag),
+						PartNumber:    aws.Int32(1),
+					},
+					{
+						ChecksumCRC32: aws.String(singlePartCRC32),
+						ETag:          aws.String(singlePartETag),
+						PartNumber:    aws.Int32(2),
+					},
+					{
+						ChecksumCRC32: aws.String(multiPartTailCRC32),
+						ETag:          aws.String(multiPartTailETag),
+						PartNumber:    aws.Int32(3),
+					},
+				},
+				expectChecksumCRC32: multiPartCRC32,
+				expectETag:          multiPartETag,
+			},
 			"CRC32": {
 				algorithm: s3types.ChecksumAlgorithmCrc32,
 				payload:   bytes.NewReader(multiPartBytes),
@@ -468,21 +489,21 @@ func toStringPtr(v string) *string {
 	return &v
 }
 
-type invalidateHash struct{}
+type failedMultipartUpload struct{}
 
-func (b *invalidateHash) ID() string {
-	return "s3manager:InvalidateHash"
+func (m *failedMultipartUpload) ID() string {
+	return "s3manager:FailedMultipartUpload"
 }
 
-func (b *invalidateHash) RegisterMiddleware(stack *middleware.Stack) error {
-	return stack.Serialize.Add(b, middleware.After)
+func (m *failedMultipartUpload) RegisterMiddleware(stack *middleware.Stack) error {
+	return stack.Serialize.Add(m, middleware.After)
 }
 
-func (b *invalidateHash) HandleSerialize(ctx context.Context, in middleware.SerializeInput, next middleware.SerializeHandler) (
+func (m *failedMultipartUpload) HandleSerialize(ctx context.Context, in middleware.SerializeInput, next middleware.SerializeHandler) (
 	out middleware.SerializeOutput, metadata middleware.Metadata, err error,
 ) {
 	if input, ok := in.Parameters.(*s3.UploadPartInput); ok && aws.ToInt32(input.PartNumber) == 2 {
-		ctx = v4.SetPayloadHash(ctx, "000")
+		return out, metadata, fmt.Errorf("multipart upload error")
 	}
 
 	return next.HandleSerialize(ctx, in)
@@ -493,7 +514,7 @@ func TestInteg_UploadFailCleanup(t *testing.T) {
 	mgr := manager.NewUploader(client, func(u *manager.Uploader) {
 		u.LeavePartsOnError = false
 		u.ClientOptions = append(u.ClientOptions, func(options *s3.Options) {
-			options.APIOptions = append(options.APIOptions, (&invalidateHash{}).RegisterMiddleware)
+			options.APIOptions = append(options.APIOptions, (&failedMultipartUpload{}).RegisterMiddleware)
 		})
 	})
 	_, err := mgr.Upload(context.Background(), &s3.PutObjectInput{

feature/s3/manager/upload.go

@@ -584,6 +584,8 @@ func (a completedParts) Less(i, j int) bool {
 // upload will perform a multipart upload using the firstBuf buffer containing
 // the first chunk of data.
 func (u *multiuploader) upload(firstBuf io.ReadSeeker, cleanup func()) (*UploadOutput, error) {
+	u.initChecksumAlgorithm()
+
 	var params s3.CreateMultipartUploadInput
 	awsutil.Copy(&params, u.in)
 
@@ -752,6 +754,25 @@ func (u *multiuploader) send(c chunk) error {
 	return nil
 }
 
+func (u *multiuploader) initChecksumAlgorithm() {
+	if u.in.ChecksumAlgorithm != "" {
+		return
+	}
+
+	switch {
+	case u.in.ChecksumCRC32 != nil:
+		u.in.ChecksumAlgorithm = types.ChecksumAlgorithmCrc32
+	case u.in.ChecksumCRC32C != nil:
+		u.in.ChecksumAlgorithm = types.ChecksumAlgorithmCrc32c
+	case u.in.ChecksumSHA1 != nil:
+		u.in.ChecksumAlgorithm = types.ChecksumAlgorithmSha1
+	case u.in.ChecksumSHA256 != nil:
+		u.in.ChecksumAlgorithm = types.ChecksumAlgorithmSha256
+	default:
+		u.in.ChecksumAlgorithm = types.ChecksumAlgorithmCrc32
+	}
+}
+
 // geterr is a thread-safe getter for the error object
 func (u *multiuploader) geterr() error {
 	u.m.Lock()