diff --git a/.changelog/9ebe24c4791541e0840da49eab6f9d97.json b/.changelog/9ebe24c4791541e0840da49eab6f9d97.json new file mode 100644 index 00000000000..f733e75674d --- /dev/null +++ b/.changelog/9ebe24c4791541e0840da49eab6f9d97.json @@ -0,0 +1,11 @@ +{ + "id": "9ebe24c4-7915-41e0-840d-a49eab6f9d97", + "type": "feature", + "description": "Enable HTTP checksums in supported services by default. New config fields, RequestChecksumCalculation and ResponseChecksumValidation, allow the caller to opt-out of this new default behavior. This feature also replaces the default MD5 checksum with CRC32.", + "modules": [ + ".", + "config", + "service/internal/checksum", + "service/s3" + ] +} \ No newline at end of file diff --git a/aws/checksum.go b/aws/checksum.go new file mode 100644 index 00000000000..4152caade10 --- /dev/null +++ b/aws/checksum.go @@ -0,0 +1,33 @@ +package aws + +// RequestChecksumCalculation controls request checksum calculation workflow +type RequestChecksumCalculation int + +const ( + // RequestChecksumCalculationUnset is the unset value for RequestChecksumCalculation + RequestChecksumCalculationUnset RequestChecksumCalculation = iota + + // RequestChecksumCalculationWhenSupported indicates request checksum will be calculated + // if the operation supports input checksums + RequestChecksumCalculationWhenSupported + + // RequestChecksumCalculationWhenRequired indicates request checksum will be calculated + // if required by the operation or if user elects to set a checksum algorithm in request + RequestChecksumCalculationWhenRequired +) + +// ResponseChecksumValidation controls response checksum validation workflow +type ResponseChecksumValidation int + +const ( + // ResponseChecksumValidationUnset is the unset value for ResponseChecksumValidation + ResponseChecksumValidationUnset ResponseChecksumValidation = iota + + // ResponseChecksumValidationWhenSupported indicates response checksum will be validated + // if the operation supports output checksums + ResponseChecksumValidationWhenSupported + + // ResponseChecksumValidationWhenRequired indicates response checksum will only + // be validated if the operation requires output checksum validation + ResponseChecksumValidationWhenRequired +) diff --git a/aws/config.go b/aws/config.go index 16000d79279..a015cc5b20c 100644 --- a/aws/config.go +++ b/aws/config.go @@ -165,6 +165,33 @@ type Config struct { // Controls how a resolved AWS account ID is handled for endpoint routing. AccountIDEndpointMode AccountIDEndpointMode + + // RequestChecksumCalculation determines when request checksum calculation is performed. + // + // There are two possible values for this setting: + // + // 1. RequestChecksumCalculationWhenSupported (default): The checksum is always calculated + // if the operation supports it, regardless of whether the user sets an algorithm in the request. + // + // 2. RequestChecksumCalculationWhenRequired: The checksum is only calculated if the user + // explicitly sets a checksum algorithm in the request. + // + // This setting is sourced from the environment variable AWS_REQUEST_CHECKSUM_CALCULATION + // or the shared config profile attribute "request_checksum_calculation". + RequestChecksumCalculation RequestChecksumCalculation + + // ResponseChecksumValidation determines when response checksum validation is performed + // + // There are two possible values for this setting: + // + // 1. ResponseChecksumValidationWhenSupported (default): The checksum is always validated + // if the operation supports it, regardless of whether the user sets the validation mode to ENABLED in request. + // + // 2. ResponseChecksumValidationWhenRequired: The checksum is only validated if the user + // explicitly sets the validation mode to ENABLED in the request + // This variable is sourced from environment variable AWS_RESPONSE_CHECKSUM_VALIDATION or + // the shared config profile attribute "response_checksum_validation". + ResponseChecksumValidation ResponseChecksumValidation } // NewConfig returns a new Config pointer that can be chained with builder diff --git a/aws/middleware/user_agent.go b/aws/middleware/user_agent.go index ab4e619073a..01d758d5ff8 100644 --- a/aws/middleware/user_agent.go +++ b/aws/middleware/user_agent.go @@ -76,19 +76,28 @@ type UserAgentFeature string // Enumerates UserAgentFeature. const ( - UserAgentFeatureResourceModel UserAgentFeature = "A" // n/a (we don't generate separate resource types) - UserAgentFeatureWaiter = "B" - UserAgentFeaturePaginator = "C" - UserAgentFeatureRetryModeLegacy = "D" // n/a (equivalent to standard) - UserAgentFeatureRetryModeStandard = "E" - UserAgentFeatureRetryModeAdaptive = "F" - UserAgentFeatureS3Transfer = "G" - UserAgentFeatureS3CryptoV1N = "H" // n/a (crypto client is external) - UserAgentFeatureS3CryptoV2 = "I" // n/a - UserAgentFeatureS3ExpressBucket = "J" - UserAgentFeatureS3AccessGrants = "K" // not yet implemented - UserAgentFeatureGZIPRequestCompression = "L" - UserAgentFeatureProtocolRPCV2CBOR = "M" + UserAgentFeatureResourceModel UserAgentFeature = "A" // n/a (we don't generate separate resource types) + UserAgentFeatureWaiter = "B" + UserAgentFeaturePaginator = "C" + UserAgentFeatureRetryModeLegacy = "D" // n/a (equivalent to standard) + UserAgentFeatureRetryModeStandard = "E" + UserAgentFeatureRetryModeAdaptive = "F" + UserAgentFeatureS3Transfer = "G" + UserAgentFeatureS3CryptoV1N = "H" // n/a (crypto client is external) + UserAgentFeatureS3CryptoV2 = "I" // n/a + UserAgentFeatureS3ExpressBucket = "J" + UserAgentFeatureS3AccessGrants = "K" // not yet implemented + UserAgentFeatureGZIPRequestCompression = "L" + UserAgentFeatureProtocolRPCV2CBOR = "M" + UserAgentFeatureRequestChecksumCRC32 = "U" + UserAgentFeatureRequestChecksumCRC32C = "V" + UserAgentFeatureRequestChecksumCRC64 = "W" + UserAgentFeatureRequestChecksumSHA1 = "X" + UserAgentFeatureRequestChecksumSHA256 = "Y" + UserAgentFeatureRequestChecksumWhenSupported = "Z" + UserAgentFeatureRequestChecksumWhenRequired = "a" + UserAgentFeatureResponseChecksumWhenSupported = "b" + UserAgentFeatureResponseChecksumWhenRequired = "c" ) // RequestUserAgent is a build middleware that set the User-Agent for the request. diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AddAwsConfigFields.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AddAwsConfigFields.java index b73fcaa2c46..dd52460d517 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AddAwsConfigFields.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AddAwsConfigFields.java @@ -84,6 +84,10 @@ public class AddAwsConfigFields implements GoIntegration { private static final String SDK_ACCOUNTID_ENDPOINT_MODE = "AccountIDEndpointMode"; + private static final String REQUEST_CHECKSUM_CALCULATION = "RequestChecksumCalculation"; + + private static final String RESPONSE_CHECKSUM_VALIDATION = "ResponseChecksumValidation"; + private static final List AWS_CONFIG_FIELDS = ListUtils.of( AwsConfigField.builder() .name(REGION_CONFIG_NAME) @@ -244,6 +248,18 @@ public class AddAwsConfigFields implements GoIntegration { .type(SdkGoTypes.Aws.AccountIDEndpointMode) .documentation("Indicates how aws account ID is applied in endpoint2.0 routing") .servicePredicate(AccountIDEndpointRouting::hasAccountIdEndpoints) + .build(), + AwsConfigField.builder() + .name(REQUEST_CHECKSUM_CALCULATION) + .type(SdkGoTypes.Aws.RequestChecksumCalculation) + .documentation("Indicates how user opt-in/out request checksum calculation") + .servicePredicate(AwsHttpChecksumGenerator::hasInputChecksumTrait) + .build(), + AwsConfigField.builder() + .name(RESPONSE_CHECKSUM_VALIDATION) + .type(SdkGoTypes.Aws.ResponseChecksumValidation) + .documentation("Indicates how user opt-in/out response checksum validation") + .servicePredicate(AwsHttpChecksumGenerator::hasOutputChecksumTrait) .build() ); diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpChecksumGenerator.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpChecksumGenerator.java index a23b71cdbc6..f21c39e665f 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpChecksumGenerator.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpChecksumGenerator.java @@ -22,6 +22,7 @@ import software.amazon.smithy.go.codegen.integration.MiddlewareRegistrar; import software.amazon.smithy.go.codegen.integration.RuntimeClientPlugin; import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.knowledge.TopDownIndex; import software.amazon.smithy.model.shapes.MemberShape; import software.amazon.smithy.model.shapes.OperationShape; import software.amazon.smithy.model.shapes.ServiceShape; @@ -73,9 +74,7 @@ public byte getOrder() { @Override public void processFinalizedModel(GoSettings settings, Model model) { ServiceShape service = settings.getService(model); - for (ShapeId operationId : service.getAllOperations()) { - final OperationShape operation = model.expectShape(operationId, OperationShape.class); - + for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) { // Create a symbol provider because one is not available in this call. SymbolProvider symbolProvider = GoCodegenPlugin.createSymbolProvider(model, settings); @@ -128,8 +127,7 @@ public void writeAdditionalFiles( boolean supportsComputeInputChecksumsWorkflow = false; boolean supportsChecksumValidationWorkflow = false; - for (ShapeId operationID : service.getAllOperations()) { - OperationShape operation = model.expectShape(operationID, OperationShape.class); + for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) { if (!hasChecksumTrait(model, service, operation)) { continue; } @@ -178,11 +176,11 @@ public List getClientPlugins() { } // return true if operation shape is decorated with `httpChecksum` trait. - private boolean hasChecksumTrait(Model model, ServiceShape service, OperationShape operation) { + private static boolean hasChecksumTrait(Model model, ServiceShape service, OperationShape operation) { return operation.hasTrait(HttpChecksumTrait.class); } - private boolean hasInputChecksumTrait(Model model, ServiceShape service, OperationShape operation) { + private static boolean hasInputChecksumTrait(Model model, ServiceShape service, OperationShape operation) { if (!hasChecksumTrait(model, service, operation)) { return false; } @@ -190,7 +188,16 @@ private boolean hasInputChecksumTrait(Model model, ServiceShape service, Operati return trait.isRequestChecksumRequired() || trait.getRequestAlgorithmMember().isPresent(); } - private boolean hasOutputChecksumTrait(Model model, ServiceShape service, OperationShape operation) { + public static boolean hasInputChecksumTrait(Model model, ServiceShape service) { + for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) { + if (hasInputChecksumTrait(model, service, operation)) { + return true; + } + } + return false; + } + + private static boolean hasOutputChecksumTrait(Model model, ServiceShape service, OperationShape operation) { if (!hasChecksumTrait(model, service, operation)) { return false; } @@ -198,6 +205,15 @@ private boolean hasOutputChecksumTrait(Model model, ServiceShape service, Operat return trait.getRequestValidationModeMember().isPresent() && !trait.getResponseAlgorithms().isEmpty(); } + public static boolean hasOutputChecksumTrait(Model model, ServiceShape service) { + for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) { + if (hasOutputChecksumTrait(model, service, operation)) { + return true; + } + } + return false; + } + private boolean isS3ServiceShape(Model model, ServiceShape service) { String serviceId = service.expectTrait(ServiceTrait.class).getSdkId(); return serviceId.equalsIgnoreCase("S3"); @@ -244,6 +260,7 @@ private void writeInputMiddlewareHelper( return $T(stack, $T{ GetAlgorithm: $L, RequireChecksum: $L, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: $L, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: $L, @@ -284,6 +301,7 @@ private void writeOutputMiddlewareHelper( writer.write(""" return $T(stack, $T{ GetValidationMode: $L, + ResponseChecksumValidation: options.ResponseChecksumValidation, ValidationAlgorithms: $L, IgnoreMultipartValidation: $L, LogValidationSkipped: true, @@ -293,7 +311,6 @@ private void writeOutputMiddlewareHelper( AwsGoDependency.SERVICE_INTERNAL_CHECKSUM).build(), SymbolUtils.createValueSymbolBuilder("OutputMiddlewareOptions", AwsGoDependency.SERVICE_INTERNAL_CHECKSUM).build(), - getRequestValidationModeAccessorFuncName(operationName), convertToGoStringList(responseAlgorithms), ignoreMultipartChecksumValidationMap.getOrDefault( diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java index c608d6b37dd..4d58e78cb3a 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java @@ -23,6 +23,7 @@ import java.util.TreeSet; import software.amazon.smithy.aws.go.codegen.customization.AwsCustomGoDependency; import software.amazon.smithy.aws.go.codegen.customization.PresignURLAutoFill; +import software.amazon.smithy.aws.traits.HttpChecksumTrait; import software.amazon.smithy.aws.traits.ServiceTrait; import software.amazon.smithy.aws.traits.protocols.AwsQueryTrait; import software.amazon.smithy.aws.traits.protocols.Ec2QueryTrait; @@ -67,7 +68,7 @@ public class AwsHttpPresignURLClientGenerator implements GoIntegration { private static final String CONVERT_TO_PRESIGN_MIDDLEWARE_NAME = "convertToPresignMiddleware"; private static final String CONVERT_TO_PRESIGN_TYPE_NAME = "presignConverter"; private static final String NOP_HTTP_CLIENT_OPTION_FUNC_NAME = "withNopHTTPClientAPIOption"; - + private static final String NO_DEFAULT_CHECKSUM_OPTION_FUNC_NAME = "withNoDefaultChecksumAPIOption"; private static final String PRESIGN_CLIENT = "PresignClient"; private static final Symbol presignClientSymbol = buildSymbol(PRESIGN_CLIENT, true); @@ -218,7 +219,11 @@ public void writeAdditionalFiles( writeConvertToPresignMiddleware(writer, model, symbolProvider, serviceShape); }); + boolean supportsComputeInputChecksumsWorkflow = false; for (OperationShape operationShape : TopDownIndex.of(model).getContainedOperations(serviceShape)) { + if (hasInputChecksumTrait(operationShape)) { + supportsComputeInputChecksumsWorkflow = true; + } if (!validOperations.contains(operationShape.getId())) { continue; } @@ -231,6 +236,10 @@ public void writeAdditionalFiles( writeS3AddAsUnsignedPayloadHelper(writer, model, symbolProvider, serviceShape, operationShape); }); } + + if (supportsComputeInputChecksumsWorkflow) { + writePresignRequestChecksumConfigHelpers(settings, goDelegator); + } } private void writePresignOperationFunction( @@ -263,6 +272,10 @@ private void writePresignOperationFunction( writer.write("clientOptFns := append(options.ClientOptions, $L)", NOP_HTTP_CLIENT_OPTION_FUNC_NAME); writer.write(""); + if (hasInputChecksumTrait(operationShape)) { + writer.write("clientOptFns = append(options.ClientOptions, $L)", NO_DEFAULT_CHECKSUM_OPTION_FUNC_NAME); + writer.write(""); + } writer.openBlock("result, _, err := c.client.invokeOperation(ctx, $S, params, clientOptFns,", ")", operationSymbol.getName(), () -> { @@ -572,6 +585,29 @@ private void writePresignClientHelpers( writer.write(""); } + private void writePresignRequestChecksumConfigHelpers( + GoSettings settings, + GoDelegator goDelegator + ) { + goDelegator.useFileWriter("api_client.go", settings.getModuleName(), goTemplate(""" + func $fn:L(options *Options) { + options.RequestChecksumCalculation = $requestChecksumCalculationWhenRequired:T + }""", + Map.of( + "fn", NO_DEFAULT_CHECKSUM_OPTION_FUNC_NAME, + "requestChecksumCalculationWhenRequired", + AwsGoDependency.AWS_CORE.valueSymbol("RequestChecksumCalculationWhenRequired") + ))); + } + + private static boolean hasInputChecksumTrait(OperationShape operation) { + if (!operation.hasTrait(HttpChecksumTrait.class)) { + return false; + } + HttpChecksumTrait trait = operation.expectTrait(HttpChecksumTrait.class); + return trait.isRequestChecksumRequired() || trait.getRequestAlgorithmMember().isPresent(); + } + /** * Writes the presigner interface used by the presign url client */ diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/SdkGoTypes.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/SdkGoTypes.java index c91a3f4c2aa..e0158ba62af 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/SdkGoTypes.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/SdkGoTypes.java @@ -39,6 +39,8 @@ public static final class Aws { public static final Symbol AccountIDEndpointModeRequired = AwsGoDependency.AWS_CORE.valueSymbol("AccountIDEndpointModeRequired"); public static final Symbol AccountIDEndpointModeDisabled = AwsGoDependency.AWS_CORE.valueSymbol("AccountIDEndpointModeDisabled"); + public static final Symbol RequestChecksumCalculation = AwsGoDependency.AWS_CORE.valueSymbol("RequestChecksumCalculation"); + public static final Symbol ResponseChecksumValidation = AwsGoDependency.AWS_CORE.valueSymbol("ResponseChecksumValidation"); public static final class Middleware { public static final Symbol GetRequiresLegacyEndpoints = AwsGoDependency.AWS_MIDDLEWARE.valueSymbol("GetRequiresLegacyEndpoints"); diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/ChecksumMetricsTracking.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/ChecksumMetricsTracking.java new file mode 100644 index 00000000000..693e1931723 --- /dev/null +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/ChecksumMetricsTracking.java @@ -0,0 +1,139 @@ +package software.amazon.smithy.aws.go.codegen.customization; + +import software.amazon.smithy.aws.go.codegen.AwsGoDependency; +import software.amazon.smithy.aws.traits.HttpChecksumTrait; +import software.amazon.smithy.codegen.core.SymbolProvider; +import software.amazon.smithy.go.codegen.GoDelegator; +import software.amazon.smithy.go.codegen.GoSettings; +import software.amazon.smithy.go.codegen.SmithyGoDependency; +import software.amazon.smithy.go.codegen.integration.GoIntegration; +import software.amazon.smithy.go.codegen.integration.MiddlewareRegistrar; +import software.amazon.smithy.go.codegen.integration.RuntimeClientPlugin; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.knowledge.TopDownIndex; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ServiceShape; + +import java.util.List; +import java.util.Map; + +import static software.amazon.smithy.go.codegen.GoWriter.goTemplate; +import static software.amazon.smithy.go.codegen.SymbolUtils.buildPackageSymbol; + +public class ChecksumMetricsTracking implements GoIntegration { + private static final MiddlewareRegistrar RequestMIDDLEWARE = MiddlewareRegistrar.builder() + .resolvedFunction(buildPackageSymbol("addRequestChecksumMetricsTracking")) + .useClientOptions() + .build(); + private static final MiddlewareRegistrar ResponseMIDDLEWARE = MiddlewareRegistrar.builder() + .resolvedFunction(buildPackageSymbol("addResponseChecksumMetricsTracking")) + .useClientOptions() + .build(); + + @Override + public List getClientPlugins() { + return List.of( + RuntimeClientPlugin.builder() + .operationPredicate((m, s, o) -> { + if (!hasInputChecksumTrait(m, s, o)) { + return false; + } + return true; + }) + .registerMiddleware(RequestMIDDLEWARE) + .build(), + RuntimeClientPlugin.builder() + .operationPredicate((m, s, o) -> { + if (!hasOutputChecksumTrait(m, s, o)) { + return false; + } + return true; + }) + .registerMiddleware(ResponseMIDDLEWARE) + .build() + ); + } + + private static boolean hasChecksumTrait(Model model, ServiceShape service, OperationShape operation) { + return operation.hasTrait(HttpChecksumTrait.class); + } + + private static boolean hasInputChecksumTrait(Model model, ServiceShape service, OperationShape operation) { + if (!hasChecksumTrait(model, service, operation)) { + return false; + } + HttpChecksumTrait trait = operation.expectTrait(HttpChecksumTrait.class); + return trait.isRequestChecksumRequired() || trait.getRequestAlgorithmMember().isPresent(); + } + + private static boolean hasOutputChecksumTrait(Model model, ServiceShape service, OperationShape operation) { + if (!hasChecksumTrait(model, service, operation)) { + return false; + } + HttpChecksumTrait trait = operation.expectTrait(HttpChecksumTrait.class); + return trait.getRequestValidationModeMember().isPresent() && !trait.getResponseAlgorithms().isEmpty(); + } + + @Override + public void writeAdditionalFiles(GoSettings settings, Model model, SymbolProvider symbolProvider, GoDelegator goDelegator) { + ServiceShape service = settings.getService(model); + boolean supportsComputeInputChecksumsWorkflow = false; + boolean supportsChecksumValidationWorkflow = false; + + for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) { + if (!hasChecksumTrait(model, service, operation)) { + continue; + } + + if (hasInputChecksumTrait(model, service, operation)) { + supportsComputeInputChecksumsWorkflow = true; + } + + if (hasOutputChecksumTrait(model, service, operation)) { + supportsChecksumValidationWorkflow = true; + } + } + + if (supportsComputeInputChecksumsWorkflow) { + goDelegator.useFileWriter("api_client.go", settings.getModuleName(), goTemplate(""" + func addRequestChecksumMetricsTracking(stack $stack:P, options Options) error { + ua, err := getOrAddRequestUserAgent(stack) + if err != nil { + return err + } + + return stack.Build.Insert(&$requestMetricsTracking:P{ + RequestChecksumCalculation: options.RequestChecksumCalculation, + UserAgent: ua, + }, "UserAgent", $before:T) + }""", + Map.of( + "stack", SmithyGoDependency.SMITHY_MIDDLEWARE.struct("Stack"), + "requestMetricsTracking", AwsGoDependency.SERVICE_INTERNAL_CHECKSUM.valueSymbol("RequestChecksumMetricsTracking"), + "before", SmithyGoDependency.SMITHY_MIDDLEWARE + .valueSymbol("Before") + ))); + } + + if (supportsChecksumValidationWorkflow) { + goDelegator.useFileWriter("api_client.go", settings.getModuleName(), goTemplate(""" + func addResponseChecksumMetricsTracking(stack $stack:P, options Options) error { + ua, err := getOrAddRequestUserAgent(stack) + if err != nil { + return err + } + + return stack.Build.Insert(&$responseMetricsTracking:P{ + ResponseChecksumValidation: options.ResponseChecksumValidation, + UserAgent: ua, + }, "UserAgent", $before:T) + }""", + Map.of( + "stack", SmithyGoDependency.SMITHY_MIDDLEWARE.struct("Stack"), + "responseMetricsTracking", AwsGoDependency.SERVICE_INTERNAL_CHECKSUM.valueSymbol("ResponseChecksumMetricsTracking"), + "before", SmithyGoDependency.SMITHY_MIDDLEWARE + .valueSymbol("Before") + ))); + } + } +} diff --git a/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration b/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration index 14a12d710aa..bc78b9d68d7 100644 --- a/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration +++ b/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration @@ -88,4 +88,4 @@ software.amazon.smithy.aws.go.codegen.customization.s3.ExpressUserAgent software.amazon.smithy.aws.go.codegen.customization.BackfillRequiredTrait software.amazon.smithy.aws.go.codegen.customization.DeprecateService software.amazon.smithy.aws.go.codegen.customization.BasicUserAgentFeatures - +software.amazon.smithy.aws.go.codegen.customization.ChecksumMetricsTracking diff --git a/config/config.go b/config/config.go index d5226cb0437..0577c61869e 100644 --- a/config/config.go +++ b/config/config.go @@ -83,6 +83,12 @@ var defaultAWSConfigResolvers = []awsConfigResolver{ // Sets the AccountIDEndpointMode if present in env var or shared config profile resolveAccountIDEndpointMode, + + // Sets the RequestChecksumCalculation if present in env var or shared config profile + resolveRequestChecksumCalculation, + + // Sets the ResponseChecksumValidation if present in env var or shared config profile + resolveResponseChecksumValidation, } // A Config represents a generic configuration value or set of values. This type diff --git a/config/env_config.go b/config/env_config.go index 3a06f1412a7..a672850f53e 100644 --- a/config/env_config.go +++ b/config/env_config.go @@ -83,6 +83,9 @@ const ( awsAccountIDEnv = "AWS_ACCOUNT_ID" awsAccountIDEndpointModeEnv = "AWS_ACCOUNT_ID_ENDPOINT_MODE" + + awsRequestChecksumCalculation = "AWS_REQUEST_CHECKSUM_CALCULATION" + awsResponseChecksumValidation = "AWS_RESPONSE_CHECKSUM_VALIDATION" ) var ( @@ -296,6 +299,12 @@ type EnvConfig struct { // Indicates whether account ID will be required/ignored in endpoint2.0 routing AccountIDEndpointMode aws.AccountIDEndpointMode + + // Indicates whether request checksum should be calculated + RequestChecksumCalculation aws.RequestChecksumCalculation + + // Indicates whether response checksum should be validated + ResponseChecksumValidation aws.ResponseChecksumValidation } // loadEnvConfig reads configuration values from the OS's environment variables. @@ -400,6 +409,13 @@ func NewEnvConfig() (EnvConfig, error) { return cfg, err } + if err := setRequestChecksumCalculationFromEnvVal(&cfg.RequestChecksumCalculation, []string{awsRequestChecksumCalculation}); err != nil { + return cfg, err + } + if err := setResponseChecksumValidationFromEnvVal(&cfg.ResponseChecksumValidation, []string{awsResponseChecksumValidation}); err != nil { + return cfg, err + } + return cfg, nil } @@ -432,6 +448,14 @@ func (c EnvConfig) getAccountIDEndpointMode(context.Context) (aws.AccountIDEndpo return c.AccountIDEndpointMode, len(c.AccountIDEndpointMode) > 0, nil } +func (c EnvConfig) getRequestChecksumCalculation(context.Context) (aws.RequestChecksumCalculation, bool, error) { + return c.RequestChecksumCalculation, c.RequestChecksumCalculation > 0, nil +} + +func (c EnvConfig) getResponseChecksumValidation(context.Context) (aws.ResponseChecksumValidation, bool, error) { + return c.ResponseChecksumValidation, c.ResponseChecksumValidation > 0, nil +} + // GetRetryMaxAttempts returns the value of AWS_MAX_ATTEMPTS if was specified, // and not 0. func (c EnvConfig) GetRetryMaxAttempts(ctx context.Context) (int, bool, error) { @@ -528,6 +552,45 @@ func setAIDEndPointModeFromEnvVal(m *aws.AccountIDEndpointMode, keys []string) e return nil } +func setRequestChecksumCalculationFromEnvVal(m *aws.RequestChecksumCalculation, keys []string) error { + for _, k := range keys { + value := os.Getenv(k) + if len(value) == 0 { + continue + } + + switch strings.ToLower(value) { + case checksumWhenSupported: + *m = aws.RequestChecksumCalculationWhenSupported + case checksumWhenRequired: + *m = aws.RequestChecksumCalculationWhenRequired + default: + return fmt.Errorf("invalid value for environment variable, %s=%s, must be when_supported/when_required", k, value) + } + } + return nil +} + +func setResponseChecksumValidationFromEnvVal(m *aws.ResponseChecksumValidation, keys []string) error { + for _, k := range keys { + value := os.Getenv(k) + if len(value) == 0 { + continue + } + + switch strings.ToLower(value) { + case checksumWhenSupported: + *m = aws.ResponseChecksumValidationWhenSupported + case checksumWhenRequired: + *m = aws.ResponseChecksumValidationWhenRequired + default: + return fmt.Errorf("invalid value for environment variable, %s=%s, must be when_supported/when_required", k, value) + } + + } + return nil +} + // GetRegion returns the AWS Region if set in the environment. Returns an empty // string if not set. func (c EnvConfig) getRegion(ctx context.Context) (string, bool, error) { diff --git a/config/env_config_test.go b/config/env_config_test.go index 02c00d37aa7..870c46509bc 100644 --- a/config/env_config_test.go +++ b/config/env_config_test.go @@ -514,7 +514,6 @@ func TestNewEnvConfig(t *testing.T) { Config: EnvConfig{ AccountIDEndpointMode: aws.AccountIDEndpointModeRequired, }, - WantErr: false, }, 47: { Env: map[string]string{ @@ -523,6 +522,52 @@ func TestNewEnvConfig(t *testing.T) { Config: EnvConfig{}, WantErr: true, }, + 48: { + Env: map[string]string{ + "AWS_REQUEST_CHECKSUM_CALCULATION": "WHEN_SUPPORTED", + }, + Config: EnvConfig{ + RequestChecksumCalculation: aws.RequestChecksumCalculationWhenSupported, + }, + }, + 49: { + Env: map[string]string{ + "AWS_REQUEST_CHECKSUM_CALCULATION": "when_required", + }, + Config: EnvConfig{ + RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired, + }, + }, + 50: { + Env: map[string]string{ + "AWS_REQUEST_CHECKSUM_CALCULATION": "blabla", + }, + Config: EnvConfig{}, + WantErr: true, + }, + 51: { + Env: map[string]string{ + "AWS_RESPONSE_CHECKSUM_VALIDATION": "WHEN_SUPPORTED", + }, + Config: EnvConfig{ + ResponseChecksumValidation: aws.ResponseChecksumValidationWhenSupported, + }, + }, + 52: { + Env: map[string]string{ + "AWS_RESPONSE_CHECKSUM_VALIDATION": "when_Required", + }, + Config: EnvConfig{ + ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired, + }, + }, + 53: { + Env: map[string]string{ + "AWS_RESPONSE_CHECKSUM_VALIDATION": "blabla", + }, + Config: EnvConfig{}, + WantErr: true, + }, } for i, c := range cases { diff --git a/config/load_options.go b/config/load_options.go index dc6c7d29a83..0810ecf16a2 100644 --- a/config/load_options.go +++ b/config/load_options.go @@ -216,8 +216,15 @@ type LoadOptions struct { // Whether S3 Express auth is disabled. S3DisableExpressAuth *bool + // Whether account id should be built into endpoint resolution AccountIDEndpointMode aws.AccountIDEndpointMode + // Specify if request checksum should be calculated + RequestChecksumCalculation aws.RequestChecksumCalculation + + // Specifies if response checksum should be validated + ResponseChecksumValidation aws.ResponseChecksumValidation + // Service endpoint override. This value is not necessarily final and is // passed to the service's EndpointResolverV2 for further delegation. BaseEndpoint string @@ -288,6 +295,14 @@ func (o LoadOptions) getAccountIDEndpointMode(ctx context.Context) (aws.AccountI return o.AccountIDEndpointMode, len(o.AccountIDEndpointMode) > 0, nil } +func (o LoadOptions) getRequestChecksumCalculation(ctx context.Context) (aws.RequestChecksumCalculation, bool, error) { + return o.RequestChecksumCalculation, o.RequestChecksumCalculation > 0, nil +} + +func (o LoadOptions) getResponseChecksumValidation(ctx context.Context) (aws.ResponseChecksumValidation, bool, error) { + return o.ResponseChecksumValidation, o.ResponseChecksumValidation > 0, nil +} + func (o LoadOptions) getBaseEndpoint(context.Context) (string, bool, error) { return o.BaseEndpoint, o.BaseEndpoint != "", nil } @@ -357,6 +372,26 @@ func WithAccountIDEndpointMode(m aws.AccountIDEndpointMode) LoadOptionsFunc { } } +// WithRequestChecksumCalculation is a helper function to construct functional options +// that sets RequestChecksumCalculation on config's LoadOptions +func WithRequestChecksumCalculation(c aws.RequestChecksumCalculation) LoadOptionsFunc { + return func(o *LoadOptions) error { + if c > 0 { + o.RequestChecksumCalculation = c + } + return nil + } +} + +// WithResponseChecksumValidation is a helper function to construct functional options +// that sets ResponseChecksumValidation on config's LoadOptions +func WithResponseChecksumValidation(v aws.ResponseChecksumValidation) LoadOptionsFunc { + return func(o *LoadOptions) error { + o.ResponseChecksumValidation = v + return nil + } +} + // getDefaultRegion returns DefaultRegion from config's LoadOptions func (o LoadOptions) getDefaultRegion(ctx context.Context) (string, bool, error) { if len(o.DefaultRegion) == 0 { diff --git a/config/provider.go b/config/provider.go index 043781f1f77..a8ff40d846b 100644 --- a/config/provider.go +++ b/config/provider.go @@ -242,6 +242,40 @@ func getAccountIDEndpointMode(ctx context.Context, configs configs) (value aws.A return } +// requestChecksumCalculationProvider provides access to the RequestChecksumCalculation +type requestChecksumCalculationProvider interface { + getRequestChecksumCalculation(context.Context) (aws.RequestChecksumCalculation, bool, error) +} + +func getRequestChecksumCalculation(ctx context.Context, configs configs) (value aws.RequestChecksumCalculation, found bool, err error) { + for _, cfg := range configs { + if p, ok := cfg.(requestChecksumCalculationProvider); ok { + value, found, err = p.getRequestChecksumCalculation(ctx) + if err != nil || found { + break + } + } + } + return +} + +// responseChecksumValidationProvider provides access to the ResponseChecksumValidation +type responseChecksumValidationProvider interface { + getResponseChecksumValidation(context.Context) (aws.ResponseChecksumValidation, bool, error) +} + +func getResponseChecksumValidation(ctx context.Context, configs configs) (value aws.ResponseChecksumValidation, found bool, err error) { + for _, cfg := range configs { + if p, ok := cfg.(responseChecksumValidationProvider); ok { + value, found, err = p.getResponseChecksumValidation(ctx) + if err != nil || found { + break + } + } + } + return +} + // ec2IMDSRegionProvider provides access to the ec2 imds region // configuration value type ec2IMDSRegionProvider interface { diff --git a/config/resolve.go b/config/resolve.go index 41009c7da06..a68bd0993f7 100644 --- a/config/resolve.go +++ b/config/resolve.go @@ -182,6 +182,36 @@ func resolveAccountIDEndpointMode(ctx context.Context, cfg *aws.Config, configs return nil } +// resolveRequestChecksumCalculation extracts the RequestChecksumCalculation from the configs slice's +// SharedConfig or EnvConfig +func resolveRequestChecksumCalculation(ctx context.Context, cfg *aws.Config, configs configs) error { + c, found, err := getRequestChecksumCalculation(ctx, configs) + if err != nil { + return err + } + + if !found { + c = aws.RequestChecksumCalculationWhenSupported + } + cfg.RequestChecksumCalculation = c + return nil +} + +// resolveResponseValidation extracts the ResponseChecksumValidation from the configs slice's +// SharedConfig or EnvConfig +func resolveResponseChecksumValidation(ctx context.Context, cfg *aws.Config, configs configs) error { + c, found, err := getResponseChecksumValidation(ctx, configs) + if err != nil { + return err + } + + if !found { + c = aws.ResponseChecksumValidationWhenSupported + } + cfg.ResponseChecksumValidation = c + return nil +} + // resolveDefaultRegion extracts the first instance of a default region and sets `aws.Config.Region` to the default // region if region had not been resolved from other sources. func resolveDefaultRegion(ctx context.Context, cfg *aws.Config, configs configs) error { diff --git a/config/resolve_test.go b/config/resolve_test.go index e14199ce923..c8df627bbb8 100644 --- a/config/resolve_test.go +++ b/config/resolve_test.go @@ -269,6 +269,80 @@ func TestResolveAccountIDEndpointMode(t *testing.T) { } } +func TestResolveRequestChecksumCalculation(t *testing.T) { + cases := map[string]struct { + RequestChecksumCalculation aws.RequestChecksumCalculation + ExpectCalculation aws.RequestChecksumCalculation + }{ + "checksum calculation when required": { + RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired, + ExpectCalculation: aws.RequestChecksumCalculationWhenRequired, + }, + "default when unset": { + ExpectCalculation: aws.RequestChecksumCalculationWhenSupported, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + var options LoadOptions + optFns := []func(*LoadOptions) error{ + WithRequestChecksumCalculation(c.RequestChecksumCalculation), + } + + for _, optFn := range optFns { + optFn(&options) + } + + configs := configs{options} + var cfg aws.Config + if err := resolveRequestChecksumCalculation(context.Background(), &cfg, configs); err != nil { + t.Fatalf("expect no error, got %v", err) + } + if e, a := c.ExpectCalculation, cfg.RequestChecksumCalculation; e != a { + t.Errorf("expect RequestChecksumCalculation to be %v, got %v", e, a) + } + }) + } +} + +func TestResolveResponseChecksumValidation(t *testing.T) { + cases := map[string]struct { + ResponseChecksumValidation aws.ResponseChecksumValidation + ExpectValidation aws.ResponseChecksumValidation + }{ + "checksum validation when required": { + ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired, + ExpectValidation: aws.ResponseChecksumValidationWhenRequired, + }, + "default when unset": { + ExpectValidation: aws.ResponseChecksumValidationWhenSupported, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + var options LoadOptions + optFns := []func(*LoadOptions) error{ + WithResponseChecksumValidation(c.ResponseChecksumValidation), + } + + for _, optFn := range optFns { + optFn(&options) + } + + configs := configs{options} + var cfg aws.Config + if err := resolveResponseChecksumValidation(context.Background(), &cfg, configs); err != nil { + t.Fatalf("expect no error, got %v", err) + } + if e, a := c.ExpectValidation, cfg.ResponseChecksumValidation; e != a { + t.Errorf("expect ResponseChecksumValidation to be %v, got %v", e, a) + } + }) + } +} + func TestResolveCredentialsProvider(t *testing.T) { var options LoadOptions optFns := []func(options *LoadOptions) error{ diff --git a/config/shared_config.go b/config/shared_config.go index d7a2b5307ea..00b071fe6f1 100644 --- a/config/shared_config.go +++ b/config/shared_config.go @@ -118,6 +118,11 @@ const ( accountIDKey = "aws_account_id" accountIDEndpointMode = "account_id_endpoint_mode" + + requestChecksumCalculationKey = "request_checksum_calculation" + responseChecksumValidationKey = "response_checksum_validation" + checksumWhenSupported = "when_supported" + checksumWhenRequired = "when_required" ) // defaultSharedConfigProfile allows for swapping the default profile for testing @@ -346,6 +351,12 @@ type SharedConfig struct { S3DisableExpressAuth *bool AccountIDEndpointMode aws.AccountIDEndpointMode + + // RequestChecksumCalculation indicates if the request checksum should be calculated + RequestChecksumCalculation aws.RequestChecksumCalculation + + // ResponseChecksumValidation indicates if the response checksum should be validated + ResponseChecksumValidation aws.ResponseChecksumValidation } func (c SharedConfig) getDefaultsMode(ctx context.Context) (value aws.DefaultsMode, ok bool, err error) { @@ -1133,6 +1144,13 @@ func (c *SharedConfig) setFromIniSection(profile string, section ini.Section) er return fmt.Errorf("failed to load %s from shared config, %w", accountIDEndpointMode, err) } + if err := updateRequestChecksumCalculation(&c.RequestChecksumCalculation, section, requestChecksumCalculationKey); err != nil { + return fmt.Errorf("failed to load %s from shared config, %w", requestChecksumCalculationKey, err) + } + if err := updateResponseChecksumValidation(&c.ResponseChecksumValidation, section, responseChecksumValidationKey); err != nil { + return fmt.Errorf("failed to load %s from shared config, %w", responseChecksumValidationKey, err) + } + // Shared Credentials creds := aws.Credentials{ AccessKeyID: section.String(accessKeyIDKey), @@ -1207,6 +1225,42 @@ func updateAIDEndpointMode(m *aws.AccountIDEndpointMode, sec ini.Section, key st return nil } +func updateRequestChecksumCalculation(m *aws.RequestChecksumCalculation, sec ini.Section, key string) error { + if !sec.Has(key) { + return nil + } + + v := sec.String(key) + switch strings.ToLower(v) { + case checksumWhenSupported: + *m = aws.RequestChecksumCalculationWhenSupported + case checksumWhenRequired: + *m = aws.RequestChecksumCalculationWhenRequired + default: + return fmt.Errorf("invalid value for shared config profile field, %s=%s, must be when_supported/when_required", key, v) + } + + return nil +} + +func updateResponseChecksumValidation(m *aws.ResponseChecksumValidation, sec ini.Section, key string) error { + if !sec.Has(key) { + return nil + } + + v := sec.String(key) + switch strings.ToLower(v) { + case checksumWhenSupported: + *m = aws.ResponseChecksumValidationWhenSupported + case checksumWhenRequired: + *m = aws.ResponseChecksumValidationWhenRequired + default: + return fmt.Errorf("invalid value for shared config profile field, %s=%s, must be when_supported/when_required", key, v) + } + + return nil +} + func (c SharedConfig) getRequestMinCompressSizeBytes(ctx context.Context) (int64, bool, error) { if c.RequestMinCompressSizeBytes == nil { return 0, false, nil @@ -1225,6 +1279,14 @@ func (c SharedConfig) getAccountIDEndpointMode(ctx context.Context) (aws.Account return c.AccountIDEndpointMode, len(c.AccountIDEndpointMode) > 0, nil } +func (c SharedConfig) getRequestChecksumCalculation(ctx context.Context) (aws.RequestChecksumCalculation, bool, error) { + return c.RequestChecksumCalculation, c.RequestChecksumCalculation > 0, nil +} + +func (c SharedConfig) getResponseChecksumValidation(ctx context.Context) (aws.ResponseChecksumValidation, bool, error) { + return c.ResponseChecksumValidation, c.ResponseChecksumValidation > 0, nil +} + func updateDefaultsMode(mode *aws.DefaultsMode, section ini.Section, key string) error { if !section.Has(key) { return nil diff --git a/config/shared_config_test.go b/config/shared_config_test.go index ee3f0705e8b..8d71818c8d4 100644 --- a/config/shared_config_test.go +++ b/config/shared_config_test.go @@ -758,6 +758,54 @@ func TestNewSharedConfig(t *testing.T) { }, Err: fmt.Errorf("invalid value for shared config profile field, account_id_endpoint_mode=blabla, must be preferred/required/disabled"), }, + "profile with request checksum calculation when supported": { + ConfigFilenames: []string{testConfigFilename}, + Profile: "request_checksum_calculation_when_supported", + Expected: SharedConfig{ + Profile: "request_checksum_calculation_when_supported", + RequestChecksumCalculation: aws.RequestChecksumCalculationWhenSupported, + }, + }, + "profile with request checksum calculation when required": { + ConfigFilenames: []string{testConfigFilename}, + Profile: "request_checksum_calculation_when_required", + Expected: SharedConfig{ + Profile: "request_checksum_calculation_when_required", + RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired, + }, + }, + "profile with invalid request checksum calculation": { + ConfigFilenames: []string{testConfigFilename}, + Profile: "request_checksum_calculation_error", + Expected: SharedConfig{ + Profile: "request_checksum_calculation_error", + }, + Err: fmt.Errorf("invalid value for shared config profile field, request_checksum_calculation=blabla, must be when_supported/when_required"), + }, + "profile with response checksum validation when supported": { + ConfigFilenames: []string{testConfigFilename}, + Profile: "response_checksum_validation_when_supported", + Expected: SharedConfig{ + Profile: "response_checksum_validation_when_supported", + ResponseChecksumValidation: aws.ResponseChecksumValidationWhenSupported, + }, + }, + "profile with response checksum validation when required": { + ConfigFilenames: []string{testConfigFilename}, + Profile: "response_checksum_validation_when_required", + Expected: SharedConfig{ + Profile: "response_checksum_validation_when_required", + ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired, + }, + }, + "profile with invalid response checksum validation": { + ConfigFilenames: []string{testConfigFilename}, + Profile: "response_checksum_validation_error", + Expected: SharedConfig{ + Profile: "response_checksum_validation_error", + }, + Err: fmt.Errorf("invalid value for shared config profile field, response_checksum_validation=blabla, must be when_supported/when_required"), + }, } for name, c := range cases { diff --git a/config/testdata/shared_config b/config/testdata/shared_config index b2cbc81873b..c7159d52bff 100644 --- a/config/testdata/shared_config +++ b/config/testdata/shared_config @@ -328,3 +328,22 @@ account_id_endpoint_mode = disabled [profile account_id_endpoint_mode_error] account_id_endpoint_mode = blabla + +[profile request_checksum_calculation_when_supported] +request_checksum_calculation = when_supported + +[profile request_checksum_calculation_when_required] +request_checksum_calculation = WHEN_REQUIRED + +[profile request_checksum_calculation_error] +request_checksum_calculation = blabla + +[profile response_checksum_validation_when_supported] +response_checksum_validation = When_SUPPORTED + +[profile response_checksum_validation_when_required] +response_checksum_validation = when_required + +[profile response_checksum_validation_error] +response_checksum_validation = blabla + diff --git a/feature/s3/manager/integ_upload_test.go b/feature/s3/manager/integ_upload_test.go index 2b980fdc89c..96383c243eb 100644 --- a/feature/s3/manager/integ_upload_test.go +++ b/feature/s3/manager/integ_upload_test.go @@ -23,7 +23,6 @@ import ( "testing" "github.com/aws/aws-sdk-go-v2/aws" - v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" "github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/service/s3" s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" @@ -160,9 +159,10 @@ func TestInteg_UploadPresetChecksum(t *testing.T) { expectETag string }{ "auto single part": { - "no checksum": { - payload: bytes.NewReader(singlePartBytes), - expectETag: singlePartETag, + "no checksum algorithm passed": { + payload: bytes.NewReader(singlePartBytes), + expectChecksumCRC32: singlePartCRC32, + expectETag: singlePartETag, }, "CRC32": { algorithm: s3types.ChecksumAlgorithmCrc32, @@ -215,29 +215,34 @@ func TestInteg_UploadPresetChecksum(t *testing.T) { expectETag: singlePartETag, }, "MD5": { - payload: bytes.NewReader(singlePartBytes), - contentMD5: singlePartMD5, - expectETag: singlePartETag, + payload: bytes.NewReader(singlePartBytes), + contentMD5: singlePartMD5, + expectChecksumCRC32: singlePartCRC32, + expectETag: singlePartETag, }, }, "auto multipart part": { - "no checksum": { + "no checksum algorithm passed": { payload: bytes.NewReader(multiPartBytes), expectParts: []s3types.CompletedPart{ { - ETag: aws.String(singlePartETag), - PartNumber: aws.Int32(1), + ChecksumCRC32: aws.String(singlePartCRC32), + ETag: aws.String(singlePartETag), + PartNumber: aws.Int32(1), }, { - ETag: aws.String(singlePartETag), - PartNumber: aws.Int32(2), + ChecksumCRC32: aws.String(singlePartCRC32), + ETag: aws.String(singlePartETag), + PartNumber: aws.Int32(2), }, { - ETag: aws.String(multiPartTailETag), - PartNumber: aws.Int32(3), + ChecksumCRC32: aws.String(multiPartTailCRC32), + ETag: aws.String(multiPartTailETag), + PartNumber: aws.Int32(3), }, }, - expectETag: multiPartETag, + expectChecksumCRC32: multiPartCRC32, + expectETag: multiPartETag, }, "CRC32": { algorithm: s3types.ChecksumAlgorithmCrc32, @@ -484,21 +489,21 @@ func toStringPtr(v string) *string { return &v } -type invalidateHash struct{} +type failedMultipartUpload struct{} -func (b *invalidateHash) ID() string { - return "s3manager:InvalidateHash" +func (m *failedMultipartUpload) ID() string { + return "s3manager:FailedMultipartUpload" } -func (b *invalidateHash) RegisterMiddleware(stack *middleware.Stack) error { - return stack.Serialize.Add(b, middleware.After) +func (m *failedMultipartUpload) RegisterMiddleware(stack *middleware.Stack) error { + return stack.Serialize.Add(m, middleware.After) } -func (b *invalidateHash) HandleSerialize(ctx context.Context, in middleware.SerializeInput, next middleware.SerializeHandler) ( +func (m *failedMultipartUpload) HandleSerialize(ctx context.Context, in middleware.SerializeInput, next middleware.SerializeHandler) ( out middleware.SerializeOutput, metadata middleware.Metadata, err error, ) { if input, ok := in.Parameters.(*s3.UploadPartInput); ok && aws.ToInt32(input.PartNumber) == 2 { - ctx = v4.SetPayloadHash(ctx, "000") + return out, metadata, fmt.Errorf("multipart upload error") } return next.HandleSerialize(ctx, in) @@ -509,7 +514,7 @@ func TestInteg_UploadFailCleanup(t *testing.T) { mgr := manager.NewUploader(client, func(u *manager.Uploader) { u.LeavePartsOnError = false u.ClientOptions = append(u.ClientOptions, func(options *s3.Options) { - options.APIOptions = append(options.APIOptions, (&invalidateHash{}).RegisterMiddleware) + options.APIOptions = append(options.APIOptions, (&failedMultipartUpload{}).RegisterMiddleware) }) }) _, err := mgr.Upload(context.Background(), &s3.PutObjectInput{ diff --git a/feature/s3/manager/upload.go b/feature/s3/manager/upload.go index 18aff6e95be..c6d04f1e99b 100644 --- a/feature/s3/manager/upload.go +++ b/feature/s3/manager/upload.go @@ -584,6 +584,8 @@ func (a completedParts) Less(i, j int) bool { // upload will perform a multipart upload using the firstBuf buffer containing // the first chunk of data. func (u *multiuploader) upload(firstBuf io.ReadSeeker, cleanup func()) (*UploadOutput, error) { + u.initChecksumAlgorithm() + var params s3.CreateMultipartUploadInput awsutil.Copy(¶ms, u.in) @@ -752,6 +754,25 @@ func (u *multiuploader) send(c chunk) error { return nil } +func (u *multiuploader) initChecksumAlgorithm() { + if u.in.ChecksumAlgorithm != "" { + return + } + + switch { + case u.in.ChecksumCRC32 != nil: + u.in.ChecksumAlgorithm = types.ChecksumAlgorithmCrc32 + case u.in.ChecksumCRC32C != nil: + u.in.ChecksumAlgorithm = types.ChecksumAlgorithmCrc32c + case u.in.ChecksumSHA1 != nil: + u.in.ChecksumAlgorithm = types.ChecksumAlgorithmSha1 + case u.in.ChecksumSHA256 != nil: + u.in.ChecksumAlgorithm = types.ChecksumAlgorithmSha256 + default: + u.in.ChecksumAlgorithm = types.ChecksumAlgorithmCrc32 + } +} + // geterr is a thread-safe getter for the error object func (u *multiuploader) geterr() error { u.m.Lock() diff --git a/service/internal/checksum/algorithms.go b/service/internal/checksum/algorithms.go index a17041c35d0..d241bf59fbb 100644 --- a/service/internal/checksum/algorithms.go +++ b/service/internal/checksum/algorithms.go @@ -30,6 +30,9 @@ const ( // AlgorithmSHA256 represents SHA256 hash algorithm AlgorithmSHA256 Algorithm = "SHA256" + + // AlgorithmCRC64NVME represents CRC64NVME hash algorithm + AlgorithmCRC64NVME Algorithm = "CRC64NVME" ) var supportedAlgorithms = []Algorithm{ diff --git a/service/internal/checksum/algorithms_test.go b/service/internal/checksum/algorithms_test.go index 7c821f1e2a5..05e46fb1050 100644 --- a/service/internal/checksum/algorithms_test.go +++ b/service/internal/checksum/algorithms_test.go @@ -1,5 +1,5 @@ -//go:build go1.16 -// +build go1.16 +//go:build go1.21 +// +build go1.21 package checksum diff --git a/service/internal/checksum/aws_chunked_encoding_test.go b/service/internal/checksum/aws_chunked_encoding_test.go index 4f208b4c099..5b991b14b86 100644 --- a/service/internal/checksum/aws_chunked_encoding_test.go +++ b/service/internal/checksum/aws_chunked_encoding_test.go @@ -1,5 +1,5 @@ -//go:build go1.16 -// +build go1.16 +//go:build go1.21 +// +build go1.21 package checksum diff --git a/service/internal/checksum/middleware_add.go b/service/internal/checksum/middleware_add.go index 1b727acbe17..11243a8048f 100644 --- a/service/internal/checksum/middleware_add.go +++ b/service/internal/checksum/middleware_add.go @@ -1,6 +1,7 @@ package checksum import ( + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/smithy-go/middleware" ) @@ -14,11 +15,16 @@ type InputMiddlewareOptions struct { // and true, or false if no algorithm is specified. GetAlgorithm func(interface{}) (string, bool) - // Forces the middleware to compute the input payload's checksum. The - // request will fail if the algorithm is not specified or unable to compute - // the checksum. + // RequireChecksum indicates whether operation model forces middleware to compute the input payload's checksum. + // If RequireChecksum is set to true, checksum will be calculated and RequestChecksumCalculation will be ignored, + // otherwise RequestChecksumCalculation will be used to indicate if checksum will be calculated RequireChecksum bool + // RequestChecksumCalculation is the user config to opt-in/out request checksum calculation. If RequireChecksum is + // set to true, checksum will be calculated and this field will be ignored, otherwise + // RequestChecksumCalculation will be used to indicate if checksum will be calculated + RequestChecksumCalculation aws.RequestChecksumCalculation + // Enables support for wrapping the serialized input payload with a // content-encoding: aws-check wrapper, and including a trailer for the // algorithm's checksum value. @@ -72,7 +78,9 @@ func AddInputMiddleware(stack *middleware.Stack, options InputMiddlewareOptions) // Initial checksum configuration look up middleware err = stack.Initialize.Add(&setupInputContext{ - GetAlgorithm: options.GetAlgorithm, + GetAlgorithm: options.GetAlgorithm, + RequireChecksum: options.RequireChecksum, + RequestChecksumCalculation: options.RequestChecksumCalculation, }, middleware.Before) if err != nil { return err @@ -81,7 +89,6 @@ func AddInputMiddleware(stack *middleware.Stack, options InputMiddlewareOptions) stack.Build.Remove("ContentChecksum") inputChecksum := &computeInputPayloadChecksum{ - RequireChecksum: options.RequireChecksum, EnableTrailingChecksum: options.EnableTrailingChecksum, EnableComputePayloadHash: options.EnableComputeSHA256PayloadHash, EnableDecodedContentLengthHeader: options.EnableDecodedContentLengthHeader, @@ -94,7 +101,6 @@ func AddInputMiddleware(stack *middleware.Stack, options InputMiddlewareOptions) if options.EnableTrailingChecksum { trailerMiddleware := &addInputChecksumTrailer{ EnableTrailingChecksum: inputChecksum.EnableTrailingChecksum, - RequireChecksum: inputChecksum.RequireChecksum, EnableComputePayloadHash: inputChecksum.EnableComputePayloadHash, EnableDecodedContentLengthHeader: inputChecksum.EnableDecodedContentLengthHeader, } @@ -126,6 +132,9 @@ type OutputMiddlewareOptions struct { // mode and true, or false if no mode is specified. GetValidationMode func(interface{}) (string, bool) + // ResponseChecksumValidation is the user config to opt-in/out response checksum validation + ResponseChecksumValidation aws.ResponseChecksumValidation + // The set of checksum algorithms that should be used for response payload // checksum validation. The algorithm(s) used will be a union of the // output's returned algorithms and this set. @@ -134,7 +143,7 @@ type OutputMiddlewareOptions struct { ValidationAlgorithms []string // If set the middleware will ignore output multipart checksums. Otherwise - // an checksum format error will be returned by the middleware. + // a checksum format error will be returned by the middleware. IgnoreMultipartValidation bool // When set the middleware will log when output does not have checksum or @@ -150,7 +159,8 @@ type OutputMiddlewareOptions struct { // checksum. func AddOutputMiddleware(stack *middleware.Stack, options OutputMiddlewareOptions) error { err := stack.Initialize.Add(&setupOutputContext{ - GetValidationMode: options.GetValidationMode, + GetValidationMode: options.GetValidationMode, + ResponseChecksumValidation: options.ResponseChecksumValidation, }, middleware.Before) if err != nil { return err diff --git a/service/internal/checksum/middleware_add_test.go b/service/internal/checksum/middleware_add_test.go index da6efe94a26..20e8c72a483 100644 --- a/service/internal/checksum/middleware_add_test.go +++ b/service/internal/checksum/middleware_add_test.go @@ -1,5 +1,5 @@ -//go:build go1.16 -// +build go1.16 +//go:build go1.21 +// +build go1.21 package checksum @@ -87,7 +87,6 @@ func TestAddInputMiddleware(t *testing.T) { }, }, expectFinalize: &computeInputPayloadChecksum{ - RequireChecksum: true, EnableTrailingChecksum: true, }, }, @@ -167,9 +166,6 @@ func TestAddInputMiddleware(t *testing.T) { var computeInput *computeInputPayloadChecksum if c.expectFinalize != nil && ok { computeInput = finalizeMW.(*computeInputPayloadChecksum) - if e, a := c.expectFinalize.RequireChecksum, computeInput.RequireChecksum; e != a { - t.Errorf("expect %v require checksum, got %v", e, a) - } if e, a := c.expectFinalize.EnableTrailingChecksum, computeInput.EnableTrailingChecksum; e != a { t.Errorf("expect %v enable trailing checksum, got %v", e, a) } diff --git a/service/internal/checksum/middleware_checksum_metrics_tracking.go b/service/internal/checksum/middleware_checksum_metrics_tracking.go new file mode 100644 index 00000000000..861a44293b1 --- /dev/null +++ b/service/internal/checksum/middleware_checksum_metrics_tracking.go @@ -0,0 +1,90 @@ +package checksum + +import ( + "context" + "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" + "github.com/aws/smithy-go/middleware" + smithyhttp "github.com/aws/smithy-go/transport/http" +) + +var supportedChecksumFeatures = map[Algorithm]awsmiddleware.UserAgentFeature{ + AlgorithmCRC32: awsmiddleware.UserAgentFeatureRequestChecksumCRC32, + AlgorithmCRC32C: awsmiddleware.UserAgentFeatureRequestChecksumCRC32C, + AlgorithmSHA1: awsmiddleware.UserAgentFeatureRequestChecksumSHA1, + AlgorithmSHA256: awsmiddleware.UserAgentFeatureRequestChecksumSHA256, + AlgorithmCRC64NVME: awsmiddleware.UserAgentFeatureRequestChecksumCRC64, +} + +// RequestChecksumMetricsTracking is the middleware to track operation request's checksum usage +type RequestChecksumMetricsTracking struct { + RequestChecksumCalculation aws.RequestChecksumCalculation + UserAgent *awsmiddleware.RequestUserAgent +} + +// ID provides the middleware identifier +func (m *RequestChecksumMetricsTracking) ID() string { + return "AWSChecksum:RequestMetricsTracking" +} + +// HandleBuild checks request checksum config and checksum value sent +// and sends corresponding feature id to user agent +func (m *RequestChecksumMetricsTracking) HandleBuild( + ctx context.Context, in middleware.BuildInput, next middleware.BuildHandler, +) ( + out middleware.BuildOutput, metadata middleware.Metadata, err error, +) { + req, ok := in.Request.(*smithyhttp.Request) + if !ok { + return out, metadata, fmt.Errorf("unknown request type %T", req) + } + + switch m.RequestChecksumCalculation { + case aws.RequestChecksumCalculationWhenSupported: + m.UserAgent.AddUserAgentFeature(awsmiddleware.UserAgentFeatureRequestChecksumWhenSupported) + case aws.RequestChecksumCalculationWhenRequired: + m.UserAgent.AddUserAgentFeature(awsmiddleware.UserAgentFeatureRequestChecksumWhenRequired) + } + + for algo, feat := range supportedChecksumFeatures { + checksumHeader := AlgorithmHTTPHeader(algo) + if checksum := req.Header.Get(checksumHeader); checksum != "" { + m.UserAgent.AddUserAgentFeature(feat) + } + } + + return next.HandleBuild(ctx, in) +} + +// ResponseChecksumMetricsTracking is the middleware to track operation response's checksum usage +type ResponseChecksumMetricsTracking struct { + ResponseChecksumValidation aws.ResponseChecksumValidation + UserAgent *awsmiddleware.RequestUserAgent +} + +// ID provides the middleware identifier +func (m *ResponseChecksumMetricsTracking) ID() string { + return "AWSChecksum:ResponseMetricsTracking" +} + +// HandleBuild checks the response checksum config and sends corresponding feature id to user agent +func (m *ResponseChecksumMetricsTracking) HandleBuild( + ctx context.Context, in middleware.BuildInput, next middleware.BuildHandler, +) ( + out middleware.BuildOutput, metadata middleware.Metadata, err error, +) { + req, ok := in.Request.(*smithyhttp.Request) + if !ok { + return out, metadata, fmt.Errorf("unknown request type %T", req) + } + + switch m.ResponseChecksumValidation { + case aws.ResponseChecksumValidationWhenSupported: + m.UserAgent.AddUserAgentFeature(awsmiddleware.UserAgentFeatureResponseChecksumWhenSupported) + case aws.ResponseChecksumValidationWhenRequired: + m.UserAgent.AddUserAgentFeature(awsmiddleware.UserAgentFeatureResponseChecksumWhenRequired) + } + + return next.HandleBuild(ctx, in) +} diff --git a/service/internal/checksum/middleware_checksum_metrics_tracking_test.go b/service/internal/checksum/middleware_checksum_metrics_tracking_test.go new file mode 100644 index 00000000000..cbbbb8d230c --- /dev/null +++ b/service/internal/checksum/middleware_checksum_metrics_tracking_test.go @@ -0,0 +1,123 @@ +//go:build go1.21 +// +build go1.21 + +package checksum + +import ( + "context" + "github.com/aws/aws-sdk-go-v2/aws" + awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" + "github.com/aws/smithy-go/middleware" + smithyhttp "github.com/aws/smithy-go/transport/http" + "net/http" + "strings" + "testing" +) + +func TestRequestChecksumMetricsTracking(t *testing.T) { + cases := map[string]struct { + requestChecksumCalculation aws.RequestChecksumCalculation + reqHeaders http.Header + expectedUserAgentHeader string + }{ + "default": { + requestChecksumCalculation: aws.RequestChecksumCalculationWhenSupported, + reqHeaders: map[string][]string{}, + expectedUserAgentHeader: " m/Z", + }, + "calculate checksum when required": { + requestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired, + reqHeaders: map[string][]string{}, + expectedUserAgentHeader: " m/a", + }, + "default with crc32 checksum": { + requestChecksumCalculation: aws.RequestChecksumCalculationWhenSupported, + reqHeaders: map[string][]string{ + "X-Amz-Checksum-Crc32": {"aa"}, + }, + expectedUserAgentHeader: " m/U,Z", + }, + "calculate checksum when required with sha256 checksum": { + requestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired, + reqHeaders: map[string][]string{ + "X-Amz-Checksum-Sha256": {"aa"}, + }, + expectedUserAgentHeader: " m/Y,a", + }, + "default with crc32c and crc64": { + requestChecksumCalculation: aws.RequestChecksumCalculationWhenSupported, + reqHeaders: map[string][]string{ + "X-Amz-Checksum-Crc32c": {"aa"}, + "X-Amz-Checksum-Crc64nvme": {"aa"}, + }, + expectedUserAgentHeader: " m/V,W,Z", + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + ua := awsmiddleware.NewRequestUserAgent() + req := smithyhttp.NewStackRequest().(*smithyhttp.Request) + req.Header = c.reqHeaders + mw := RequestChecksumMetricsTracking{ + RequestChecksumCalculation: c.requestChecksumCalculation, + UserAgent: ua, + } + mw.HandleBuild(context.Background(), + middleware.BuildInput{Request: req}, + middleware.BuildHandlerFunc(func(ctx context.Context, in middleware.BuildInput) (out middleware.BuildOutput, metadata middleware.Metadata, err error) { + return + })) + + ua.HandleBuild(context.Background(), middleware.BuildInput{Request: req}, + middleware.BuildHandlerFunc(func(ctx context.Context, in middleware.BuildInput) (out middleware.BuildOutput, metadata middleware.Metadata, err error) { + return + })) + + if e, a := c.expectedUserAgentHeader, req.Header["User-Agent"][0]; !strings.Contains(a, e) { + t.Errorf("expected user agent header to include %s, got %s", e, a) + } + }) + } +} + +func TestResponseChecksumMetricsTracking(t *testing.T) { + cases := map[string]struct { + responseChecksumValidation aws.ResponseChecksumValidation + expectedUserAgentHeader string + }{ + "default": { + responseChecksumValidation: aws.ResponseChecksumValidationWhenSupported, + expectedUserAgentHeader: " m/b", + }, + "validate checksum when required": { + responseChecksumValidation: aws.ResponseChecksumValidationWhenRequired, + expectedUserAgentHeader: " m/c", + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + ua := awsmiddleware.NewRequestUserAgent() + req := smithyhttp.NewStackRequest().(*smithyhttp.Request) + mw := ResponseChecksumMetricsTracking{ + ResponseChecksumValidation: c.responseChecksumValidation, + UserAgent: ua, + } + mw.HandleBuild(context.Background(), + middleware.BuildInput{Request: req}, + middleware.BuildHandlerFunc(func(ctx context.Context, in middleware.BuildInput) (out middleware.BuildOutput, metadata middleware.Metadata, err error) { + return + })) + + ua.HandleBuild(context.Background(), middleware.BuildInput{Request: req}, + middleware.BuildHandlerFunc(func(ctx context.Context, in middleware.BuildInput) (out middleware.BuildOutput, metadata middleware.Metadata, err error) { + return + })) + + if e, a := c.expectedUserAgentHeader, req.Header["User-Agent"][0]; !strings.Contains(a, e) { + t.Errorf("expected user agent header to contain %s, got %s", e, a) + } + }) + } +} diff --git a/service/internal/checksum/middleware_compute_input_checksum.go b/service/internal/checksum/middleware_compute_input_checksum.go index 7ffca33f0ef..71a51bf747a 100644 --- a/service/internal/checksum/middleware_compute_input_checksum.go +++ b/service/internal/checksum/middleware_compute_input_checksum.go @@ -7,6 +7,7 @@ import ( "hash" "io" "strconv" + "strings" v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" internalcontext "github.com/aws/aws-sdk-go-v2/internal/context" @@ -16,7 +17,6 @@ import ( ) const ( - contentMD5Header = "Content-Md5" streamingUnsignedPayloadTrailerPayloadHash = "STREAMING-UNSIGNED-PAYLOAD-TRAILER" ) @@ -49,13 +49,6 @@ type computeInputPayloadChecksum struct { // the Algorithm's header is already set on the request. EnableTrailingChecksum bool - // States that a checksum is required to be included for the operation. If - // Input does not specify a checksum, fallback to built in MD5 checksum is - // used. - // - // Replaces smithy-go's ContentChecksum middleware. - RequireChecksum bool - // Enables support for computing the SHA256 checksum of input payloads // along with the algorithm specified checksum. Prevents downstream // middleware handlers (computePayloadSHA256) re-reading the payload. @@ -110,6 +103,15 @@ func (m *computeInputPayloadChecksum) HandleFinalize( ) ( out middleware.FinalizeOutput, metadata middleware.Metadata, err error, ) { + var checksum string + algorithm, ok, err := getInputAlgorithm(ctx) + if err != nil { + return out, metadata, err + } + if !ok { + return next.HandleFinalize(ctx, in) + } + req, ok := in.Request.(*smithyhttp.Request) if !ok { return out, metadata, computeInputHeaderChecksumError{ @@ -117,8 +119,6 @@ func (m *computeInputPayloadChecksum) HandleFinalize( } } - var algorithm Algorithm - var checksum string defer func() { if algorithm == "" || checksum == "" || err != nil { return @@ -130,29 +130,14 @@ func (m *computeInputPayloadChecksum) HandleFinalize( }) }() - // If no algorithm was specified, and the operation requires a checksum, - // fallback to the legacy content MD5 checksum. - algorithm, ok, err = getInputAlgorithm(ctx) - if err != nil { - return out, metadata, err - } else if !ok { - if m.RequireChecksum { - checksum, err = setMD5Checksum(ctx, req) - if err != nil { - return out, metadata, computeInputHeaderChecksumError{ - Msg: "failed to compute stream's MD5 checksum", - Err: err, - } - } - algorithm = Algorithm("MD5") + // If any checksum header is already set nothing to do. + for header := range req.Header { + h := strings.ToUpper(header) + if strings.HasPrefix(h, "X-AMZ-CHECKSUM-") { + algorithm = Algorithm(strings.TrimPrefix(h, "X-AMZ-CHECKSUM-")) + checksum = req.Header.Get(header) + return next.HandleFinalize(ctx, in) } - return next.HandleFinalize(ctx, in) - } - - // If the checksum header is already set nothing to do. - checksumHeader := AlgorithmHTTPHeader(algorithm) - if checksum = req.Header.Get(checksumHeader); checksum != "" { - return next.HandleFinalize(ctx, in) } computePayloadHash := m.EnableComputePayloadHash @@ -217,6 +202,7 @@ func (m *computeInputPayloadChecksum) HandleFinalize( } } + checksumHeader := AlgorithmHTTPHeader(algorithm) req.Header.Set(checksumHeader, checksum) if computePayloadHash { @@ -248,7 +234,6 @@ func (e computeInputTrailingChecksumError) Unwrap() error { return e.Err } // - Trailing checksums are supported. type addInputChecksumTrailer struct { EnableTrailingChecksum bool - RequireChecksum bool EnableComputePayloadHash bool EnableDecodedContentLengthHeader bool } @@ -264,6 +249,16 @@ func (m *addInputChecksumTrailer) HandleFinalize( ) ( out middleware.FinalizeOutput, metadata middleware.Metadata, err error, ) { + algorithm, ok, err := getInputAlgorithm(ctx) + if err != nil { + return out, metadata, computeInputTrailingChecksumError{ + Msg: "failed to get algorithm", + Err: err, + } + } else if !ok { + return next.HandleFinalize(ctx, in) + } + if enabled, _ := middleware.GetStackValue(ctx, useTrailer{}).(bool); !enabled { return next.HandleFinalize(ctx, in) } @@ -281,26 +276,13 @@ func (m *addInputChecksumTrailer) HandleFinalize( } } - // If no algorithm was specified, there is nothing to do. - algorithm, ok, err := getInputAlgorithm(ctx) - if err != nil { - return out, metadata, computeInputTrailingChecksumError{ - Msg: "failed to get algorithm", - Err: err, - } - } else if !ok { - return out, metadata, computeInputTrailingChecksumError{ - Msg: "no algorithm specified", + // If any checksum header is already set nothing to do. + for header := range req.Header { + if strings.HasPrefix(strings.ToLower(header), "x-amz-checksum-") { + return next.HandleFinalize(ctx, in) } } - // If the checksum header is already set before finalize could run, there - // is nothing to do. - checksumHeader := AlgorithmHTTPHeader(algorithm) - if req.Header.Get(checksumHeader) != "" { - return next.HandleFinalize(ctx, in) - } - stream := req.GetStream() streamLength, err := getRequestStreamLength(req) if err != nil { @@ -444,39 +426,3 @@ func getRequestStreamLength(req *smithyhttp.Request) (int64, error) { return -1, nil } - -// setMD5Checksum computes the MD5 of the request payload and sets it to the -// Content-MD5 header. Returning the MD5 base64 encoded string or error. -// -// If the MD5 is already set as the Content-MD5 header, that value will be -// returned, and nothing else will be done. -// -// If the payload is empty, no MD5 will be computed. No error will be returned. -// Empty payloads do not have an MD5 value. -// -// Replaces the smithy-go middleware for httpChecksum trait. -func setMD5Checksum(ctx context.Context, req *smithyhttp.Request) (string, error) { - if v := req.Header.Get(contentMD5Header); len(v) != 0 { - return v, nil - } - stream := req.GetStream() - if stream == nil { - return "", nil - } - - if !req.IsStreamSeekable() { - return "", fmt.Errorf( - "unseekable stream is not supported for computing md5 checksum") - } - - v, err := computeMD5Checksum(stream) - if err != nil { - return "", err - } - if err := req.RewindStream(); err != nil { - return "", fmt.Errorf("failed to rewind stream after computing MD5 checksum, %w", err) - } - // set the 'Content-MD5' header - req.Header.Set(contentMD5Header, string(v)) - return string(v), nil -} diff --git a/service/internal/checksum/middleware_compute_input_checksum_test.go b/service/internal/checksum/middleware_compute_input_checksum_test.go index c8e97de22de..a2e9b02df83 100644 --- a/service/internal/checksum/middleware_compute_input_checksum_test.go +++ b/service/internal/checksum/middleware_compute_input_checksum_test.go @@ -1,5 +1,5 @@ -//go:build go1.16 -// +build go1.16 +//go:build go1.21 +// +build go1.21 package checksum @@ -93,33 +93,96 @@ func TestComputeInputPayloadChecksum(t *testing.T) { "CRC32": "AAAAAA==", }, }, - "no algorithm": { + "http no algorithm checksum header preset": { buildInput: middleware.BuildInput{ Request: func() *smithyhttp.Request { r := smithyhttp.NewStackRequest().(*smithyhttp.Request) - r.URL, _ = url.Parse("https://example.aws") + r.URL, _ = url.Parse("http://example.aws") + r.ContentLength = 11 + r.Header.Set(AlgorithmHTTPHeader(AlgorithmCRC32), "AAAAAA==") + r = requestMust(r.SetStream(bytes.NewReader([]byte("hello world")))) + return r + }(), + }, + expectHeader: http.Header{ + "X-Amz-Checksum-Crc32": []string{"AAAAAA=="}, + }, + expectContentLength: 11, + expectPayload: []byte("hello world"), + }, + "http no algorithm set": { + buildInput: middleware.BuildInput{ + Request: func() *smithyhttp.Request { + r := smithyhttp.NewStackRequest().(*smithyhttp.Request) + r.URL, _ = url.Parse("http://example.aws") r = requestMust(r.SetStream(strings.NewReader("hello world"))) r.ContentLength = 11 return r }(), }, + expectContentLength: 11, expectHeader: http.Header{}, + expectPayload: []byte("hello world"), + }, + "https no algorithm set": { + buildInput: middleware.BuildInput{ + Request: func() *smithyhttp.Request { + r := smithyhttp.NewStackRequest().(*smithyhttp.Request) + r.URL, _ = url.Parse("https://example.aws") + r = requestMust(r.SetStream(strings.NewReader("hello world"))) + r.ContentLength = 11 + return r + }(), + }, expectContentLength: 11, + expectHeader: http.Header{}, expectPayload: []byte("hello world"), }, - "nil stream no algorithm require checksum": { - optionsFn: func(o *computeInputPayloadChecksum) { - o.RequireChecksum = true + "http crc64 checksum header preset": { + initContext: func(ctx context.Context) context.Context { + return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32)) }, buildInput: middleware.BuildInput{ Request: func() *smithyhttp.Request { r := smithyhttp.NewStackRequest().(*smithyhttp.Request) r.URL, _ = url.Parse("http://example.aws") + r.ContentLength = 11 + r.Header.Set(AlgorithmHTTPHeader(AlgorithmCRC64NVME), "S2Zv/ZHmbVs=") + r = requestMust(r.SetStream(bytes.NewReader([]byte("hello world")))) return r }(), }, - expectContentLength: -1, - expectHeader: http.Header{}, + expectHeader: http.Header{ + "X-Amz-Checksum-Crc64nvme": []string{"S2Zv/ZHmbVs="}, + }, + expectContentLength: 11, + expectPayload: []byte("hello world"), + expectChecksumMetadata: map[string]string{ + "CRC64NVME": "S2Zv/ZHmbVs=", + }, + }, + "https crc64 checksum header preset": { + initContext: func(ctx context.Context) context.Context { + return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32)) + }, + buildInput: middleware.BuildInput{ + Request: func() *smithyhttp.Request { + r := smithyhttp.NewStackRequest().(*smithyhttp.Request) + r.URL, _ = url.Parse("https://example.aws") + r.ContentLength = 11 + r.Header.Set(AlgorithmHTTPHeader(AlgorithmCRC64NVME), "S2Zv/ZHmbVs=") + r = requestMust(r.SetStream(bytes.NewReader([]byte("hello world")))) + return r + }(), + }, + expectHeader: http.Header{ + "X-Amz-Checksum-Crc64nvme": []string{"S2Zv/ZHmbVs="}, + }, + expectContentLength: 11, + expectPayload: []byte("hello world"), + expectChecksumMetadata: map[string]string{ + "CRC64NVME": "S2Zv/ZHmbVs=", + }, }, }, @@ -254,94 +317,27 @@ func TestComputeInputPayloadChecksum(t *testing.T) { "CRC32": "AAAAAA==", }, }, - "http no algorithm require checksum": { - optionsFn: func(o *computeInputPayloadChecksum) { - o.RequireChecksum = true - }, - buildInput: middleware.BuildInput{ - Request: func() *smithyhttp.Request { - r := smithyhttp.NewStackRequest().(*smithyhttp.Request) - r.URL, _ = url.Parse("http://example.aws") - r.ContentLength = 11 - r = requestMust(r.SetStream(bytes.NewReader([]byte("hello world")))) - return r - }(), - }, - expectHeader: http.Header{ - "Content-Md5": []string{"XrY7u+Ae7tCTyyK7j1rNww=="}, - }, - expectContentLength: 11, - expectPayload: []byte("hello world"), - expectChecksumMetadata: map[string]string{ - "MD5": "XrY7u+Ae7tCTyyK7j1rNww==", - }, - }, - "http no algorithm require checksum header preset": { - optionsFn: func(o *computeInputPayloadChecksum) { - o.RequireChecksum = true - }, - buildInput: middleware.BuildInput{ - Request: func() *smithyhttp.Request { - r := smithyhttp.NewStackRequest().(*smithyhttp.Request) - r.URL, _ = url.Parse("http://example.aws") - r.ContentLength = 11 - r.Header.Set("Content-MD5", "XrY7u+Ae7tCTyyK7j1rNww==") - r = requestMust(r.SetStream(bytes.NewReader([]byte("hello world")))) - return r - }(), - }, - expectHeader: http.Header{ - "Content-Md5": []string{"XrY7u+Ae7tCTyyK7j1rNww=="}, - }, - expectContentLength: 11, - expectPayload: []byte("hello world"), - expectChecksumMetadata: map[string]string{ - "MD5": "XrY7u+Ae7tCTyyK7j1rNww==", - }, - }, - "https no algorithm require checksum": { - optionsFn: func(o *computeInputPayloadChecksum) { - o.RequireChecksum = true - }, - buildInput: middleware.BuildInput{ - Request: func() *smithyhttp.Request { - r := smithyhttp.NewStackRequest().(*smithyhttp.Request) - r.URL, _ = url.Parse("https://example.aws") - r.ContentLength = 11 - r = requestMust(r.SetStream(bytes.NewReader([]byte("hello world")))) - return r - }(), - }, - expectHeader: http.Header{ - "Content-Md5": []string{"XrY7u+Ae7tCTyyK7j1rNww=="}, - }, - expectContentLength: 11, - expectPayload: []byte("hello world"), - expectChecksumMetadata: map[string]string{ - "MD5": "XrY7u+Ae7tCTyyK7j1rNww==", - }, - }, "http seekable": { initContext: func(ctx context.Context) context.Context { - return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32)) + return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32C)) }, buildInput: middleware.BuildInput{ Request: func() *smithyhttp.Request { r := smithyhttp.NewStackRequest().(*smithyhttp.Request) r.URL, _ = url.Parse("http://example.aws") r.ContentLength = 11 - r = requestMust(r.SetStream(bytes.NewReader([]byte("hello world")))) + r = requestMust(r.SetStream(bytes.NewReader([]byte("Hello world")))) return r }(), }, expectHeader: http.Header{ - "X-Amz-Checksum-Crc32": []string{"DUoRhQ=="}, + "X-Amz-Checksum-Crc32c": []string{"crUfeA=="}, }, expectContentLength: 11, - expectPayload: []byte("hello world"), - expectPayloadHash: "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9", + expectPayload: []byte("Hello world"), + expectPayloadHash: "64ec88ca00b268e5ba1a35678a1b5316d212f4f366b2477232534a8aeca37f3c", expectChecksumMetadata: map[string]string{ - "CRC32": "DUoRhQ==", + "CRC32C": "crUfeA==", }, }, "http payload hash already set": { @@ -474,7 +470,7 @@ func TestComputeInputPayloadChecksum(t *testing.T) { "build error": { "unknown algorithm": { initContext: func(ctx context.Context) context.Context { - return internalcontext.SetChecksumInputAlgorithm(ctx, string("unknown")) + return internalcontext.SetChecksumInputAlgorithm(ctx, "unknown") }, buildInput: middleware.BuildInput{ Request: func() *smithyhttp.Request { @@ -487,9 +483,9 @@ func TestComputeInputPayloadChecksum(t *testing.T) { expectErr: "failed to parse algorithm", expectBuildErr: true, }, - "no algorithm require checksum unseekable stream": { - optionsFn: func(o *computeInputPayloadChecksum) { - o.RequireChecksum = true + "unsupported algorithm": { + initContext: func(ctx context.Context) context.Context { + return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC64NVME)) }, buildInput: middleware.BuildInput{ Request: func() *smithyhttp.Request { @@ -499,12 +495,12 @@ func TestComputeInputPayloadChecksum(t *testing.T) { return r }(), }, - expectErr: "unseekable stream is not supported", + expectErr: "failed to parse algorithm", expectBuildErr: true, }, "http unseekable stream": { initContext: func(ctx context.Context) context.Context { - return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32)) + return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmSHA1)) }, buildInput: middleware.BuildInput{ Request: func() *smithyhttp.Request { @@ -627,54 +623,54 @@ func TestComputeInputPayloadChecksum(t *testing.T) { }, "https seekable": { initContext: func(ctx context.Context) context.Context { - return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32)) + return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmSHA1)) }, buildInput: middleware.BuildInput{ Request: func() *smithyhttp.Request { r := smithyhttp.NewStackRequest().(*smithyhttp.Request) r.URL, _ = url.Parse("https://example.aws") r.ContentLength = 11 - r = requestMust(r.SetStream(bytes.NewReader([]byte("hello world")))) + r = requestMust(r.SetStream(bytes.NewReader([]byte("Hello world")))) return r }(), }, expectHeader: http.Header{ "Content-Encoding": []string{"aws-chunked"}, "X-Amz-Decoded-Content-Length": []string{"11"}, - "X-Amz-Trailer": []string{"x-amz-checksum-crc32"}, + "X-Amz-Trailer": []string{"x-amz-checksum-sha1"}, }, - expectContentLength: 52, - expectPayload: []byte("b\r\nhello world\r\n0\r\nx-amz-checksum-crc32:DUoRhQ==\r\n\r\n"), + expectContentLength: 71, + expectPayload: []byte("b\r\nHello world\r\n0\r\nx-amz-checksum-sha1:e1AsOh9IyGCa4hLN+2Od7jlnP14=\r\n\r\n"), expectPayloadHash: "STREAMING-UNSIGNED-PAYLOAD-TRAILER", expectDeferToFinalize: true, expectChecksumMetadata: map[string]string{ - "CRC32": "DUoRhQ==", + "SHA1": "e1AsOh9IyGCa4hLN+2Od7jlnP14=", }, }, "https seekable unknown length": { initContext: func(ctx context.Context) context.Context { - return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32)) + return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32C)) }, buildInput: middleware.BuildInput{ Request: func() *smithyhttp.Request { r := smithyhttp.NewStackRequest().(*smithyhttp.Request) r.URL, _ = url.Parse("https://example.aws") r.ContentLength = -1 - r = requestMust(r.SetStream(bytes.NewReader([]byte("hello world")))) + r = requestMust(r.SetStream(bytes.NewReader([]byte("Hello world")))) return r }(), }, expectHeader: http.Header{ "Content-Encoding": []string{"aws-chunked"}, "X-Amz-Decoded-Content-Length": []string{"11"}, - "X-Amz-Trailer": []string{"x-amz-checksum-crc32"}, + "X-Amz-Trailer": []string{"x-amz-checksum-crc32c"}, }, - expectContentLength: 52, - expectPayload: []byte("b\r\nhello world\r\n0\r\nx-amz-checksum-crc32:DUoRhQ==\r\n\r\n"), + expectContentLength: 53, + expectPayload: []byte("b\r\nHello world\r\n0\r\nx-amz-checksum-crc32c:crUfeA==\r\n\r\n"), expectPayloadHash: "STREAMING-UNSIGNED-PAYLOAD-TRAILER", expectDeferToFinalize: true, expectChecksumMetadata: map[string]string{ - "CRC32": "DUoRhQ==", + "CRC32C": "crUfeA==", }, }, "https no compute payload hash": { @@ -706,12 +702,12 @@ func TestComputeInputPayloadChecksum(t *testing.T) { }, }, "https no decode content length": { - initContext: func(ctx context.Context) context.Context { - return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32)) - }, optionsFn: func(o *computeInputPayloadChecksum) { o.EnableDecodedContentLengthHeader = false }, + initContext: func(ctx context.Context) context.Context { + return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32)) + }, buildInput: middleware.BuildInput{ Request: func() *smithyhttp.Request { r := smithyhttp.NewStackRequest().(*smithyhttp.Request) @@ -778,7 +774,6 @@ func TestComputeInputPayloadChecksum(t *testing.T) { } trailerMiddleware := &addInputChecksumTrailer{ EnableTrailingChecksum: m.EnableTrailingChecksum, - RequireChecksum: m.RequireChecksum, EnableComputePayloadHash: m.EnableComputePayloadHash, EnableDecodedContentLengthHeader: m.EnableDecodedContentLengthHeader, } @@ -920,7 +915,7 @@ func TestComputeInputPayloadChecksum(t *testing.T) { // assert computed input checksums metadata computedMetadata, ok := GetComputedInputChecksums(metadata) - if e, a := ok, (c.expectChecksumMetadata != nil); e != a { + if e, a := (c.expectChecksumMetadata != nil), ok; e != a { t.Fatalf("expect checksum metadata %t, got %t, %v", e, a, computedMetadata) } if c.expectChecksumMetadata != nil { diff --git a/service/internal/checksum/middleware_setup_context.go b/service/internal/checksum/middleware_setup_context.go index 3db73afe7e8..7fcd29a3814 100644 --- a/service/internal/checksum/middleware_setup_context.go +++ b/service/internal/checksum/middleware_setup_context.go @@ -2,11 +2,16 @@ package checksum import ( "context" + "github.com/aws/aws-sdk-go-v2/aws" internalcontext "github.com/aws/aws-sdk-go-v2/internal/context" "github.com/aws/smithy-go/middleware" ) +const ( + checksumValidationModeEnabled = "ENABLED" +) + // setupChecksumContext is the initial middleware that looks up the input // used to configure checksum behavior. This middleware must be executed before // input validation step or any other checksum middleware. @@ -17,6 +22,16 @@ type setupInputContext struct { // Given the input parameter value, the function must return the algorithm // and true, or false if no algorithm is specified. GetAlgorithm func(interface{}) (string, bool) + + // RequireChecksum indicates whether operation model forces middleware to compute the input payload's checksum. + // If RequireChecksum is set to true, checksum will be calculated and RequestChecksumCalculation will be ignored, + // otherwise RequestChecksumCalculation will be used to indicate if checksum will be calculated + RequireChecksum bool + + // RequestChecksumCalculation is the user config to opt-in/out request checksum calculation. If RequireChecksum is + // set to true, checksum will be calculated and this field will be ignored, otherwise + // RequestChecksumCalculation will be used to indicate if checksum will be calculated + RequestChecksumCalculation aws.RequestChecksumCalculation } // ID for the middleware @@ -31,13 +46,13 @@ func (m *setupInputContext) HandleInitialize( ) ( out middleware.InitializeOutput, metadata middleware.Metadata, err error, ) { - // Check if validation algorithm is specified. - if m.GetAlgorithm != nil { - // check is input resource has a checksum algorithm - algorithm, ok := m.GetAlgorithm(in.Parameters) - if ok && len(algorithm) != 0 { - ctx = internalcontext.SetChecksumInputAlgorithm(ctx, algorithm) - } + if algorithm, ok := m.GetAlgorithm(in.Parameters); ok { + ctx = internalcontext.SetChecksumInputAlgorithm(ctx, algorithm) + return next.HandleInitialize(ctx, in) + } + + if m.RequireChecksum || m.RequestChecksumCalculation == aws.RequestChecksumCalculationWhenSupported { + ctx = internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32)) } return next.HandleInitialize(ctx, in) @@ -50,6 +65,9 @@ type setupOutputContext struct { // Given the input parameter value, the function must return the validation // mode and true, or false if no mode is specified. GetValidationMode func(interface{}) (string, bool) + + // ResponseChecksumValidation states user config to opt-in/out checksum validation + ResponseChecksumValidation aws.ResponseChecksumValidation } // ID for the middleware @@ -64,13 +82,11 @@ func (m *setupOutputContext) HandleInitialize( ) ( out middleware.InitializeOutput, metadata middleware.Metadata, err error, ) { - // Check if validation mode is specified. - if m.GetValidationMode != nil { - // check is input resource has a checksum algorithm - mode, ok := m.GetValidationMode(in.Parameters) - if ok && len(mode) != 0 { - ctx = setContextOutputValidationMode(ctx, mode) - } + + mode, _ := m.GetValidationMode(in.Parameters) + + if m.ResponseChecksumValidation == aws.ResponseChecksumValidationWhenSupported || mode == checksumValidationModeEnabled { + ctx = setContextOutputValidationMode(ctx, checksumValidationModeEnabled) } return next.HandleInitialize(ctx, in) diff --git a/service/internal/checksum/middleware_setup_context_test.go b/service/internal/checksum/middleware_setup_context_test.go index e629ee088d7..81c7250940a 100644 --- a/service/internal/checksum/middleware_setup_context_test.go +++ b/service/internal/checksum/middleware_setup_context_test.go @@ -1,10 +1,11 @@ -//go:build go1.16 -// +build go1.16 +//go:build go1.21 +// +build go1.21 package checksum import ( "context" + "github.com/aws/aws-sdk-go-v2/aws" "testing" internalcontext "github.com/aws/aws-sdk-go-v2/internal/context" @@ -17,29 +18,47 @@ func TestSetupInput(t *testing.T) { } cases := map[string]struct { - inputParams interface{} - getAlgorithm func(interface{}) (string, bool) - expectValue string + inputParams interface{} + getAlgorithm func(interface{}) (string, bool) + RequireChecksum bool + RequestChecksumCalculation aws.RequestChecksumCalculation + expectValue string }{ - "nil accessor": { + "user config require checksum and algorithm unset": { + RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired, + getAlgorithm: func(v interface{}) (string, bool) { + return "", false + }, expectValue: "", }, - "found empty": { - inputParams: Params{Value: ""}, + "require checksum found empty": { + RequireChecksum: true, + inputParams: Params{Value: ""}, getAlgorithm: func(v interface{}) (string, bool) { vv := v.(Params) return vv.Value, true }, expectValue: "", }, - "found not set": { - inputParams: Params{Value: ""}, + "user config require checksum found empty": { + RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired, + inputParams: Params{Value: ""}, getAlgorithm: func(v interface{}) (string, bool) { - return "", false + vv := v.(Params) + return vv.Value, true }, expectValue: "", }, - "found": { + "require checksum and found": { + RequireChecksum: true, + inputParams: Params{Value: "abc123"}, + getAlgorithm: func(v interface{}) (string, bool) { + vv := v.(Params) + return vv.Value, true + }, + expectValue: "abc123", + }, + "user config support checksum and found": { inputParams: Params{Value: "abc123"}, getAlgorithm: func(v interface{}) (string, bool) { vv := v.(Params) @@ -47,12 +66,37 @@ func TestSetupInput(t *testing.T) { }, expectValue: "abc123", }, + "user config require checksum and found": { + RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired, + inputParams: Params{Value: "abc123"}, + getAlgorithm: func(v interface{}) (string, bool) { + vv := v.(Params) + return vv.Value, true + }, + expectValue: "abc123", + }, + "require checksum unset and use default": { + RequireChecksum: true, + getAlgorithm: func(v interface{}) (string, bool) { + return "", false + }, + expectValue: "CRC32", + }, + "user config support checksum and use default": { + RequestChecksumCalculation: aws.RequestChecksumCalculationWhenSupported, + getAlgorithm: func(v interface{}) (string, bool) { + return "", false + }, + expectValue: "CRC32", + }, } for name, c := range cases { t.Run(name, func(t *testing.T) { m := setupInputContext{ - GetAlgorithm: c.getAlgorithm, + GetAlgorithm: c.getAlgorithm, + RequireChecksum: c.RequireChecksum, + RequestChecksumCalculation: c.RequestChecksumCalculation, } _, _, err := m.HandleInitialize(context.Background(), @@ -83,42 +127,54 @@ func TestSetupOutput(t *testing.T) { } cases := map[string]struct { - inputParams interface{} - getValidationMode func(interface{}) (string, bool) - expectValue string + inputParams interface{} + ResponseChecksumValidation aws.ResponseChecksumValidation + getValidationMode func(interface{}) (string, bool) + expectValue string }{ - "nil accessor": { - expectValue: "", + "user config support checksum found empty": { + ResponseChecksumValidation: aws.ResponseChecksumValidationWhenSupported, + inputParams: Params{Value: ""}, + getValidationMode: func(v interface{}) (string, bool) { + vv := v.(Params) + return vv.Value, true + }, + expectValue: "ENABLED", }, - "found empty": { - inputParams: Params{Value: ""}, + "user config support checksum found invalid value": { + ResponseChecksumValidation: aws.ResponseChecksumValidationWhenSupported, + inputParams: Params{Value: "abc123"}, getValidationMode: func(v interface{}) (string, bool) { vv := v.(Params) return vv.Value, true }, - expectValue: "", + expectValue: "ENABLED", }, - "found not set": { - inputParams: Params{Value: ""}, + "user config require checksum found invalid value": { + ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired, + inputParams: Params{Value: "abc123"}, getValidationMode: func(v interface{}) (string, bool) { - return "", false + vv := v.(Params) + return vv.Value, true }, expectValue: "", }, - "found": { - inputParams: Params{Value: "abc123"}, + "user config require checksum found valid value": { + ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired, + inputParams: Params{Value: "ENABLED"}, getValidationMode: func(v interface{}) (string, bool) { vv := v.(Params) return vv.Value, true }, - expectValue: "abc123", + expectValue: "ENABLED", }, } for name, c := range cases { t.Run(name, func(t *testing.T) { m := setupOutputContext{ - GetValidationMode: c.getValidationMode, + GetValidationMode: c.getValidationMode, + ResponseChecksumValidation: c.ResponseChecksumValidation, } _, _, err := m.HandleInitialize(context.Background(), diff --git a/service/internal/checksum/middleware_validate_output.go b/service/internal/checksum/middleware_validate_output.go index 9fde12d86d7..14096a1ce31 100644 --- a/service/internal/checksum/middleware_validate_output.go +++ b/service/internal/checksum/middleware_validate_output.go @@ -55,7 +55,7 @@ func (m *validateOutputPayloadChecksum) ID() string { } // HandleDeserialize is a Deserialize middleware that wraps the HTTP response -// body with an io.ReadCloser that will validate the its checksum. +// body with an io.ReadCloser that will validate its checksum. func (m *validateOutputPayloadChecksum) HandleDeserialize( ctx context.Context, in middleware.DeserializeInput, next middleware.DeserializeHandler, ) ( @@ -66,8 +66,7 @@ func (m *validateOutputPayloadChecksum) HandleDeserialize( return out, metadata, err } - // If there is no validation mode specified nothing is supported. - if mode := getContextOutputValidationMode(ctx); mode != "ENABLED" { + if mode := getContextOutputValidationMode(ctx); mode != checksumValidationModeEnabled { return out, metadata, err } @@ -90,8 +89,6 @@ func (m *validateOutputPayloadChecksum) HandleDeserialize( algorithmToUse = algorithm } - // TODO this must validate the validation mode is set to enabled. - logger := middleware.GetLogger(ctx) // Skip validation if no checksum algorithm or checksum is available. diff --git a/service/internal/checksum/middleware_validate_output_test.go b/service/internal/checksum/middleware_validate_output_test.go index 0bf923e51e7..2b2d4bbca6d 100644 --- a/service/internal/checksum/middleware_validate_output_test.go +++ b/service/internal/checksum/middleware_validate_output_test.go @@ -1,5 +1,5 @@ -//go:build go1.16 -// +build go1.16 +//go:build go1.21 +// +build go1.21 package checksum @@ -49,7 +49,21 @@ func TestValidateOutputPayloadChecksum(t *testing.T) { expectAlgorithmsUsed: []string{"CRC32"}, expectPayload: []byte("hello world"), }, - "failure": { + "no checksum required": { + response: &smithyhttp.Response{ + Response: &http.Response{ + StatusCode: 200, + Header: func() http.Header { + h := http.Header{} + h.Set(AlgorithmHTTPHeader(AlgorithmCRC32C), "crUfeA==") + return h + }(), + Body: ioutil.NopCloser(strings.NewReader("Hello world")), + }, + }, + expectPayload: []byte("Hello world"), + }, + "checksum mismatch failure": { modifyContext: func(ctx context.Context) context.Context { return setContextOutputValidationMode(ctx, "ENABLED") }, @@ -101,19 +115,6 @@ func TestValidateOutputPayloadChecksum(t *testing.T) { expectLogged: "no supported checksum", expectPayload: []byte("hello world"), }, - "no output validation model": { - response: &smithyhttp.Response{ - Response: &http.Response{ - StatusCode: 200, - Header: func() http.Header { - h := http.Header{} - return h - }(), - Body: ioutil.NopCloser(strings.NewReader("hello world")), - }, - }, - expectPayload: []byte("hello world"), - }, "unknown output validation model": { modifyContext: func(ctx context.Context) context.Context { return setContextOutputValidationMode(ctx, "something else") @@ -189,7 +190,7 @@ func TestValidateOutputPayloadChecksum(t *testing.T) { validateOutput := validateOutputPayloadChecksum{ Algorithms: []Algorithm{ - AlgorithmSHA1, AlgorithmCRC32, AlgorithmCRC32C, + AlgorithmSHA1, AlgorithmCRC32, AlgorithmCRC32C, AlgorithmSHA256, }, LogValidationSkipped: true, LogMultipartValidationSkipped: true, diff --git a/service/internal/integrationtest/s3/checksum_test.go b/service/internal/integrationtest/s3/checksum_test.go index 02d6994f41c..1014a66225a 100644 --- a/service/internal/integrationtest/s3/checksum_test.go +++ b/service/internal/integrationtest/s3/checksum_test.go @@ -66,7 +66,14 @@ func TestInteg_ObjectChecksums(t *testing.T) { }, getObjectChecksumMode: s3types.ChecksumModeEnabled, expectPayload: []byte("abc123"), - expectLogged: "Response has no supported checksum.", + expectComputedChecksums: &s3.ComputedInputChecksumsMetadata{ + ComputedChecksums: map[string]string{ + "CRC32": "zwK7XA==", + }, + }, + expectAlgorithmsUsed: &s3.ChecksumValidationMetadata{ + AlgorithmsUsed: []string{"CRC32"}, + }, }, "preset checksum": { params: &s3.PutObjectInput{ @@ -170,7 +177,14 @@ func TestInteg_ObjectChecksums(t *testing.T) { }, getObjectChecksumMode: s3types.ChecksumModeEnabled, expectPayload: []byte("abc123"), - expectLogged: "Response has no supported checksum.", + expectComputedChecksums: &s3.ComputedInputChecksumsMetadata{ + ComputedChecksums: map[string]string{ + "CRC32": "zwK7XA==", + }, + }, + expectAlgorithmsUsed: &s3.ChecksumValidationMetadata{ + AlgorithmsUsed: []string{"CRC32"}, + }, }, "preset checksum": { params: &s3.PutObjectInput{ @@ -251,7 +265,14 @@ func TestInteg_ObjectChecksums(t *testing.T) { "no checksum": { params: &s3.PutObjectInput{}, getObjectChecksumMode: s3types.ChecksumModeEnabled, - expectLogged: "Response has no supported checksum.", + expectComputedChecksums: &s3.ComputedInputChecksumsMetadata{ + ComputedChecksums: map[string]string{ + "CRC32": "AAAAAA==", + }, + }, + expectAlgorithmsUsed: &s3.ChecksumValidationMetadata{ + AlgorithmsUsed: []string{"CRC32"}, + }, }, "preset checksum": { params: &s3.PutObjectInput{ @@ -304,7 +325,14 @@ func TestInteg_ObjectChecksums(t *testing.T) { Body: bytes.NewBuffer([]byte{}), }, getObjectChecksumMode: s3types.ChecksumModeEnabled, - expectLogged: "Response has no supported checksum.", + expectComputedChecksums: &s3.ComputedInputChecksumsMetadata{ + ComputedChecksums: map[string]string{ + "CRC32": "AAAAAA==", + }, + }, + expectAlgorithmsUsed: &s3.ChecksumValidationMetadata{ + AlgorithmsUsed: []string{"CRC32"}, + }, }, "preset checksum": { params: &s3.PutObjectInput{ @@ -394,7 +422,7 @@ func TestInteg_ObjectChecksums(t *testing.T) { } // assert computed input checksums metadata computedChecksums, ok := s3.GetComputedInputChecksumsMetadata(putResult.ResultMetadata) - if e, a := ok, (c.expectComputedChecksums != nil); e != a { + if e, a := (c.expectComputedChecksums != nil), ok; e != a { t.Fatalf("expect computed checksum metadata %t, got %t, %v", e, a, computedChecksums) } if c.expectComputedChecksums != nil { @@ -442,7 +470,7 @@ func TestInteg_ObjectChecksums(t *testing.T) { // assert checksum validation metadata algorithmsUsed, ok := s3.GetChecksumValidationMetadata(getResult.ResultMetadata) - if e, a := ok, (c.expectAlgorithmsUsed != nil); e != a { + if e, a := (c.expectAlgorithmsUsed != nil), ok; e != a { t.Fatalf("expect algorithms used metadata %t, got %t, %v", e, a, algorithmsUsed) } if c.expectAlgorithmsUsed != nil { @@ -468,7 +496,7 @@ func TestInteg_RequireChecksum(t *testing.T) { expectComputedChecksums []string }{ "no algorithm": { - expectComputedChecksums: []string{"MD5"}, + expectComputedChecksums: []string{"CRC32"}, }, "with algorithm": { checksumAlgorithm: types.ChecksumAlgorithmCrc32c, diff --git a/service/s3/api_client.go b/service/s3/api_client.go index 08e432799a9..5cadcf369eb 100644 --- a/service/s3/api_client.go +++ b/service/s3/api_client.go @@ -449,15 +449,17 @@ func setResolvedDefaultsMode(o *Options) { // NewFromConfig returns a new client from the provided config. func NewFromConfig(cfg aws.Config, optFns ...func(*Options)) *Client { opts := Options{ - Region: cfg.Region, - DefaultsMode: cfg.DefaultsMode, - RuntimeEnvironment: cfg.RuntimeEnvironment, - HTTPClient: cfg.HTTPClient, - Credentials: cfg.Credentials, - APIOptions: cfg.APIOptions, - Logger: cfg.Logger, - ClientLogMode: cfg.ClientLogMode, - AppID: cfg.AppID, + Region: cfg.Region, + DefaultsMode: cfg.DefaultsMode, + RuntimeEnvironment: cfg.RuntimeEnvironment, + HTTPClient: cfg.HTTPClient, + Credentials: cfg.Credentials, + APIOptions: cfg.APIOptions, + Logger: cfg.Logger, + ClientLogMode: cfg.ClientLogMode, + AppID: cfg.AppID, + RequestChecksumCalculation: cfg.RequestChecksumCalculation, + ResponseChecksumValidation: cfg.ResponseChecksumValidation, } resolveAWSRetryerProvider(cfg, &opts) resolveAWSRetryMaxAttempts(cfg, &opts) @@ -845,6 +847,30 @@ func addUserAgentRetryMode(stack *middleware.Stack, options Options) error { return nil } +func addRequestChecksumMetricsTracking(stack *middleware.Stack, options Options) error { + ua, err := getOrAddRequestUserAgent(stack) + if err != nil { + return err + } + + return stack.Build.Insert(&internalChecksum.RequestChecksumMetricsTracking{ + RequestChecksumCalculation: options.RequestChecksumCalculation, + UserAgent: ua, + }, "UserAgent", middleware.Before) +} + +func addResponseChecksumMetricsTracking(stack *middleware.Stack, options Options) error { + ua, err := getOrAddRequestUserAgent(stack) + if err != nil { + return err + } + + return stack.Build.Insert(&internalChecksum.ResponseChecksumMetricsTracking{ + ResponseChecksumValidation: options.ResponseChecksumValidation, + UserAgent: ua, + }, "UserAgent", middleware.Before) +} + func resolveTracerProvider(options *Options) { if options.TracerProvider == nil { options.TracerProvider = &tracing.NopTracerProvider{} @@ -1148,6 +1174,10 @@ func (c presignConverter) convertToPresignMiddleware(stack *middleware.Stack, op return nil } +func withNoDefaultChecksumAPIOption(options *Options) { + options.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired +} + func addRequestResponseLogging(stack *middleware.Stack, o Options) error { return stack.Deserialize.Add(&smithyhttp.RequestResponseLogger{ LogRequest: o.ClientLogMode.IsRequest(), diff --git a/service/s3/api_op_CreateBucketMetadataTableConfiguration.go b/service/s3/api_op_CreateBucketMetadataTableConfiguration.go index 4e61399544b..cf89007da4c 100644 --- a/service/s3/api_op_CreateBucketMetadataTableConfiguration.go +++ b/service/s3/api_op_CreateBucketMetadataTableConfiguration.go @@ -170,6 +170,9 @@ func (c *Client) addOperationCreateBucketMetadataTableConfigurationMiddlewares(s if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpCreateBucketMetadataTableConfigurationValidationMiddleware(stack); err != nil { return err } @@ -253,6 +256,7 @@ func addCreateBucketMetadataTableConfigurationInputChecksumMiddlewares(stack *mi return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getCreateBucketMetadataTableConfigurationRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_DeleteObjects.go b/service/s3/api_op_DeleteObjects.go index 7ab24a882bd..f5c0d338844 100644 --- a/service/s3/api_op_DeleteObjects.go +++ b/service/s3/api_op_DeleteObjects.go @@ -347,6 +347,9 @@ func (c *Client) addOperationDeleteObjectsMiddlewares(stack *middleware.Stack, o if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpDeleteObjectsValidationMiddleware(stack); err != nil { return err } @@ -430,6 +433,7 @@ func addDeleteObjectsInputChecksumMiddlewares(stack *middleware.Stack, options O return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getDeleteObjectsRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_GetObject.go b/service/s3/api_op_GetObject.go index 3b1f59cc7d8..7b13772a010 100644 --- a/service/s3/api_op_GetObject.go +++ b/service/s3/api_op_GetObject.go @@ -720,6 +720,9 @@ func (c *Client) addOperationGetObjectMiddlewares(stack *middleware.Stack, optio if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addResponseChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpGetObjectValidationMiddleware(stack); err != nil { return err } @@ -799,6 +802,7 @@ func getGetObjectRequestValidationModeMember(input interface{}) (string, bool) { func addGetObjectOutputChecksumMiddlewares(stack *middleware.Stack, options Options) error { return internalChecksum.AddOutputMiddleware(stack, internalChecksum.OutputMiddlewareOptions{ GetValidationMode: getGetObjectRequestValidationModeMember, + ResponseChecksumValidation: options.ResponseChecksumValidation, ValidationAlgorithms: []string{"CRC32", "CRC32C", "SHA256", "SHA1"}, IgnoreMultipartValidation: true, LogValidationSkipped: true, diff --git a/service/s3/api_op_PutBucketAccelerateConfiguration.go b/service/s3/api_op_PutBucketAccelerateConfiguration.go index 737f8bfde92..0165e30f662 100644 --- a/service/s3/api_op_PutBucketAccelerateConfiguration.go +++ b/service/s3/api_op_PutBucketAccelerateConfiguration.go @@ -185,6 +185,9 @@ func (c *Client) addOperationPutBucketAccelerateConfigurationMiddlewares(stack * if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketAccelerateConfigurationValidationMiddleware(stack); err != nil { return err } @@ -265,6 +268,7 @@ func addPutBucketAccelerateConfigurationInputChecksumMiddlewares(stack *middlewa return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketAccelerateConfigurationRequestAlgorithmMember, RequireChecksum: false, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketAcl.go b/service/s3/api_op_PutBucketAcl.go index eeac549e338..a123daaecef 100644 --- a/service/s3/api_op_PutBucketAcl.go +++ b/service/s3/api_op_PutBucketAcl.go @@ -329,6 +329,9 @@ func (c *Client) addOperationPutBucketAclMiddlewares(stack *middleware.Stack, op if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketAclValidationMiddleware(stack); err != nil { return err } @@ -412,6 +415,7 @@ func addPutBucketAclInputChecksumMiddlewares(stack *middleware.Stack, options Op return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketAclRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketCors.go b/service/s3/api_op_PutBucketCors.go index 9eaa35fe83a..149fac241ba 100644 --- a/service/s3/api_op_PutBucketCors.go +++ b/service/s3/api_op_PutBucketCors.go @@ -207,6 +207,9 @@ func (c *Client) addOperationPutBucketCorsMiddlewares(stack *middleware.Stack, o if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketCorsValidationMiddleware(stack); err != nil { return err } @@ -290,6 +293,7 @@ func addPutBucketCorsInputChecksumMiddlewares(stack *middleware.Stack, options O return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketCorsRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketEncryption.go b/service/s3/api_op_PutBucketEncryption.go index 85841860316..aeffbc8f35a 100644 --- a/service/s3/api_op_PutBucketEncryption.go +++ b/service/s3/api_op_PutBucketEncryption.go @@ -284,6 +284,9 @@ func (c *Client) addOperationPutBucketEncryptionMiddlewares(stack *middleware.St if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketEncryptionValidationMiddleware(stack); err != nil { return err } @@ -367,6 +370,7 @@ func addPutBucketEncryptionInputChecksumMiddlewares(stack *middleware.Stack, opt return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketEncryptionRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketLifecycleConfiguration.go b/service/s3/api_op_PutBucketLifecycleConfiguration.go index 24e7f2fa1cc..5d51b656742 100644 --- a/service/s3/api_op_PutBucketLifecycleConfiguration.go +++ b/service/s3/api_op_PutBucketLifecycleConfiguration.go @@ -293,6 +293,9 @@ func (c *Client) addOperationPutBucketLifecycleConfigurationMiddlewares(stack *m if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketLifecycleConfigurationValidationMiddleware(stack); err != nil { return err } @@ -376,6 +379,7 @@ func addPutBucketLifecycleConfigurationInputChecksumMiddlewares(stack *middlewar return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketLifecycleConfigurationRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketLogging.go b/service/s3/api_op_PutBucketLogging.go index 01380c4bac8..bd2492cdd4d 100644 --- a/service/s3/api_op_PutBucketLogging.go +++ b/service/s3/api_op_PutBucketLogging.go @@ -214,6 +214,9 @@ func (c *Client) addOperationPutBucketLoggingMiddlewares(stack *middleware.Stack if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketLoggingValidationMiddleware(stack); err != nil { return err } @@ -297,6 +300,7 @@ func addPutBucketLoggingInputChecksumMiddlewares(stack *middleware.Stack, option return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketLoggingRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketOwnershipControls.go b/service/s3/api_op_PutBucketOwnershipControls.go index 7eeb45dac42..47263aa2a7c 100644 --- a/service/s3/api_op_PutBucketOwnershipControls.go +++ b/service/s3/api_op_PutBucketOwnershipControls.go @@ -156,6 +156,9 @@ func (c *Client) addOperationPutBucketOwnershipControlsMiddlewares(stack *middle if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketOwnershipControlsValidationMiddleware(stack); err != nil { return err } @@ -229,6 +232,7 @@ func addPutBucketOwnershipControlsInputChecksumMiddlewares(stack *middleware.Sta return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: nil, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketPolicy.go b/service/s3/api_op_PutBucketPolicy.go index 4c0c875a6c0..ef79d7dbf78 100644 --- a/service/s3/api_op_PutBucketPolicy.go +++ b/service/s3/api_op_PutBucketPolicy.go @@ -256,6 +256,9 @@ func (c *Client) addOperationPutBucketPolicyMiddlewares(stack *middleware.Stack, if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketPolicyValidationMiddleware(stack); err != nil { return err } @@ -339,6 +342,7 @@ func addPutBucketPolicyInputChecksumMiddlewares(stack *middleware.Stack, options return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketPolicyRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketReplication.go b/service/s3/api_op_PutBucketReplication.go index 2655ff24709..44c6e3eeb05 100644 --- a/service/s3/api_op_PutBucketReplication.go +++ b/service/s3/api_op_PutBucketReplication.go @@ -225,6 +225,9 @@ func (c *Client) addOperationPutBucketReplicationMiddlewares(stack *middleware.S if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketReplicationValidationMiddleware(stack); err != nil { return err } @@ -308,6 +311,7 @@ func addPutBucketReplicationInputChecksumMiddlewares(stack *middleware.Stack, op return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketReplicationRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketRequestPayment.go b/service/s3/api_op_PutBucketRequestPayment.go index 14b2a5b6940..d92624a9797 100644 --- a/service/s3/api_op_PutBucketRequestPayment.go +++ b/service/s3/api_op_PutBucketRequestPayment.go @@ -172,6 +172,9 @@ func (c *Client) addOperationPutBucketRequestPaymentMiddlewares(stack *middlewar if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketRequestPaymentValidationMiddleware(stack); err != nil { return err } @@ -255,6 +258,7 @@ func addPutBucketRequestPaymentInputChecksumMiddlewares(stack *middleware.Stack, return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketRequestPaymentRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketTagging.go b/service/s3/api_op_PutBucketTagging.go index 5d9761c0062..dbfe97cc3b6 100644 --- a/service/s3/api_op_PutBucketTagging.go +++ b/service/s3/api_op_PutBucketTagging.go @@ -204,6 +204,9 @@ func (c *Client) addOperationPutBucketTaggingMiddlewares(stack *middleware.Stack if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketTaggingValidationMiddleware(stack); err != nil { return err } @@ -287,6 +290,7 @@ func addPutBucketTaggingInputChecksumMiddlewares(stack *middleware.Stack, option return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketTaggingRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketVersioning.go b/service/s3/api_op_PutBucketVersioning.go index fdff8fced53..7c5738f2dbb 100644 --- a/service/s3/api_op_PutBucketVersioning.go +++ b/service/s3/api_op_PutBucketVersioning.go @@ -208,6 +208,9 @@ func (c *Client) addOperationPutBucketVersioningMiddlewares(stack *middleware.St if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketVersioningValidationMiddleware(stack); err != nil { return err } @@ -291,6 +294,7 @@ func addPutBucketVersioningInputChecksumMiddlewares(stack *middleware.Stack, opt return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketVersioningRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketWebsite.go b/service/s3/api_op_PutBucketWebsite.go index be78f71af5c..78c46c9a5d6 100644 --- a/service/s3/api_op_PutBucketWebsite.go +++ b/service/s3/api_op_PutBucketWebsite.go @@ -227,6 +227,9 @@ func (c *Client) addOperationPutBucketWebsiteMiddlewares(stack *middleware.Stack if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketWebsiteValidationMiddleware(stack); err != nil { return err } @@ -310,6 +313,7 @@ func addPutBucketWebsiteInputChecksumMiddlewares(stack *middleware.Stack, option return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketWebsiteRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutObject.go b/service/s3/api_op_PutObject.go index 37ea14349a7..a469b6bb84f 100644 --- a/service/s3/api_op_PutObject.go +++ b/service/s3/api_op_PutObject.go @@ -836,6 +836,9 @@ func (c *Client) addOperationPutObjectMiddlewares(stack *middleware.Stack, optio if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutObjectValidationMiddleware(stack); err != nil { return err } @@ -922,6 +925,7 @@ func addPutObjectInputChecksumMiddlewares(stack *middleware.Stack, options Optio return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutObjectRequestAlgorithmMember, RequireChecksum: false, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: true, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, @@ -965,6 +969,8 @@ func (c *PresignClient) PresignPutObject(ctx context.Context, params *PutObjectI } clientOptFns := append(options.ClientOptions, withNopHTTPClientAPIOption) + clientOptFns = append(options.ClientOptions, withNoDefaultChecksumAPIOption) + result, _, err := c.client.invokeOperation(ctx, "PutObject", params, clientOptFns, c.client.addOperationPutObjectMiddlewares, presignConverter(options).convertToPresignMiddleware, diff --git a/service/s3/api_op_PutObjectAcl.go b/service/s3/api_op_PutObjectAcl.go index 6c13ee838b0..6c8b89a1bb7 100644 --- a/service/s3/api_op_PutObjectAcl.go +++ b/service/s3/api_op_PutObjectAcl.go @@ -381,6 +381,9 @@ func (c *Client) addOperationPutObjectAclMiddlewares(stack *middleware.Stack, op if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutObjectAclValidationMiddleware(stack); err != nil { return err } @@ -464,6 +467,7 @@ func addPutObjectAclInputChecksumMiddlewares(stack *middleware.Stack, options Op return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutObjectAclRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutObjectLegalHold.go b/service/s3/api_op_PutObjectLegalHold.go index be713b88c94..691aa6c19f0 100644 --- a/service/s3/api_op_PutObjectLegalHold.go +++ b/service/s3/api_op_PutObjectLegalHold.go @@ -196,6 +196,9 @@ func (c *Client) addOperationPutObjectLegalHoldMiddlewares(stack *middleware.Sta if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutObjectLegalHoldValidationMiddleware(stack); err != nil { return err } @@ -279,6 +282,7 @@ func addPutObjectLegalHoldInputChecksumMiddlewares(stack *middleware.Stack, opti return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutObjectLegalHoldRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutObjectLockConfiguration.go b/service/s3/api_op_PutObjectLockConfiguration.go index e1f7930f2ee..82b0dc9deff 100644 --- a/service/s3/api_op_PutObjectLockConfiguration.go +++ b/service/s3/api_op_PutObjectLockConfiguration.go @@ -187,6 +187,9 @@ func (c *Client) addOperationPutObjectLockConfigurationMiddlewares(stack *middle if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutObjectLockConfigurationValidationMiddleware(stack); err != nil { return err } @@ -270,6 +273,7 @@ func addPutObjectLockConfigurationInputChecksumMiddlewares(stack *middleware.Sta return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutObjectLockConfigurationRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutObjectRetention.go b/service/s3/api_op_PutObjectRetention.go index 7c5a611a245..0887dda7c2c 100644 --- a/service/s3/api_op_PutObjectRetention.go +++ b/service/s3/api_op_PutObjectRetention.go @@ -203,6 +203,9 @@ func (c *Client) addOperationPutObjectRetentionMiddlewares(stack *middleware.Sta if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutObjectRetentionValidationMiddleware(stack); err != nil { return err } @@ -286,6 +289,7 @@ func addPutObjectRetentionInputChecksumMiddlewares(stack *middleware.Stack, opti return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutObjectRetentionRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutObjectTagging.go b/service/s3/api_op_PutObjectTagging.go index 2703eef4eee..0877ae0d817 100644 --- a/service/s3/api_op_PutObjectTagging.go +++ b/service/s3/api_op_PutObjectTagging.go @@ -239,6 +239,9 @@ func (c *Client) addOperationPutObjectTaggingMiddlewares(stack *middleware.Stack if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutObjectTaggingValidationMiddleware(stack); err != nil { return err } @@ -322,6 +325,7 @@ func addPutObjectTaggingInputChecksumMiddlewares(stack *middleware.Stack, option return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutObjectTaggingRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutPublicAccessBlock.go b/service/s3/api_op_PutPublicAccessBlock.go index d1d05a6f447..d4d05ffee33 100644 --- a/service/s3/api_op_PutPublicAccessBlock.go +++ b/service/s3/api_op_PutPublicAccessBlock.go @@ -190,6 +190,9 @@ func (c *Client) addOperationPutPublicAccessBlockMiddlewares(stack *middleware.S if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutPublicAccessBlockValidationMiddleware(stack); err != nil { return err } @@ -273,6 +276,7 @@ func addPutPublicAccessBlockInputChecksumMiddlewares(stack *middleware.Stack, op return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutPublicAccessBlockRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_RestoreObject.go b/service/s3/api_op_RestoreObject.go index 022dad38eec..697b621c4b1 100644 --- a/service/s3/api_op_RestoreObject.go +++ b/service/s3/api_op_RestoreObject.go @@ -341,6 +341,9 @@ func (c *Client) addOperationRestoreObjectMiddlewares(stack *middleware.Stack, o if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpRestoreObjectValidationMiddleware(stack); err != nil { return err } @@ -421,6 +424,7 @@ func addRestoreObjectInputChecksumMiddlewares(stack *middleware.Stack, options O return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getRestoreObjectRequestAlgorithmMember, RequireChecksum: false, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_UploadPart.go b/service/s3/api_op_UploadPart.go index c4b4de29066..d7d8d30e93e 100644 --- a/service/s3/api_op_UploadPart.go +++ b/service/s3/api_op_UploadPart.go @@ -512,6 +512,9 @@ func (c *Client) addOperationUploadPartMiddlewares(stack *middleware.Stack, opti if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpUploadPartValidationMiddleware(stack); err != nil { return err } @@ -598,6 +601,7 @@ func addUploadPartInputChecksumMiddlewares(stack *middleware.Stack, options Opti return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getUploadPartRequestAlgorithmMember, RequireChecksum: false, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: true, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, @@ -642,6 +646,8 @@ func (c *PresignClient) PresignUploadPart(ctx context.Context, params *UploadPar } clientOptFns := append(options.ClientOptions, withNopHTTPClientAPIOption) + clientOptFns = append(options.ClientOptions, withNoDefaultChecksumAPIOption) + result, _, err := c.client.invokeOperation(ctx, "UploadPart", params, clientOptFns, c.client.addOperationUploadPartMiddlewares, presignConverter(options).convertToPresignMiddleware, diff --git a/service/s3/internal/customizations/presign_test.go b/service/s3/internal/customizations/presign_test.go index 5321ec99ff9..0651b7f631c 100644 --- a/service/s3/internal/customizations/presign_test.go +++ b/service/s3/internal/customizations/presign_test.go @@ -190,7 +190,7 @@ func TestPutObject_PresignURL(t *testing.T) { Bucket: aws.String("mock-bucket"), Key: aws.String("mockkey"), Body: strings.NewReader("hello world"), - ChecksumAlgorithm: s3types.ChecksumAlgorithmCrc32c, + ChecksumAlgorithm: s3types.ChecksumAlgorithmSha1, ChecksumCRC32: aws.String("DUoRhQ=="), }, expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", @@ -209,6 +209,30 @@ func TestPutObject_PresignURL(t *testing.T) { "Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"}, }, }, + "standard case with checksum algo set": { + input: s3.PutObjectInput{ + Bucket: aws.String("mock-bucket"), + Key: aws.String("mockkey"), + Body: strings.NewReader("hello world"), + ChecksumAlgorithm: s3types.ChecksumAlgorithmCrc32c, + }, + expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", + expectRequestURIQuery: []string{ + "X-Amz-Expires=900", + "X-Amz-Credential", + "X-Amz-Date", + "x-id=PutObject", + "X-Amz-Signature", + "X-Amz-Sdk-Checksum-Algorithm", + "X-Amz-Checksum-Crc32c", + }, + expectMethod: "PUT", + expectSignedHeader: http.Header{ + "Content-Length": []string{"11"}, + "Content-Type": []string{"application/octet-stream"}, + "Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"}, + }, + }, "standard case with checksum empty body": { input: s3.PutObjectInput{ Bucket: aws.String("mock-bucket"), diff --git a/service/s3/options.go b/service/s3/options.go index 8c67e4c6218..6b98e8802de 100644 --- a/service/s3/options.go +++ b/service/s3/options.go @@ -92,6 +92,12 @@ type Options struct { // The region to send requests to. (Required) Region string + // Indicates how user opt-in/out request checksum calculation + RequestChecksumCalculation aws.RequestChecksumCalculation + + // Indicates how user opt-in/out response checksum validation + ResponseChecksumValidation aws.ResponseChecksumValidation + // RetryMaxAttempts specifies the maximum number attempts an API client will call // an operation that fails with a retryable error. A value of 0 is ignored, and // will not be used to configure the API client created default retryer, or modify diff --git a/service/s3/snapshot/api_op_CreateBucketMetadataTableConfiguration.go.snap b/service/s3/snapshot/api_op_CreateBucketMetadataTableConfiguration.go.snap index 4b009efa878..50ba96c834d 100644 --- a/service/s3/snapshot/api_op_CreateBucketMetadataTableConfiguration.go.snap +++ b/service/s3/snapshot/api_op_CreateBucketMetadataTableConfiguration.go.snap @@ -23,6 +23,7 @@ CreateBucketMetadataTableConfiguration Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_DeleteObjects.go.snap b/service/s3/snapshot/api_op_DeleteObjects.go.snap index 8481825c051..58d3f780bad 100644 --- a/service/s3/snapshot/api_op_DeleteObjects.go.snap +++ b/service/s3/snapshot/api_op_DeleteObjects.go.snap @@ -23,6 +23,7 @@ DeleteObjects Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_GetObject.go.snap b/service/s3/snapshot/api_op_GetObject.go.snap index 058fe9dd341..ef8825ecc3e 100644 --- a/service/s3/snapshot/api_op_GetObject.go.snap +++ b/service/s3/snapshot/api_op_GetObject.go.snap @@ -23,6 +23,7 @@ GetObject Build stack step ClientRequestID ComputeContentLength + AWSChecksum:ResponseMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketAccelerateConfiguration.go.snap b/service/s3/snapshot/api_op_PutBucketAccelerateConfiguration.go.snap index 32a6302623a..44bfde9eb46 100644 --- a/service/s3/snapshot/api_op_PutBucketAccelerateConfiguration.go.snap +++ b/service/s3/snapshot/api_op_PutBucketAccelerateConfiguration.go.snap @@ -23,6 +23,7 @@ PutBucketAccelerateConfiguration Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketAcl.go.snap b/service/s3/snapshot/api_op_PutBucketAcl.go.snap index 61a75fe98d4..87575123558 100644 --- a/service/s3/snapshot/api_op_PutBucketAcl.go.snap +++ b/service/s3/snapshot/api_op_PutBucketAcl.go.snap @@ -23,6 +23,7 @@ PutBucketAcl Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketCors.go.snap b/service/s3/snapshot/api_op_PutBucketCors.go.snap index 62bf43b1bd8..7c4eb8d46a2 100644 --- a/service/s3/snapshot/api_op_PutBucketCors.go.snap +++ b/service/s3/snapshot/api_op_PutBucketCors.go.snap @@ -23,6 +23,7 @@ PutBucketCors Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketEncryption.go.snap b/service/s3/snapshot/api_op_PutBucketEncryption.go.snap index 8e45c4971da..bf1c5cb05b6 100644 --- a/service/s3/snapshot/api_op_PutBucketEncryption.go.snap +++ b/service/s3/snapshot/api_op_PutBucketEncryption.go.snap @@ -23,6 +23,7 @@ PutBucketEncryption Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketLifecycleConfiguration.go.snap b/service/s3/snapshot/api_op_PutBucketLifecycleConfiguration.go.snap index e5c424450e8..583801997ef 100644 --- a/service/s3/snapshot/api_op_PutBucketLifecycleConfiguration.go.snap +++ b/service/s3/snapshot/api_op_PutBucketLifecycleConfiguration.go.snap @@ -23,6 +23,7 @@ PutBucketLifecycleConfiguration Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketLogging.go.snap b/service/s3/snapshot/api_op_PutBucketLogging.go.snap index 0a076d55237..8738a13e938 100644 --- a/service/s3/snapshot/api_op_PutBucketLogging.go.snap +++ b/service/s3/snapshot/api_op_PutBucketLogging.go.snap @@ -23,6 +23,7 @@ PutBucketLogging Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketOwnershipControls.go.snap b/service/s3/snapshot/api_op_PutBucketOwnershipControls.go.snap index e00b5fe5e25..0cefc18b637 100644 --- a/service/s3/snapshot/api_op_PutBucketOwnershipControls.go.snap +++ b/service/s3/snapshot/api_op_PutBucketOwnershipControls.go.snap @@ -23,6 +23,7 @@ PutBucketOwnershipControls Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketPolicy.go.snap b/service/s3/snapshot/api_op_PutBucketPolicy.go.snap index af9cfdafffa..4cb4d52b26c 100644 --- a/service/s3/snapshot/api_op_PutBucketPolicy.go.snap +++ b/service/s3/snapshot/api_op_PutBucketPolicy.go.snap @@ -23,6 +23,7 @@ PutBucketPolicy Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketReplication.go.snap b/service/s3/snapshot/api_op_PutBucketReplication.go.snap index 113d84299c9..67fd7742c9d 100644 --- a/service/s3/snapshot/api_op_PutBucketReplication.go.snap +++ b/service/s3/snapshot/api_op_PutBucketReplication.go.snap @@ -23,6 +23,7 @@ PutBucketReplication Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketRequestPayment.go.snap b/service/s3/snapshot/api_op_PutBucketRequestPayment.go.snap index c5008714204..3fea86f7735 100644 --- a/service/s3/snapshot/api_op_PutBucketRequestPayment.go.snap +++ b/service/s3/snapshot/api_op_PutBucketRequestPayment.go.snap @@ -23,6 +23,7 @@ PutBucketRequestPayment Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketTagging.go.snap b/service/s3/snapshot/api_op_PutBucketTagging.go.snap index 54855bd1cdf..d12d844b565 100644 --- a/service/s3/snapshot/api_op_PutBucketTagging.go.snap +++ b/service/s3/snapshot/api_op_PutBucketTagging.go.snap @@ -23,6 +23,7 @@ PutBucketTagging Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketVersioning.go.snap b/service/s3/snapshot/api_op_PutBucketVersioning.go.snap index 7ac57c9f441..9b04b8e15f2 100644 --- a/service/s3/snapshot/api_op_PutBucketVersioning.go.snap +++ b/service/s3/snapshot/api_op_PutBucketVersioning.go.snap @@ -23,6 +23,7 @@ PutBucketVersioning Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketWebsite.go.snap b/service/s3/snapshot/api_op_PutBucketWebsite.go.snap index 57b1997313c..4d62cf8ecbd 100644 --- a/service/s3/snapshot/api_op_PutBucketWebsite.go.snap +++ b/service/s3/snapshot/api_op_PutBucketWebsite.go.snap @@ -23,6 +23,7 @@ PutBucketWebsite Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutObject.go.snap b/service/s3/snapshot/api_op_PutObject.go.snap index ff941c819d8..861d7ce62aa 100644 --- a/service/s3/snapshot/api_op_PutObject.go.snap +++ b/service/s3/snapshot/api_op_PutObject.go.snap @@ -23,6 +23,7 @@ PutObject Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware S3100Continue diff --git a/service/s3/snapshot/api_op_PutObjectAcl.go.snap b/service/s3/snapshot/api_op_PutObjectAcl.go.snap index 115a13398d1..fb6f3fce035 100644 --- a/service/s3/snapshot/api_op_PutObjectAcl.go.snap +++ b/service/s3/snapshot/api_op_PutObjectAcl.go.snap @@ -23,6 +23,7 @@ PutObjectAcl Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutObjectLegalHold.go.snap b/service/s3/snapshot/api_op_PutObjectLegalHold.go.snap index c60f6008d31..5c3b59f1e69 100644 --- a/service/s3/snapshot/api_op_PutObjectLegalHold.go.snap +++ b/service/s3/snapshot/api_op_PutObjectLegalHold.go.snap @@ -23,6 +23,7 @@ PutObjectLegalHold Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutObjectLockConfiguration.go.snap b/service/s3/snapshot/api_op_PutObjectLockConfiguration.go.snap index 51a3000e87d..d0456681261 100644 --- a/service/s3/snapshot/api_op_PutObjectLockConfiguration.go.snap +++ b/service/s3/snapshot/api_op_PutObjectLockConfiguration.go.snap @@ -23,6 +23,7 @@ PutObjectLockConfiguration Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutObjectRetention.go.snap b/service/s3/snapshot/api_op_PutObjectRetention.go.snap index b8a8d4b71f1..9273cdc017f 100644 --- a/service/s3/snapshot/api_op_PutObjectRetention.go.snap +++ b/service/s3/snapshot/api_op_PutObjectRetention.go.snap @@ -23,6 +23,7 @@ PutObjectRetention Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutObjectTagging.go.snap b/service/s3/snapshot/api_op_PutObjectTagging.go.snap index ebd21b58c8c..88f87bbee9c 100644 --- a/service/s3/snapshot/api_op_PutObjectTagging.go.snap +++ b/service/s3/snapshot/api_op_PutObjectTagging.go.snap @@ -23,6 +23,7 @@ PutObjectTagging Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutPublicAccessBlock.go.snap b/service/s3/snapshot/api_op_PutPublicAccessBlock.go.snap index 212f2688124..c6e2b3d6746 100644 --- a/service/s3/snapshot/api_op_PutPublicAccessBlock.go.snap +++ b/service/s3/snapshot/api_op_PutPublicAccessBlock.go.snap @@ -23,6 +23,7 @@ PutPublicAccessBlock Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_RestoreObject.go.snap b/service/s3/snapshot/api_op_RestoreObject.go.snap index 43182c060ec..830a3791363 100644 --- a/service/s3/snapshot/api_op_RestoreObject.go.snap +++ b/service/s3/snapshot/api_op_RestoreObject.go.snap @@ -23,6 +23,7 @@ RestoreObject Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_UploadPart.go.snap b/service/s3/snapshot/api_op_UploadPart.go.snap index 09b15500600..0fbe9a23f05 100644 --- a/service/s3/snapshot/api_op_UploadPart.go.snap +++ b/service/s3/snapshot/api_op_UploadPart.go.snap @@ -23,6 +23,7 @@ UploadPart Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware S3100Continue