Skip to content

Commit

Permalink
feat: flexible checksum updates (#2808)
Browse files Browse the repository at this point in the history 
* add config and change default behavior of checksum middleware

* regenerate client

* regenerate clients

* merge protocol test code from main

* merge from main

* add more test cases

* add changelog

* modify s3 internal test

* separate checksum config check and workflow

* restore s3 test

* remove unused md5 header

* separate checksum config and workflow

* change default checksum to const

* add checksum unset enum and modify comment of cfg

* change comment

* Update aws/checksum.go

* change checksum value check logic

* remove old check

* correct unseekable stream logic without tls and its test cases

* revert extra codegen

* change tmv1 upload test cases after introducing flex checksum

* add error test case for crc64

* change test name

* default tmv1 checksum and add flex checksum metrics tracking

* regenerate client and add metrics mw test

* add comment to exported type

* update s3 snapshot

* update tmv1 integ test

* exclude default checksum from presign op

* reorder feature id and simplify metric tracking test

* update changelog

---------

Co-authored-by: Tianyi Wang <wty@amazon.com>
Co-authored-by: Luc Talatinian <102624213+lucix-aws@users.noreply.github.com>
  • Loading branch information
@wty-Bryant @lucix-aws
3 people authored Jan 14, 2025
1 parent 4ffbb7c commit 6636822
Show file tree
Hide file tree
Showing 89 changed files with 1,520 additions and 338 deletions.
11 changes: 11 additions & 0 deletions .changelog/9ebe24c4791541e0840da49eab6f9d97.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"id": "9ebe24c4-7915-41e0-840d-a49eab6f9d97",
"type": "feature",
"description": "S3 client behavior is updated to always calculate a checksum by default for operations that support it (such as PutObject or UploadPart), or require it (such as DeleteObjects). The checksum algorithm used by default now becomes CRC32. Checksum behavior can be configured using `when_supported` and `when_required` options - in code using RequestChecksumCalculation, in shared config using request_checksum_calculation, or as env variable using AWS_REQUEST_CHECKSUM_CALCULATION. The S3 client attempts to validate response checksums for all S3 API operations that support checksums. However, if the SDK has not implemented the specified checksum algorithm then this validation is skipped. Checksum validation behavior can be configured using `when_supported` and `when_required` options - in code using ResponseChecksumValidation, in shared config using response_checksum_validation, or as env variable using AWS_RESPONSE_CHECKSUM_VALIDATION.",
"modules": [
".",
"config",
"service/internal/checksum",
"service/s3"
]
}
33 changes: 33 additions & 0 deletions aws/checksum.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package aws

// RequestChecksumCalculation controls request checksum calculation workflow
type RequestChecksumCalculation int

const (
// RequestChecksumCalculationUnset is the unset value for RequestChecksumCalculation
RequestChecksumCalculationUnset RequestChecksumCalculation = iota

// RequestChecksumCalculationWhenSupported indicates request checksum will be calculated
// if the operation supports input checksums
RequestChecksumCalculationWhenSupported

// RequestChecksumCalculationWhenRequired indicates request checksum will be calculated
// if required by the operation or if user elects to set a checksum algorithm in request
RequestChecksumCalculationWhenRequired
)

// ResponseChecksumValidation controls response checksum validation workflow
type ResponseChecksumValidation int

const (
// ResponseChecksumValidationUnset is the unset value for ResponseChecksumValidation
ResponseChecksumValidationUnset ResponseChecksumValidation = iota

// ResponseChecksumValidationWhenSupported indicates response checksum will be validated
// if the operation supports output checksums
ResponseChecksumValidationWhenSupported

// ResponseChecksumValidationWhenRequired indicates response checksum will only
// be validated if the operation requires output checksum validation
ResponseChecksumValidationWhenRequired
)
27 changes: 27 additions & 0 deletions aws/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,33 @@ type Config struct {

// Controls how a resolved AWS account ID is handled for endpoint routing.
AccountIDEndpointMode AccountIDEndpointMode

// RequestChecksumCalculation determines when request checksum calculation is performed.
//
// There are two possible values for this setting:
//
// 1. RequestChecksumCalculationWhenSupported (default): The checksum is always calculated
// if the operation supports it, regardless of whether the user sets an algorithm in the request.
//
// 2. RequestChecksumCalculationWhenRequired: The checksum is only calculated if the user
// explicitly sets a checksum algorithm in the request.
//
// This setting is sourced from the environment variable AWS_REQUEST_CHECKSUM_CALCULATION
// or the shared config profile attribute "request_checksum_calculation".
RequestChecksumCalculation RequestChecksumCalculation

// ResponseChecksumValidation determines when response checksum validation is performed
//
// There are two possible values for this setting:
//
// 1. ResponseChecksumValidationWhenSupported (default): The checksum is always validated
// if the operation supports it, regardless of whether the user sets the validation mode to ENABLED in request.
//
// 2. ResponseChecksumValidationWhenRequired: The checksum is only validated if the user
// explicitly sets the validation mode to ENABLED in the request
// This variable is sourced from environment variable AWS_RESPONSE_CHECKSUM_VALIDATION or
// the shared config profile attribute "response_checksum_validation".
ResponseChecksumValidation ResponseChecksumValidation
}

// NewConfig returns a new Config pointer that can be chained with builder
Expand Down
35 changes: 22 additions & 13 deletions aws/middleware/user_agent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,19 +76,28 @@ type UserAgentFeature string

// Enumerates UserAgentFeature.
const (
UserAgentFeatureResourceModel UserAgentFeature = "A" // n/a (we don't generate separate resource types)
UserAgentFeatureWaiter = "B"
UserAgentFeaturePaginator = "C"
UserAgentFeatureRetryModeLegacy = "D" // n/a (equivalent to standard)
UserAgentFeatureRetryModeStandard = "E"
UserAgentFeatureRetryModeAdaptive = "F"
UserAgentFeatureS3Transfer = "G"
UserAgentFeatureS3CryptoV1N = "H" // n/a (crypto client is external)
UserAgentFeatureS3CryptoV2 = "I" // n/a
UserAgentFeatureS3ExpressBucket = "J"
UserAgentFeatureS3AccessGrants = "K" // not yet implemented
UserAgentFeatureGZIPRequestCompression = "L"
UserAgentFeatureProtocolRPCV2CBOR = "M"
UserAgentFeatureResourceModel UserAgentFeature = "A" // n/a (we don't generate separate resource types)
UserAgentFeatureWaiter = "B"
UserAgentFeaturePaginator = "C"
UserAgentFeatureRetryModeLegacy = "D" // n/a (equivalent to standard)
UserAgentFeatureRetryModeStandard = "E"
UserAgentFeatureRetryModeAdaptive = "F"
UserAgentFeatureS3Transfer = "G"
UserAgentFeatureS3CryptoV1N = "H" // n/a (crypto client is external)
UserAgentFeatureS3CryptoV2 = "I" // n/a
UserAgentFeatureS3ExpressBucket = "J"
UserAgentFeatureS3AccessGrants = "K" // not yet implemented
UserAgentFeatureGZIPRequestCompression = "L"
UserAgentFeatureProtocolRPCV2CBOR = "M"
UserAgentFeatureRequestChecksumCRC32 = "U"
UserAgentFeatureRequestChecksumCRC32C = "V"
UserAgentFeatureRequestChecksumCRC64 = "W"
UserAgentFeatureRequestChecksumSHA1 = "X"
UserAgentFeatureRequestChecksumSHA256 = "Y"
UserAgentFeatureRequestChecksumWhenSupported = "Z"
UserAgentFeatureRequestChecksumWhenRequired = "a"
UserAgentFeatureResponseChecksumWhenSupported = "b"
UserAgentFeatureResponseChecksumWhenRequired = "c"
)

// RequestUserAgent is a build middleware that set the User-Agent for the request.
Expand Down
16 changes: 16 additions & 0 deletions ...ws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AddAwsConfigFields.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,10 @@ public class AddAwsConfigFields implements GoIntegration {

private static final String SDK_ACCOUNTID_ENDPOINT_MODE = "AccountIDEndpointMode";

private static final String REQUEST_CHECKSUM_CALCULATION = "RequestChecksumCalculation";

private static final String RESPONSE_CHECKSUM_VALIDATION = "ResponseChecksumValidation";

private static final List<AwsConfigField> AWS_CONFIG_FIELDS = ListUtils.of(
AwsConfigField.builder()
.name(REGION_CONFIG_NAME)
Expand Down Expand Up @@ -244,6 +248,18 @@ public class AddAwsConfigFields implements GoIntegration {
.type(SdkGoTypes.Aws.AccountIDEndpointMode)
.documentation("Indicates how aws account ID is applied in endpoint2.0 routing")
.servicePredicate(AccountIDEndpointRouting::hasAccountIdEndpoints)
.build(),
AwsConfigField.builder()
.name(REQUEST_CHECKSUM_CALCULATION)
.type(SdkGoTypes.Aws.RequestChecksumCalculation)
.documentation("Indicates how user opt-in/out request checksum calculation")
.servicePredicate(AwsHttpChecksumGenerator::hasInputChecksumTrait)
.build(),
AwsConfigField.builder()
.name(RESPONSE_CHECKSUM_VALIDATION)
.type(SdkGoTypes.Aws.ResponseChecksumValidation)
.documentation("Indicates how user opt-in/out response checksum validation")
.servicePredicate(AwsHttpChecksumGenerator::hasOutputChecksumTrait)
.build()
);

Expand Down
35 changes: 26 additions & 9 deletions ...codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpChecksumGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import software.amazon.smithy.go.codegen.integration.MiddlewareRegistrar;
import software.amazon.smithy.go.codegen.integration.RuntimeClientPlugin;
import software.amazon.smithy.model.Model;
import software.amazon.smithy.model.knowledge.TopDownIndex;
import software.amazon.smithy.model.shapes.MemberShape;
import software.amazon.smithy.model.shapes.OperationShape;
import software.amazon.smithy.model.shapes.ServiceShape;
Expand Down Expand Up @@ -73,9 +74,7 @@ public byte getOrder() {
@Override
public void processFinalizedModel(GoSettings settings, Model model) {
ServiceShape service = settings.getService(model);
for (ShapeId operationId : service.getAllOperations()) {
final OperationShape operation = model.expectShape(operationId, OperationShape.class);

for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) {
// Create a symbol provider because one is not available in this call.
SymbolProvider symbolProvider = GoCodegenPlugin.createSymbolProvider(model, settings);

Expand Down Expand Up @@ -128,8 +127,7 @@ public void writeAdditionalFiles(
boolean supportsComputeInputChecksumsWorkflow = false;
boolean supportsChecksumValidationWorkflow = false;

for (ShapeId operationID : service.getAllOperations()) {
OperationShape operation = model.expectShape(operationID, OperationShape.class);
for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) {
if (!hasChecksumTrait(model, service, operation)) {
continue;
}
Expand Down Expand Up @@ -178,26 +176,44 @@ public List<RuntimeClientPlugin> getClientPlugins() {
}

// return true if operation shape is decorated with `httpChecksum` trait.
private boolean hasChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
private static boolean hasChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
return operation.hasTrait(HttpChecksumTrait.class);
}

private boolean hasInputChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
private static boolean hasInputChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
if (!hasChecksumTrait(model, service, operation)) {
return false;
}
HttpChecksumTrait trait = operation.expectTrait(HttpChecksumTrait.class);
return trait.isRequestChecksumRequired() || trait.getRequestAlgorithmMember().isPresent();
}

private boolean hasOutputChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
public static boolean hasInputChecksumTrait(Model model, ServiceShape service) {
for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) {
if (hasInputChecksumTrait(model, service, operation)) {
return true;
}
}
return false;
}

private static boolean hasOutputChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
if (!hasChecksumTrait(model, service, operation)) {
return false;
}
HttpChecksumTrait trait = operation.expectTrait(HttpChecksumTrait.class);
return trait.getRequestValidationModeMember().isPresent() && !trait.getResponseAlgorithms().isEmpty();
}

public static boolean hasOutputChecksumTrait(Model model, ServiceShape service) {
for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) {
if (hasOutputChecksumTrait(model, service, operation)) {
return true;
}
}
return false;
}

private boolean isS3ServiceShape(Model model, ServiceShape service) {
String serviceId = service.expectTrait(ServiceTrait.class).getSdkId();
return serviceId.equalsIgnoreCase("S3");
Expand Down Expand Up @@ -244,6 +260,7 @@ private void writeInputMiddlewareHelper(
return $T(stack, $T{
GetAlgorithm: $L,
RequireChecksum: $L,
RequestChecksumCalculation: options.RequestChecksumCalculation,
EnableTrailingChecksum: $L,
EnableComputeSHA256PayloadHash: true,
EnableDecodedContentLengthHeader: $L,
Expand Down Expand Up @@ -284,6 +301,7 @@ private void writeOutputMiddlewareHelper(
writer.write("""
return $T(stack, $T{
GetValidationMode: $L,
ResponseChecksumValidation: options.ResponseChecksumValidation,
ValidationAlgorithms: $L,
IgnoreMultipartValidation: $L,
LogValidationSkipped: true,
Expand All @@ -293,7 +311,6 @@ private void writeOutputMiddlewareHelper(
AwsGoDependency.SERVICE_INTERNAL_CHECKSUM).build(),
SymbolUtils.createValueSymbolBuilder("OutputMiddlewareOptions",
AwsGoDependency.SERVICE_INTERNAL_CHECKSUM).build(),

getRequestValidationModeAccessorFuncName(operationName),
convertToGoStringList(responseAlgorithms),
ignoreMultipartChecksumValidationMap.getOrDefault(
Expand Down
38 changes: 37 additions & 1 deletion ...src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.util.TreeSet;
import software.amazon.smithy.aws.go.codegen.customization.AwsCustomGoDependency;
import software.amazon.smithy.aws.go.codegen.customization.PresignURLAutoFill;
import software.amazon.smithy.aws.traits.HttpChecksumTrait;
import software.amazon.smithy.aws.traits.ServiceTrait;
import software.amazon.smithy.aws.traits.protocols.AwsQueryTrait;
import software.amazon.smithy.aws.traits.protocols.Ec2QueryTrait;
Expand Down Expand Up @@ -67,7 +68,7 @@ public class AwsHttpPresignURLClientGenerator implements GoIntegration {
private static final String CONVERT_TO_PRESIGN_MIDDLEWARE_NAME = "convertToPresignMiddleware";
private static final String CONVERT_TO_PRESIGN_TYPE_NAME = "presignConverter";
private static final String NOP_HTTP_CLIENT_OPTION_FUNC_NAME = "withNopHTTPClientAPIOption";

private static final String NO_DEFAULT_CHECKSUM_OPTION_FUNC_NAME = "withNoDefaultChecksumAPIOption";
private static final String PRESIGN_CLIENT = "PresignClient";
private static final Symbol presignClientSymbol = buildSymbol(PRESIGN_CLIENT, true);

Expand Down Expand Up @@ -218,7 +219,11 @@ public void writeAdditionalFiles(
writeConvertToPresignMiddleware(writer, model, symbolProvider, serviceShape);
});

boolean supportsComputeInputChecksumsWorkflow = false;
for (OperationShape operationShape : TopDownIndex.of(model).getContainedOperations(serviceShape)) {
if (hasInputChecksumTrait(operationShape)) {
supportsComputeInputChecksumsWorkflow = true;
}
if (!validOperations.contains(operationShape.getId())) {
continue;
}
Expand All @@ -231,6 +236,10 @@ public void writeAdditionalFiles(
writeS3AddAsUnsignedPayloadHelper(writer, model, symbolProvider, serviceShape, operationShape);
});
}

if (supportsComputeInputChecksumsWorkflow) {
writePresignRequestChecksumConfigHelpers(settings, goDelegator);
}
}

private void writePresignOperationFunction(
Expand Down Expand Up @@ -263,6 +272,10 @@ private void writePresignOperationFunction(

writer.write("clientOptFns := append(options.ClientOptions, $L)", NOP_HTTP_CLIENT_OPTION_FUNC_NAME);
writer.write("");
if (hasInputChecksumTrait(operationShape)) {
writer.write("clientOptFns = append(options.ClientOptions, $L)", NO_DEFAULT_CHECKSUM_OPTION_FUNC_NAME);
writer.write("");
}

writer.openBlock("result, _, err := c.client.invokeOperation(ctx, $S, params, clientOptFns,", ")",
operationSymbol.getName(), () -> {
Expand Down Expand Up @@ -572,6 +585,29 @@ private void writePresignClientHelpers(
writer.write("");
}

private void writePresignRequestChecksumConfigHelpers(
GoSettings settings,
GoDelegator goDelegator
) {
goDelegator.useFileWriter("api_client.go", settings.getModuleName(), goTemplate("""
func $fn:L(options *Options) {
options.RequestChecksumCalculation = $requestChecksumCalculationWhenRequired:T
}""",
Map.of(
"fn", NO_DEFAULT_CHECKSUM_OPTION_FUNC_NAME,
"requestChecksumCalculationWhenRequired",
AwsGoDependency.AWS_CORE.valueSymbol("RequestChecksumCalculationWhenRequired")
)));
}

private static boolean hasInputChecksumTrait(OperationShape operation) {
if (!operation.hasTrait(HttpChecksumTrait.class)) {
return false;
}
HttpChecksumTrait trait = operation.expectTrait(HttpChecksumTrait.class);
return trait.isRequestChecksumRequired() || trait.getRequestAlgorithmMember().isPresent();
}

/**
* Writes the presigner interface used by the presign url client
*/
Expand Down
2 changes: 2 additions & 0 deletions ...smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/SdkGoTypes.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ public static final class Aws {
public static final Symbol AccountIDEndpointModeRequired = AwsGoDependency.AWS_CORE.valueSymbol("AccountIDEndpointModeRequired");
public static final Symbol AccountIDEndpointModeDisabled = AwsGoDependency.AWS_CORE.valueSymbol("AccountIDEndpointModeDisabled");

public static final Symbol RequestChecksumCalculation = AwsGoDependency.AWS_CORE.valueSymbol("RequestChecksumCalculation");
public static final Symbol ResponseChecksumValidation = AwsGoDependency.AWS_CORE.valueSymbol("ResponseChecksumValidation");

public static final class Middleware {
public static final Symbol GetRequiresLegacyEndpoints = AwsGoDependency.AWS_MIDDLEWARE.valueSymbol("GetRequiresLegacyEndpoints");
Expand Down
Loading

0 comments on commit 6636822

Please sign in to comment.