Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CRC64NVME checksum algo #853

Merged
merged 7 commits into from
Dec 6, 2024
Merged

CRC64NVME checksum algo #853

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
7c53c54
add new algo and tests for new features
TingDaoK Oct 18, 2024
562e9ec
update the algo number
TingDaoK Oct 30, 2024
619b17b
trivial
TingDaoK Oct 30, 2024
7752810
Merge branch 'main' into new-checksum-algo
TingDaoK Dec 5, 2024
90c6273
submodules
TingDaoK Dec 5, 2024
9963aa2
revert test change
TingDaoK Dec 5, 2024
a3070b0
use release
TingDaoK Dec 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -358,6 +358,7 @@ jobs:
cd crt/aws-c-http/tests/py_localhost/
Start-Process -NoNewWindow python .\server.py
Start-Process -NoNewWindow python .\non_tls_server.py
cd ../../../../
python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
python builder.pyz localhost-test -p ${{ env.PACKAGE_NAME }} downstream

Expand Down
2 changes: 1 addition & 1 deletion crt/aws-c-common
Open in desktop
Submodule aws-c-common updated 3 files
+1 −1 .github/workflows/clang-tidy.yml
+12 −6 .github/workflows/codecov.yml
+2 −3 cmake/AwsPrebuildDependency.cmake
2 changes: 1 addition & 1 deletion crt/aws-c-s3
Open in desktop
Submodule aws-c-s3 updated 32 files
+94 −37 .github/workflows/ci.yml
+18 −13 .github/workflows/codecov.yml
+39 −14 include/aws/s3/private/s3_checksums.h
+4 −0 include/aws/s3/private/s3_client_impl.h
+1 −1 include/aws/s3/private/s3_meta_request_impl.h
+3 −0 include/aws/s3/private/s3_request.h
+5 −9 include/aws/s3/private/s3_request_messages.h
+8 −35 include/aws/s3/private/s3_util.h
+4 −3 include/aws/s3/s3_client.h
+73 −3 source/s3_auto_ranged_put.c
+1 −1 source/s3_checksum_stream.c
+183 −105 source/s3_checksums.c
+6 −6 source/s3_chunk_stream.c
+6 −27 source/s3_client.c
+3,158 −2,863 source/s3_endpoint_resolver/aws_s3_endpoint_rule_set.c
+49 −30 source/s3_meta_request.c
+78 −38 source/s3_request_messages.c
+12 −20 source/s3_util.c
+11 −4 tests/CMakeLists.txt
+14 −11 tests/s3_checksum_stream_test.c
+6 −41 tests/s3_checksums_crc32_tests.c
+6 −41 tests/s3_checksums_crc32c_tests.c
+148 −0 tests/s3_checksums_crc64nvme_tests.c
+6 −38 tests/s3_checksums_sha1_tests.c
+6 −54 tests/s3_checksums_sha256_tests.c
+1 −3 tests/s3_checksums_test_case_helper.h
+35 −12 tests/s3_data_plane_tests.c
+4 −1 tests/s3_list_objects_tests.c
+203 −0 tests/s3_mock_server_tests.c
+5 −0 tests/s3_request_messages_tests.c
+63 −1 tests/s3_tester.c
+1 −0 tests/s3_tester.h
5 changes: 4 additions & 1 deletion src/main/java/software/amazon/awssdk/crt/s3/ChecksumAlgorithm.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@ public enum ChecksumAlgorithm {

SHA1(3),

SHA256(4);
SHA256(4),

CRC64NVME(5);

ChecksumAlgorithm(int nativeValue) {
this.nativeValue = nativeValue;
Expand All @@ -46,6 +48,7 @@ private static Map<Integer, ChecksumAlgorithm> buildEnumMapping() {
enumMapping.put(CRC32.getNativeValue(), CRC32);
enumMapping.put(SHA1.getNativeValue(), SHA1);
enumMapping.put(SHA256.getNativeValue(), SHA256);
enumMapping.put(CRC64NVME.getNativeValue(), CRC64NVME);
return enumMapping;
}

Expand Down
140 changes: 111 additions & 29 deletions src/test/java/software/amazon/awssdk/crt/test/S3ClientTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,8 +276,8 @@ public void testS3ClientCreateDestroyHttpProxyOptions() {
proxyOptions.setAuthorizationUsername("username");
proxyOptions.setAuthorizationPassword("password");
try (S3Client client = createS3Client(new S3ClientOptions()
.withRegion(REGION)
.withProxyOptions(proxyOptions), elg)) {
.withRegion(REGION)
.withProxyOptions(proxyOptions), elg)) {
}
}
}
Expand Down Expand Up @@ -449,7 +449,9 @@ public void testS3GetErrorFinishedResponseContextHasAllData() {
S3MetaRequestResponseHandler responseHandler = new S3MetaRequestResponseHandler() {

@Override
public int onResponseBody(ByteBuffer bodyBytesIn, long objectRangeStart, long objectRangeEnd) { return 0; }
public int onResponseBody(ByteBuffer bodyBytesIn, long objectRangeStart, long objectRangeEnd) {
return 0;
}

@Override
public void onFinished(S3FinishedResponseContext context) {
Expand Down Expand Up @@ -864,12 +866,13 @@ private byte[] createTestPayload(int size) {
}

private String uploadObjectPathInit(String objectPath) {
return UPLOAD_DIR+objectPath;
return UPLOAD_DIR + objectPath;
}

private void testS3PutHelper(boolean useFile, boolean unknownContentLength, String objectPath, boolean s3express,
int contentLength, boolean contentMD5) throws IOException {
S3ClientOptions clientOptions = new S3ClientOptions().withRegion(REGION).withEnableS3Express(s3express).withComputeContentMd5(contentMD5);
S3ClientOptions clientOptions = new S3ClientOptions().withRegion(REGION).withEnableS3Express(s3express)
.withComputeContentMd5(contentMD5);
Path uploadFilePath = Files.createTempFile("testS3PutFilePath", ".txt");
try (S3Client client = createS3Client(clientOptions)) {
CompletableFuture<Integer> onFinishedFuture = new CompletableFuture<>();
Expand Down Expand Up @@ -933,7 +936,7 @@ public long getLength() {
.withMetaRequestType(MetaRequestType.PUT_OBJECT).withHttpRequest(httpRequest)
.withResponseHandler(responseHandler);

if(!contentMD5) {
if (!contentMD5) {
ChecksumConfig checksumConfig = new ChecksumConfig().withChecksumAlgorithm(ChecksumAlgorithm.SHA1)
.withChecksumLocation(ChecksumLocation.TRAILER).withValidateChecksum(true);
metaRequestOptions = metaRequestOptions.withChecksumConfig(checksumConfig);
Expand Down Expand Up @@ -1030,7 +1033,8 @@ public void testS3PutNonexistentFilePath() throws IOException {
new HttpHeader("Host", ENDPOINT),
new HttpHeader("Content-Length", String.valueOf(1024)),
};
HttpRequest httpRequest = new HttpRequest("PUT", uploadObjectPathInit("/put_nonexistent_file"), headers, null);
HttpRequest httpRequest = new HttpRequest("PUT", uploadObjectPathInit("/put_nonexistent_file"), headers,
null);

S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions()
.withMetaRequestType(MetaRequestType.PUT_OBJECT)
Expand Down Expand Up @@ -1112,7 +1116,8 @@ public long getLength() {
HttpHeader[] headers = { new HttpHeader("Host", ENDPOINT),
new HttpHeader("Content-Length", Integer.valueOf(payload.capacity()).toString()), };

HttpRequest httpRequest = new HttpRequest("PUT", uploadObjectPathInit("/put_object_test_128MB"), headers, payloadStream);
HttpRequest httpRequest = new HttpRequest("PUT", uploadObjectPathInit("/put_object_test_128MB"), headers,
payloadStream);

S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions()
.withMetaRequestType(MetaRequestType.PUT_OBJECT)
Expand Down Expand Up @@ -1156,7 +1161,7 @@ public long getLength() {
new HttpHeader("Content-Length", Integer.valueOf(payloadResume.capacity()).toString()), };

HttpRequest httpRequestResume = new HttpRequest("PUT",
uploadObjectPathInit("/put_object_test_128MB"), headersResume, payloadStreamResume);
uploadObjectPathInit("/put_object_test_128MB"), headersResume, payloadStreamResume);

CompletableFuture<Integer> onFinishedFutureResume = new CompletableFuture<>();
CompletableFuture<Void> onProgressFutureResume = new CompletableFuture<>();
Expand All @@ -1183,13 +1188,17 @@ public long getLength() {
}
}

@Test
public void testS3PutTrailerChecksums() {
skipIfAndroid();
skipIfNetworkUnavailable();
Assume.assumeTrue(hasAwsCredentials());
private void testS3RoundTripWithChecksumHelper(ChecksumAlgorithm algo, ChecksumLocation location, boolean MPU,
boolean provide_full_object_checksum) throws IOException {

S3ClientOptions clientOptions = new S3ClientOptions().withRegion(REGION);
if (MPU) {
clientOptions.withPartSize(5 * 1024 * 1024);
clientOptions.withMultipartUploadThreshold(5 * 1024 * 1024);
} else {
clientOptions.withPartSize(10 * 1024 * 1024);
clientOptions.withMultipartUploadThreshold(20 * 1024 * 1024);
}
try (S3Client client = createS3Client(clientOptions)) {
CompletableFuture<Integer> onPutFinishedFuture = new CompletableFuture<>();
S3MetaRequestResponseHandler responseHandler = new S3MetaRequestResponseHandler() {
Expand All @@ -1212,7 +1221,7 @@ public void onFinished(S3FinishedResponseContext context) {
}
};

final ByteBuffer payload = ByteBuffer.wrap(createTestPayload(1024 * 1024));
final ByteBuffer payload = ByteBuffer.wrap(createTestPayload(10 * 1024 * 1024));

HttpRequestBodyStream payloadStream = new HttpRequestBodyStream() {
@Override
Expand All @@ -1231,14 +1240,36 @@ public long getLength() {
return payload.capacity();
}
};
ArrayList<HttpHeader> headers = new ArrayList<HttpHeader>();
headers.add(new HttpHeader("Host", ENDPOINT));
headers.add(new HttpHeader("Content-Length", Integer.valueOf(payload.capacity()).toString()));
if (provide_full_object_checksum) {
switch (algo) {
case CRC32:
headers.add(new HttpHeader("x-amz-checksum-crc32", "1BObvg=="));
break;
case CRC64NVME:
headers.add(new HttpHeader("x-amz-checksum-crc64nvme", "fIa08UXfyzk="));
break;

HttpHeader[] headers = { new HttpHeader("Host", ENDPOINT),
new HttpHeader("Content-Length", Integer.valueOf(payload.capacity()).toString()), };
default:
Assert.fail("Unsupported checksum algorithm for full object checksum");
break;
}
}

String objectPath = uploadObjectPathInit("/prefix/round_trip/java_round_trip_test_fc.txt");
HttpRequest httpRequest = new HttpRequest("PUT", objectPath, headers, payloadStream);
ChecksumConfig config = new ChecksumConfig().withChecksumAlgorithm(ChecksumAlgorithm.CRC32)
.withChecksumLocation(ChecksumLocation.TRAILER);
String objectPath = uploadObjectPathInit(
"/prefix/round_trip/java_round_trip_test_fc_" + location.name() + "_" + algo.name())
+ (MPU ? "_mpu" : "") + (provide_full_object_checksum ? "_full_object" : "");
HttpRequest httpRequest = new HttpRequest("PUT", objectPath, headers.toArray(new HttpHeader[0]),
payloadStream);
ChecksumConfig config = new ChecksumConfig();

if (!provide_full_object_checksum) {
/* If the checksum provided for the full object via header, skip the checksum from client. */
config.withChecksumAlgorithm(algo)
.withChecksumLocation(location);
}
S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions()
.withMetaRequestType(MetaRequestType.PUT_OBJECT).withHttpRequest(httpRequest)
.withResponseHandler(responseHandler)
Expand Down Expand Up @@ -1276,17 +1307,18 @@ public void onFinished(S3FinishedResponseContext context) {
new RuntimeException("Checksum was not validated"));
return;
}
if (context.getChecksumAlgorithm() != ChecksumAlgorithm.CRC32) {
if (context.getChecksumAlgorithm() != algo) {
onGetFinishedFuture.completeExceptionally(
new RuntimeException("Checksum was not validated via CRC32"));
new RuntimeException("Checksum was not validated via expected algo: " + algo.name()));
return;
}
onGetFinishedFuture.complete(Integer.valueOf(context.getErrorCode()));
}
};
ArrayList<ChecksumAlgorithm> algorList = new ArrayList<ChecksumAlgorithm>();
algorList.add(ChecksumAlgorithm.CRC32);
algorList.add(ChecksumAlgorithm.CRC64NVME);
algorList.add(ChecksumAlgorithm.CRC32C);
algorList.add(ChecksumAlgorithm.CRC32);
algorList.add(ChecksumAlgorithm.SHA1);
algorList.add(ChecksumAlgorithm.SHA256);
ChecksumConfig validateChecksumConfig = new ChecksumConfig().withValidateChecksum(true)
Expand All @@ -1304,6 +1336,54 @@ public void onFinished(S3FinishedResponseContext context) {
}
}

@Test
public void testS3PutTrailerChecksums() throws Exception {
skipIfAndroid();
skipIfNetworkUnavailable();
Assume.assumeTrue(hasAwsCredentials());
testS3RoundTripWithChecksumHelper(ChecksumAlgorithm.CRC64NVME, ChecksumLocation.TRAILER, false, false);
}

@Test
public void testS3PutHeaderChecksums() throws Exception {
skipIfAndroid();
skipIfNetworkUnavailable();
Assume.assumeTrue(hasAwsCredentials());
testS3RoundTripWithChecksumHelper(ChecksumAlgorithm.CRC64NVME, ChecksumLocation.HEADER, false, false);
}

@Test
public void testS3RoundTripWithFullObjectMPU() throws Exception {
skipIfAndroid();
skipIfNetworkUnavailable();
Assume.assumeTrue(hasAwsCredentials());
testS3RoundTripWithChecksumHelper(ChecksumAlgorithm.CRC64NVME, ChecksumLocation.NONE, true, true);
}

@Test
public void testS3RoundTripWithFullObjectSinglePart() throws Exception {
skipIfAndroid();
skipIfNetworkUnavailable();
Assume.assumeTrue(hasAwsCredentials());
testS3RoundTripWithChecksumHelper(ChecksumAlgorithm.CRC64NVME, ChecksumLocation.NONE, false, true);
}

@Test
public void testS3RoundTripWithFullObjectMPUCRC32() throws Exception {
skipIfAndroid();
skipIfNetworkUnavailable();
Assume.assumeTrue(hasAwsCredentials());
testS3RoundTripWithChecksumHelper(ChecksumAlgorithm.CRC32, ChecksumLocation.NONE, true, true);
}

@Test
public void testS3RoundTripWithFullObjectSinglePartCRC32() throws Exception {
skipIfAndroid();
skipIfNetworkUnavailable();
Assume.assumeTrue(hasAwsCredentials());
testS3RoundTripWithChecksumHelper(ChecksumAlgorithm.CRC32, ChecksumLocation.NONE, false, true);
}

@Test
public void testS3GetS3ExpressOverride() throws Exception {
skipIfNetworkUnavailable();
Expand Down Expand Up @@ -1396,7 +1476,7 @@ public void onFinished(S3FinishedResponseContext context) {
}
}

private void putS3ExpressHelper(String region, S3Client client) throws Exception{
private void putS3ExpressHelper(String region, S3Client client) throws Exception {

CompletableFuture<Integer> onFinishedFuture = new CompletableFuture<>();
S3MetaRequestResponseHandler responseHandler = new S3MetaRequestResponseHandler() {
Expand All @@ -1420,7 +1500,8 @@ public void onFinished(S3FinishedResponseContext context) {
};

HttpHeader[] headers = {
new HttpHeader("Host", region.equals("us-east-1")? S3EXPRESS_ENDPOINT_USE1_AZ4 : S3EXPRESS_ENDPOINT_USW2_AZ1),
new HttpHeader("Host",
region.equals("us-east-1") ? S3EXPRESS_ENDPOINT_USE1_AZ4 : S3EXPRESS_ENDPOINT_USW2_AZ1),
};
HttpRequest httpRequest;
String path = uploadObjectPathInit("/put_object_test_10MB.txt");
Expand Down Expand Up @@ -1507,7 +1588,6 @@ public void testS3Copy() {
String COPY_SOURCE_KEY = "crt-canary-obj.txt";
String X_AMZ_COPY_SOURCE_HEADER = "x-amz-copy-source";


S3ClientOptions clientOptions = new S3ClientOptions().withRegion(REGION);
try (S3Client client = createS3Client(clientOptions)) {
CompletableFuture<Integer> onFinishedFuture = new CompletableFuture<>();
Expand Down Expand Up @@ -1733,7 +1813,8 @@ public void onFinished(S3FinishedResponseContext context) {
concurrentSlots.release();

if (context.getErrorCode() != 0) {
onFinishedFuture.completeExceptionally(makeExceptionFromFinishedResponseContext(context));
onFinishedFuture
.completeExceptionally(makeExceptionFromFinishedResponseContext(context));
return;
}

Expand Down Expand Up @@ -1862,7 +1943,8 @@ public void onFinished(S3FinishedResponseContext context) {
concurrentSlots.release();

if (context.getErrorCode() != 0) {
onFinishedFuture.completeExceptionally(makeExceptionFromFinishedResponseContext(context));
onFinishedFuture
.completeExceptionally(makeExceptionFromFinishedResponseContext(context));
return;
}

Expand Down
Loading