add helper for large buffers

include/aws/checksums/crc.h

@@ -20,6 +20,15 @@ AWS_EXTERN_C_BEGIN
  */
 AWS_CHECKSUMS_API uint32_t aws_checksums_crc32(const uint8_t *input, int length, uint32_t previous_crc32);
 
+/**
+ * The entry point function to perform a CRC32 (Ethernet, gzip) computation.
+ * Supports buffer lengths up to uint64_t max.
+ * Selects a suitable implementation based on hardware capabilities.
+ * Pass 0 in the previousCrc32 parameter as an initial value unless continuing
+ * to update a running crc in a subsequent call.
+ */
+AWS_CHECKSUMS_API uint32_t aws_checksums_crc32_u64(const uint8_t *input, uint64_t length, uint32_t previous_crc32);
+
 /**
  * The entry point function to perform a Castagnoli CRC32c (iSCSI) computation.
  * Selects a suitable implementation based on hardware capabilities.
@@ -28,6 +37,15 @@ AWS_CHECKSUMS_API uint32_t aws_checksums_crc32(const uint8_t *input, int length,
  */
 AWS_CHECKSUMS_API uint32_t aws_checksums_crc32c(const uint8_t *input, int length, uint32_t previous_crc32c);
 
+/**
+ * The entry point function to perform a Castagnoli CRC32c (iSCSI) computation.
+ * Supports buffer lengths up to uint64_t max.
+ * Selects a suitable implementation based on hardware capabilities.
+ * Pass 0 in the previousCrc32 parameter as an initial value unless continuing
+ * to update a running crc in a subsequent call.
+ */
+AWS_CHECKSUMS_API uint32_t aws_checksums_crc32c_u64(const uint8_t *input, uint64_t length, uint32_t previous_crc32c);
+
 /**
  * The entry point function to perform a CRC64-NVME (a.k.a. CRC64-Rocksoft) computation.
  * Selects a suitable implementation based on hardware capabilities.
@@ -38,6 +56,17 @@ AWS_CHECKSUMS_API uint32_t aws_checksums_crc32c(const uint8_t *input, int length
  */
 AWS_CHECKSUMS_API uint64_t aws_checksums_crc64nvme(const uint8_t *input, int length, uint64_t previous_crc64);
 
+/**
+ * The entry point function to perform a CRC64-NVME (a.k.a. CRC64-Rocksoft) computation.
+ * Supports buffer lengths up to uint64_t max.
+ * Selects a suitable implementation based on hardware capabilities.
+ * Pass 0 in the previousCrc64 parameter as an initial value unless continuing
+ * to update a running crc in a subsequent call.
+ * There are many variants of CRC64 algorithms. This CRC64 variant is bit-reflected (based on
+ * the non bit-reflected polynomial 0xad93d23594c93659) and inverts the CRC input and output bits.
+ */
+AWS_CHECKSUMS_API uint64_t aws_checksums_crc64nvme_u64(const uint8_t *input, uint64_t length, uint64_t previous_crc64);
+
 AWS_EXTERN_C_END
 AWS_POP_SANE_WARNING_LEVEL
 

include/aws/checksums/private/crc_util.h

@@ -0,0 +1,26 @@
+#ifndef AWS_CHECKSUMS_PRIVATE_CRC_UTIL_H
+#define AWS_CHECKSUMS_PRIVATE_CRC_UTIL_H
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+
+#include <aws/common/stdint.h>
+#include <limits.h>
+
+#define large_buffer_apply_impl(Name, T) \
+  T aws_large_buffer_apply_##Name(T (*checksum_fn)(const uint8_t *, int, T), \
+    const uint8_t *buffer, \
+    uint64_t length, \
+    T previous) { \
+    T val = previous; \
+    while (length > INT_MAX) { \
+        val = checksum_fn(buffer, INT_MAX, val); \
+        buffer += (size_t)INT_MAX; \
+        length -= (uint64_t)INT_MAX; \
+    } \
+    val = checksum_fn(buffer, (int)length, val); \
+    return val; \
+  } \
+
+#endif /* AWS_CHECKSUMS_PRIVATE_CRC_UTIL_H */

source/crc.c

@@ -4,9 +4,12 @@
  */
 #include <aws/checksums/crc.h>
 #include <aws/checksums/private/crc_priv.h>
+#include <aws/checksums/private/crc_util.h>
 
 #include <aws/common/cpuid.h>
 
+large_buffer_apply_impl(crc32, uint32_t)
+
 static uint32_t (*s_crc32c_fn_ptr)(const uint8_t *input, int length, uint32_t previous_crc32c) = 0;
 static uint32_t (*s_crc32_fn_ptr)(const uint8_t *input, int length, uint32_t previous_crc32) = 0;
 
@@ -25,6 +28,10 @@ uint32_t aws_checksums_crc32(const uint8_t *input, int length, uint32_t previous
     return s_crc32_fn_ptr(input, length, previous_crc32);
 }
 
+uint32_t aws_checksums_crc32_u64(const uint8_t *input, uint64_t length, uint32_t previous_crc32) {
+    return aws_large_buffer_apply_crc32(aws_checksums_crc32, input, length, previous_crc32);
+}
+
 uint32_t aws_checksums_crc32c(const uint8_t *input, int length, uint32_t previous_crc32c) {
     if (AWS_UNLIKELY(!s_crc32c_fn_ptr)) {
 #if defined(AWS_USE_CPU_EXTENSIONS) && defined(AWS_ARCH_INTEL_X64)
@@ -46,3 +53,7 @@ uint32_t aws_checksums_crc32c(const uint8_t *input, int length, uint32_t previou
 
     return s_crc32c_fn_ptr(input, length, previous_crc32c);
 }
+
+uint32_t aws_checksums_crc32c_u64(const uint8_t *input, uint64_t length, uint32_t previous_crc32) {
+    return aws_large_buffer_apply_crc32(aws_checksums_crc32c, input, length, previous_crc32);
+}

source/crc64.c

@@ -5,8 +5,11 @@
 
 #include <aws/checksums/crc.h>
 #include <aws/checksums/private/crc64_priv.h>
+#include <aws/checksums/private/crc_util.h>
 #include <aws/common/cpuid.h>
 
+large_buffer_apply_impl(crc64, uint64_t)
+
 AWS_ALIGNED_TYPEDEF(uint8_t, checksums_maxks_shifts_type[6][16], 16);
 // Intel PSHUFB / ARM VTBL patterns for left/right shifts and masks
 checksums_maxks_shifts_type aws_checksums_masks_shifts = {
@@ -122,3 +125,7 @@ uint64_t aws_checksums_crc64nvme(const uint8_t *input, int length, uint64_t prev
 
     return s_crc64nvme_fn_ptr(input, length, prev_crc64);
 }
+
+uint64_t aws_checksums_crc64nvme_u64(const uint8_t *input, uint64_t length, uint64_t previous_crc64) {
+    return aws_large_buffer_apply_crc64(aws_checksums_crc64nvme, input, length, previous_crc64);
+}

tests/CMakeLists.txt

@@ -9,6 +9,8 @@ file(GLOB TESTS ${TEST_HDRS} ${TEST_SRC})
 
 add_test_case(test_crc32c)
 add_test_case(test_crc32)
+add_test_case(test_large_buffer_crc32)
 add_test_case(test_crc64nvme)
+add_test_case(test_large_buffer_crc64)
 
 generate_test_driver(${PROJECT_NAME}-tests)

tests/crc64_test.c

@@ -126,3 +126,18 @@ static int s_test_crc64nvme(struct aws_allocator *allocator, void *ctx) {
 }
 
 AWS_TEST_CASE(test_crc64nvme, s_test_crc64nvme)
+
+static int s_test_large_buffer_crc64(struct aws_allocator *allocator, void *ctx) {
+    (void)ctx;
+#if SIZE_BITS == 32
+    return AWS_OP_SKIP;
+#else
+    const size_t len = 3 * 1024 * 1024 * 1024ULL;
+    const uint8_t *many_zeroes = aws_mem_calloc(allocator, len, sizeof(uint8_t));
+    uint64_t result = aws_checksums_crc64nvme_u64(many_zeroes, len, 0);
+    aws_mem_release(allocator, many_zeroes);
+    ASSERT_HEX_EQUALS(0xa1dddd7c6fd17075, result);
+    return AWS_OP_SUCCESS;
+#endif
+}
+AWS_TEST_CASE(test_large_buffer_crc64, s_test_large_buffer_crc64)

tests/crc_test.c

@@ -183,3 +183,19 @@ static int s_test_crc32(struct aws_allocator *allocator, void *ctx) {
     return res;
 }
 AWS_TEST_CASE(test_crc32, s_test_crc32)
+
+
+static int s_test_large_buffer_crc32(struct aws_allocator *allocator, void *ctx) {
+    (void)ctx;
+#if SIZE_BITS == 32
+    return AWS_OP_SKIP;
+#else
+    const size_t len = 3 * 1024 * 1024 * 1024ULL;
+    const uint8_t *many_zeroes = aws_mem_calloc(allocator, len, sizeof(uint8_t));
+    uint32_t result = aws_checksums_crc32_u64(many_zeroes, len, 0);
+    aws_mem_release(allocator, many_zeroes);
+    ASSERT_HEX_EQUALS(0x480BBE37, result);
+    return AWS_OP_SUCCESS;
+#endif
+}
+AWS_TEST_CASE(test_large_buffer_crc32, s_test_large_buffer_crc32)