Merge branch 'main' into RequestResponseWorkspace

.github/workflows/ci.yml

@@ -358,6 +358,7 @@ jobs:
         cd crt/aws-c-http/tests/py_localhost/
         Start-Process -NoNewWindow python .\server.py
         Start-Process -NoNewWindow python .\non_tls_server.py
+        cd ../../../../
         python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
         python builder.pyz localhost-test -p ${{ env.PACKAGE_NAME }} downstream
 

CMakeLists.txt

@@ -56,46 +56,47 @@ if (BUILD_DEPS)
     set(BUILD_TESTING OFF)
     add_subdirectory(crt/aws-c-common)
     if (UNIX AND NOT APPLE)
-        set(BUILD_LIBSSL OFF CACHE BOOL "Don't need libssl, only need libcrypto")
+        include(AwsPrebuildDependency)
+
+        set(AWSLC_CMAKE_ARGUMENTS
+            -DBUILD_LIBSSL=OFF
+            -DBUILD_TESTING=OFF
+        )
+
         message("Build with FIPS: " ${CRT_FIPS})
         if (CRT_FIPS)
-            set(FIPS ON CACHE BOOL "FIPS compliance")
-            set(PERL_EXECUTABLE "perl")
+            list(APPEND AWSLC_CMAKE_ARGUMENTS -DFIPS=ON)
+            list(APPEND AWSLC_CMAKE_ARGUMENTS -DDISABLE_PERL=OFF)
+
+            # Pick up GO_PATH env-var, set by aws-crt-builder when cross-compiling, see:
+            # https://github.com/awslabs/aws-crt-builder/blob/31307c808ed9f2ea1eb16503b25a9b582f886481/builder/imports/golang.py#L84
+            # https://github.com/awslabs/aws-crt-builder/blob/31307c808ed9f2ea1eb16503b25a9b582f886481/builder/actions/cmake.py#L110
             if (DEFINED ENV{GO_PATH})
-                set(GO_EXECUTABLE $ENV{GO_PATH}/go)
+                list(APPEND AWSLC_CMAKE_ARGUMENTS -DGO_EXECUTABLE=$ENV{GO_PATH}/go)
                 message(STATUS "Overriding GO_EXECUTABLE to ${GO_EXECUTABLE}")
             endif()
         else()
-            set(DISABLE_PERL ON CACHE BOOL "Disable codegen")
-            set(DISABLE_GO ON CACHE BOOL "Disable codegen")
+            list(APPEND AWSLC_CMAKE_ARGUMENTS -DDISABLE_PERL=ON)  # Disable codegen
+            list(APPEND AWSLC_CMAKE_ARGUMENTS -DDISABLE_GO=ON)  # Disable codegen
         endif()
 
         if(CMAKE_C_COMPILER_ID MATCHES "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS "5.0")
-            set(DISABLE_PERL OFF CACHE BOOL "Build with Perl to avoid using pre-compiled binary with AVX512")
-            set(PERL_EXECUTABLE "perl")
-            set(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX ON CACHE BOOL "Disable AVX512 on old GCC that not supports it")
+            # Disable AVX512 on old GCC that not supports it
+            list(APPEND AWSLC_CMAKE_ARGUMENTS -DMY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX=ON)
         endif()
 
-        # temporarily disable certain warnings as errors for the aws-lc build
-        set(OLD_CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
-        if (NOT MSVC)
-            check_c_compiler_flag(-Wno-stringop-overflow HAS_WNO_STRINGOP_OVERFLOW)
-            if (HAS_WNO_STRINGOP_OVERFLOW)
-                set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-stringop-overflow")
-            endif()
-
-            check_c_compiler_flag(-Wno-array-parameter HAS_WNO_ARRAY_PARAMETER)
-            if (HAS_WNO_ARRAY_PARAMETER)
-                set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-array-parameter")
-            endif()
+        if (ANDROID)
+            list(APPEND AWSLC_CMAKE_ARGUMENTS -DANDROID_DEPS_DIR=${ANDROID_DEPS_DIR})
+            list(APPEND AWSLC_CMAKE_ARGUMENTS -DAWS_LIBRARY_OUTPUT_DIR="${AWS_LIBRARY_OUTPUT_DIR}")
         endif()
 
-        add_subdirectory(crt/aws-lc)
-
-        # restore previous build flags
-        set(CMAKE_C_FLAGS "${OLD_CMAKE_C_FLAGS}")
+        # s2n-tls uses libcrypto during its configuration, so we need to prebuild aws-lc.
+        aws_prebuild_dependency(
+            DEPENDENCY_NAME AWSLC
+            SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/crt/aws-lc
+            CMAKE_ARGUMENTS ${AWSLC_CMAKE_ARGUMENTS}
+        )
 
-        set(SEARCH_LIBCRYPTO OFF CACHE BOOL "Let S2N use libcrypto from AWS-LC.")
         set(UNSAFE_TREAT_WARNINGS_AS_ERRORS OFF CACHE BOOL "Disable warnings-as-errors when building S2N")
         add_subdirectory(crt/s2n)
     endif()
@@ -120,7 +121,7 @@ include(AwsPlatformDetect)
 include(AwsSharedLibSetup)
 include(AwsCRuntime)
 
-if (CRT_FIPS AND NOT FIPS)
+if (CRT_FIPS AND NOT AWSLC_PREBUILT)
     message(FATAL_ERROR "CRT_FIPS can only be set when build with aws-lc.")
 endif()
 

README.md

@@ -161,13 +161,15 @@ Platforms without FIPS compliance are also included in this jar, for compatibili
 > [!WARNING]
 > The classifier, and platforms with FIPS compliance are subject to change in the future.
 
+Platforms with FIPS compliance use [AWS-LC](https://github.com/aws/aws-lc) as their cryptographic module ([NIST Certificate #4816](https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/4816)).
+
 ## GraalVM support
 
 Since version v0.29.20, GraalVM native image was supported. You can compile your application with AWS CRT in a GraalVM native image project without any additional configuration.
 
 Since version v0.31.1, GraalVM support was updated. Instead of packaging the JNI shared libraries with native image as resources, the corresponding shared lib will be written to the same directory as the native image.
 In this way, it reduces the native image size around 30% (142 MB to 101 MB for a sample application), and avoids the extra loading time needed for extracting the JNI lib to the temporary path for load. No additional configuration needed.
-> [!NOTE] 
+> [!NOTE]
 > The JNI shared lib must be in the same directory as the GraalVM native image. If you move the native image, you must move this file too. It is `aws-crt-jni.dll` on Windows, `libaws-crt-jni.dylib` on macOS, and `libaws-crt-jni.so` on Unix.
 
 ## System Properties

codebuild/cd/generic-unix-build.sh

@@ -29,10 +29,20 @@ chmod a+x builder
 GIT_TAG=$(git describe --tags)
 
 ./builder build -p aws-crt-java --target=$AWS_CRT_TARGET run_tests=false
-# Builder corss-compiles the shared lib to `target/cmake-build/aws-crt-java/`, move it to the expected path for mvn to generate the jar.
-mv target/cmake-build/aws-crt-java/* target/cmake-build/
+
+# When cross-compiling with builder, the shared lib gets an extra "/aws-crt-java/" in its path.
+# Move it to expected location.
+if [ -d target/cmake-build/aws-crt-java/lib ]; then
+    mv target/cmake-build/aws-crt-java/lib target/cmake-build/lib
+fi
+
+# Double check that shared lib is where we expect
+if ! find target/cmake-build/lib -type f -name "*.so" | grep -q .; then
+  echo "No .so files found"
+  exit 1
+fi
 
 JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 mvn -B package -DskipTests -Dshared-lib.skip=true -Dcrt.classifier=$CLASSIFIER
 
-aws s3 cp --recursive --include "*.so" target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/lib
+aws s3 cp --recursive --exclude "*" --include "*.so" target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/lib
 aws s3 cp target/ s3://aws-crt-java-pipeline/${GIT_TAG}/jar/ --recursive --exclude "*" --include "aws-crt*.jar"

codebuild/cd/linux-aarch64-fips-build.sh

@@ -18,8 +18,19 @@ chmod a+x builder
 GIT_TAG=$(git describe --tags)
 
 ./builder build -p aws-crt-java run_tests=false --target=linux-arm64 --cmake-extra=-DCRT_FIPS=ON
-mv target/cmake-build/aws-crt-java/* target/cmake-build/
+
+# When cross-compiling with builder, the shared lib gets an extra "/aws-crt-java/" in its path.
+# Move it to expected location.
+if [ -d target/cmake-build/aws-crt-java/lib ]; then
+    mv target/cmake-build/aws-crt-java/lib target/cmake-build/lib
+fi
+
+# Double check that shared lib is where we expect
+if ! find target/cmake-build/lib -type f -name "*.so" | grep -q .; then
+  echo "No .so files found"
+  exit 1
+fi
 
 JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 mvn -B package -DskipTests -Dshared-lib.skip=true -Dcrt.classifier=linux-aarch_64-fips
 
-aws s3 cp --recursive --include "*.so" target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/fips_lib
+aws s3 cp --recursive --exclude "*" --include "*.so" target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/fips_lib

codebuild/cd/manylinux-x64-fips-build.yml

@@ -11,9 +11,8 @@ phases:
       - git submodule update --init
       # double check aws-lc is the FIPS approved branch.
       - bash ./codebuild/cd/test-fips-branch.sh
-      - curl -OL https://go.dev/dl/go1.21.6.linux-amd64.tar.gz && mkdir ./go
-      - tar -C ./go -xvf go1.21.6.linux-amd64.tar.gz
-      - export PATH=$PATH:./go/go/bin
+      # aws-lc FIPS build requires golang for codegen
+      - yum install -y golang
       - mvn -B package -DskipTests -Dcrt.classifier=linux-x86_64-fips -Dcmake.crt_fips=ON
 
   post_build:

codebuild/cd/musl-linux-build.sh

@@ -23,5 +23,11 @@ docker container prune -f
 # Upload the artifacts to S3
 export GIT_TAG=$(git describe --tags)
 
-aws s3 cp --recursive --include "*.so" target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/lib
+# Double check that shared lib is where we expect
+if ! find target/cmake-build/lib -type f -name "*.so" | grep -q .; then
+  echo "No .so files found"
+  exit 1
+fi
+
+aws s3 cp --recursive --exclude "*" --include "*.so" target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/lib
 aws s3 cp target/ s3://aws-crt-java-pipeline/${GIT_TAG}/jar/ --recursive --exclude "*" --include "aws-crt*.jar"

src/main/java/software/amazon/awssdk/crt/s3/ChecksumAlgorithm.java

@@ -20,7 +20,9 @@ public enum ChecksumAlgorithm {
 
     SHA1(3),
 
-    SHA256(4);
+    SHA256(4),
+
+    CRC64NVME(5);
 
     ChecksumAlgorithm(int nativeValue) {
         this.nativeValue = nativeValue;
@@ -46,6 +48,7 @@ private static Map<Integer, ChecksumAlgorithm> buildEnumMapping() {
         enumMapping.put(CRC32.getNativeValue(), CRC32);
         enumMapping.put(SHA1.getNativeValue(), SHA1);
         enumMapping.put(SHA256.getNativeValue(), SHA256);
+        enumMapping.put(CRC64NVME.getNativeValue(), CRC64NVME);
         return enumMapping;
     }
 

src/native/event_loop_group.c

@@ -5,6 +5,7 @@
 
 #include <jni.h>
 
+#include <aws/common/shutdown_types.h>
 #include <aws/io/event_loop.h>
 #include <aws/io/logging.h>
 

src/native/http2_stream_manager.c

@@ -82,8 +82,9 @@ static void s_on_stream_manager_shutdown_complete_callback(void *user_data) {
     }
 
     /* We're done with this wrapper, free it. */
+    JavaVM *jvm = binding->jvm;
     s_destroy_manager_binding(binding, env);
-    aws_jni_release_thread_env(binding->jvm, env);
+    aws_jni_release_thread_env(jvm, env);
     /********** JNI ENV RELEASE **********/
 }
 
@@ -340,8 +341,9 @@ static void s_on_stream_acquired(struct aws_http_stream *stream, int error_code,
         }
     }
     AWS_FATAL_ASSERT(!aws_jni_check_and_clear_exception(env));
+    JavaVM *jvm = callback_data->jvm;
     s_cleanup_sm_acquire_stream_callback_data(callback_data, env);
-    aws_jni_release_thread_env(callback_data->jvm, env);
+    aws_jni_release_thread_env(jvm, env);
     /********** JNI ENV RELEASE **********/
 }
 

src/native/http_request_response.c

@@ -306,8 +306,9 @@ void aws_java_http_stream_on_stream_destroy_fn(void *user_data) {
         return;
     }
     /* Native stream destroyed, release the binding. */
+    JavaVM *jvm = binding->jvm;
     aws_http_stream_binding_release(env, binding);
-    aws_jni_release_thread_env(binding->jvm, env);
+    aws_jni_release_thread_env(jvm, env);
     /********** JNI ENV RELEASE **********/
 }