From ae65a9d83ebe3167b3f686d39008688b025962a1 Mon Sep 17 00:00:00 2001 From: Dmitriy Musatkin <63878209+DmitriyMusatkin@users.noreply.github.com> Date: Mon, 9 Dec 2024 22:53:23 -0800 Subject: [PATCH 1/5] switch ci to use roles (#850) --- .github/workflows/ci.yml | 467 ++++++++++++++++++++++----------------- 1 file changed, 270 insertions(+), 197 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e61412cf7..872b9ed9b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,21 +12,23 @@ concurrency: cancel-in-progress: true env: - BUILDER_VERSION: v0.9.67 + BUILDER_VERSION: v0.9.73 BUILDER_SOURCE: releases BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net PACKAGE_NAME: aws-crt-java LINUX_BASE_IMAGE: ubuntu-18-x64 RUN: ${{ github.run_id }}-${{ github.run_number }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + CRT_CI_ROLE: ${{ secrets.CRT_CI_ROLE_ARN }} + AWS_DEFAULT_REGION: us-east-1 AWS_REGION: us-east-1 AWS_DEVICE_FARM_REGION: us-west-2 # Device Farm only available in us-west-2 region +permissions: + id-token: write # This is required for requesting the JWT + jobs: linux-compat: - runs-on: ubuntu-22.04 # latest + runs-on: ubuntu-24.04 # latest strategy: matrix: image: @@ -38,17 +40,21 @@ jobs: - rhel8-x64 #- manylinux2014-x86 until we find 32-bit linux binaries we can use steps: - - name: Checkout Sources - uses: actions/checkout@v4 - with: - submodules: true - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Checkout Sources + uses: actions/checkout@v4 + with: + submodules: true + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} linux-compiler-compat: - runs-on: ubuntu-22.04 # latest + runs-on: ubuntu-24.04 # latest strategy: matrix: compiler: @@ -58,72 +64,85 @@ jobs: - clang-9 - clang-10 - clang-11 + - clang-15 + - clang-17 - gcc-4.8 - gcc-5 - gcc-6 - gcc-7 - gcc-8 + - gcc-11 steps: - - name: Checkout Sources - uses: actions/checkout@v4 - with: - submodules: true - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler }} + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Checkout Sources + uses: actions/checkout@v4 + with: + submodules: true + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler }} linux-arm: name: ARM (${{ matrix.arch }}) - runs-on: ubuntu-22.04 # latest + runs-on: ubuntu-24.04 # latest strategy: fail-fast: false matrix: arch: [armv6, armv7, arm64] steps: - - name: Build ${{ env.PACKAGE_NAME }} - run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} --target=linux-${{ matrix.arch }} --spec=downstream - - + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} --target=linux-${{ matrix.arch }} --spec=downstream linux-fips-x64: - runs-on: ubuntu-22.04 # latest + runs-on: ubuntu-24.04 # latest steps: - - name: Checkout Sources - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 0 - - name: Build ${{ env.PACKAGE_NAME }} - run: | - bash codebuild/cd/test-fips-branch.sh - export CRT_FIPS=ON - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} - - + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Checkout Sources + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 + - name: Build ${{ env.PACKAGE_NAME }} + run: | + bash codebuild/cd/test-fips-branch.sh + export CRT_FIPS=ON + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} linux-fips-armv8: - runs-on: ubuntu-22.04 # latest + runs-on: ubuntu-24.04 # latest steps: - - name: Checkout Sources - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 0 - - name: Build ${{ env.PACKAGE_NAME }} - run: | - bash codebuild/cd/test-fips-branch.sh - export CRT_FIPS=ON - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} --target=linux-arm64 --spec=downstream --cmake-extra=-DCRT_FIPS=ON - - + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Checkout Sources + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 + - name: Build ${{ env.PACKAGE_NAME }} + run: | + bash codebuild/cd/test-fips-branch.sh + export CRT_FIPS=ON + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} --target=linux-arm64 --spec=downstream --cmake-extra=-DCRT_FIPS=ON linux-musl-x64: runs-on: ubuntu-24.04 # latest @@ -133,10 +152,14 @@ jobs: - alpine-3.16-x64 - openwrt-x64-openjdk8 steps: - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} linux-musl-arm: runs-on: ubuntu-24.04 # latest @@ -146,39 +169,51 @@ jobs: - alpine-3.16-armv7 - alpine-3.16-arm64 steps: - - name: Install qemu/docker - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + role-duration-seconds: 14400 # these tests run slow and easily reach default cred expiry, hence change expiry to 4hrs + - name: Install qemu/docker + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} raspberry: - runs-on: ubuntu-22.04 # latest + runs-on: ubuntu-24.04 # latest strategy: matrix: image: - raspbian-bullseye steps: - - name: Install qemu/docker - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} - + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Install qemu/docker + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} windows: runs-on: windows-2022 # latest steps: - - name: Checkout Sources - uses: actions/checkout@v4 - with: - submodules: true - - name: Build ${{ env.PACKAGE_NAME }} + consumers - run: | - python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" - python builder.pyz build -p ${{ env.PACKAGE_NAME }} --spec=downstream + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Checkout Sources + uses: actions/checkout@v4 + with: + submodules: true + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" + python builder.pyz build -p ${{ env.PACKAGE_NAME }} --spec=downstream windows-vc14: runs-on: windows-2019 # windows-2019 is last env with Visual Studio 2015 (v14.0) @@ -186,127 +221,154 @@ jobs: matrix: arch: [x86, x64] steps: - - name: Checkout Sources - uses: actions/checkout@v4 - with: - submodules: true - - name: Build ${{ env.PACKAGE_NAME }} + consumers - env: - AWS_CMAKE_TOOLSET: v140 - run: | - python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" - python builder.pyz build -p ${{ env.PACKAGE_NAME }} downstream + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Checkout Sources + uses: actions/checkout@v4 + with: + submodules: true + - name: Build ${{ env.PACKAGE_NAME }} + consumers + env: + AWS_CMAKE_TOOLSET: v140 + run: | + python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" + python builder.pyz build -p ${{ env.PACKAGE_NAME }} downstream macos: runs-on: macos-14 #latest steps: - - name: Checkout Sources - uses: actions/checkout@v4 - with: - submodules: true - - name: Build ${{ env.PACKAGE_NAME }} + consumers - run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} --spec=downstream - python3 codebuild/macos_compatibility_check.py + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Checkout Sources + uses: actions/checkout@v4 + with: + submodules: true + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} --spec=downstream + python3 codebuild/macos_compatibility_check.py macos-x64: runs-on: macos-14-large #latest steps: - - name: Checkout Sources - uses: actions/checkout@v4 - with: - submodules: true - - name: Build ${{ env.PACKAGE_NAME }} + consumers - run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} --spec=downstream - python3 codebuild/macos_compatibility_check.py + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Checkout Sources + uses: actions/checkout@v4 + with: + submodules: true + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} --spec=downstream + python3 codebuild/macos_compatibility_check.py android: # ubuntu-24.04 comes with Android tooling name: Android runs-on: ubuntu-24.04 # latest steps: - - name: Checkout Sources - uses: actions/checkout@v4 - with: - submodules: true - # Setup JDK 11 - - name: set up JDK 11 - uses: actions/setup-java@v4 - with: - java-version: '11' - distribution: 'temurin' - # Build and publish locally for the test app to find the SNAPSHOT version - - name: Build ${{ env.PACKAGE_NAME }} - run: | - # Manually set -Xmx (max heap size) to something huge (tested 2g and that works, but why not go bigger). - # Only in CI, gradle daemon runs out of memory during "lintAnalyzeDebug" task, unless you specify it this way. - # You'd think Java's default of 25% RAM (ubuntu24 runner has 12g, so max 4g) would be sufficient, but no. - # You'd think setting -Xmx via gradle.properties would help, but no. - ./gradlew :android:crt:build -Dorg.gradle.jvmargs="-Xmx8g" - ./gradlew -PnewVersion="1.0.0-SNAPSHOT" :android:crt:publishToMavenLocal - # Setup files required by test app for Device Farm testing - - name: Setup Android Test Files - run: | - cd src/test/android/testapp/src/main/assets - python3 -m pip install boto3 - python3 ./android_file_creation.py - - name: Build Test App - run: | - cd src/test/android/testapp - ../../../../gradlew assembledebug - ../../../../gradlew assembleAndroidTest - - name: Device Farm Tests Highly Available - run: | - echo "Running Device Farm Python Script" - python3 ./.github/workflows/run_android_ci.py \ - --run_id ${{ github.run_id }} \ - --run_attempt ${{ github.run_attempt }} \ - --project_arn $(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/DeviceFarm/ProjectArn" --query "SecretString" | cut -f5 -d\" | cut -f1 -d'\') \ - --device_pool_arn $(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/DeviceFarm/DevicePoolArn" --query "SecretString" | cut -f5 -d\" | cut -f1 -d'\') \ - --device_pool highly_available - - name: Device Farm Tests Android 8.0.0 - run: | - echo "Running Device Farm Python Script" - python3 ./.github/workflows/run_android_ci.py \ - --run_id ${{ github.run_id }} \ - --run_attempt ${{ github.run_attempt }} \ - --project_arn $(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/DeviceFarm/ProjectArn" --query "SecretString" | cut -f5 -d\" | cut -f1 -d'\') \ - --device_pool_arn $(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/DeviceFarm/DevicePoolArnAndroid8" --query "SecretString" | cut -f5 -d\" | cut -f1 -d'\') \ - --device_pool android_8 + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Checkout Sources + uses: actions/checkout@v4 + with: + submodules: true + # Setup JDK 11 + - name: set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + # Build and publish locally for the test app to find the SNAPSHOT version + - name: Build ${{ env.PACKAGE_NAME }} + run: | + # Manually set -Xmx (max heap size) to something huge (tested 2g and that works, but why not go bigger). + # Only in CI, gradle daemon runs out of memory during "lintAnalyzeDebug" task, unless you specify it this way. + # You'd think Java's default of 25% RAM (ubuntu24 runner has 12g, so max 4g) would be sufficient, but no. + # You'd think setting -Xmx via gradle.properties would help, but no. + ./gradlew :android:crt:build -Dorg.gradle.jvmargs="-Xmx8g" + ./gradlew -PnewVersion="1.0.0-SNAPSHOT" :android:crt:publishToMavenLocal + # Setup files required by test app for Device Farm testing + - name: Setup Android Test Files + run: | + cd src/test/android/testapp/src/main/assets + python3 -m pip install boto3 + python3 ./android_file_creation.py + - name: Build Test App + run: | + cd src/test/android/testapp + ../../../../gradlew assembledebug + ../../../../gradlew assembleAndroidTest + - name: Device Farm Tests Highly Available + run: | + echo "Running Device Farm Python Script" + python3 ./.github/workflows/run_android_ci.py \ + --run_id ${{ github.run_id }} \ + --run_attempt ${{ github.run_attempt }} \ + --project_arn $(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/DeviceFarm/ProjectArn" --query "SecretString" | cut -f5 -d\" | cut -f1 -d'\') \ + --device_pool_arn $(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/DeviceFarm/DevicePoolArn" --query "SecretString" | cut -f5 -d\" | cut -f1 -d'\') \ + --device_pool highly_available + - name: Device Farm Tests Android 8.0.0 + run: | + echo "Running Device Farm Python Script" + python3 ./.github/workflows/run_android_ci.py \ + --run_id ${{ github.run_id }} \ + --run_attempt ${{ github.run_attempt }} \ + --project_arn $(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/DeviceFarm/ProjectArn" --query "SecretString" | cut -f5 -d\" | cut -f1 -d'\') \ + --device_pool_arn $(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/DeviceFarm/DevicePoolArnAndroid8" --query "SecretString" | cut -f5 -d\" | cut -f1 -d'\') \ + --device_pool android_8 # check that docs can still build check-docs: runs-on: ubuntu-22.04 # use same version as docs.yml steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Check docs - run: | - ./make-docs.sh + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - uses: actions/checkout@v4 + with: + submodules: true + - name: Check docs + run: | + ./make-docs.sh check-submodules: - runs-on: ubuntu-22.04 # latest + runs-on: ubuntu-24.04 # latest steps: - - name: Checkout Source - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 0 - - name: Check Submodules - # note: using "@main" because "@${{env.BUILDER_VERSION}}" doesn't work - # https://github.com/actions/runner/issues/480 - uses: awslabs/aws-crt-builder/.github/actions/check-submodules@main - + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Checkout Source + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 + - name: Check Submodules + # note: using "@main" because "@${{env.BUILDER_VERSION}}" doesn't work + # https://github.com/actions/runner/issues/480 + uses: awslabs/aws-crt-builder/.github/actions/check-submodules@main localhost-test-linux: - runs-on: ubuntu-22.04 # latest + runs-on: ubuntu-24.04 # latest steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Checkout uses: actions/checkout@v4 with: @@ -325,6 +387,10 @@ jobs: localhost-test-macos: runs-on: macos-14 # latest steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Checkout uses: actions/checkout@v4 with: @@ -346,6 +412,10 @@ jobs: localhost-test-win: runs-on: windows-2022 # latest steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Checkout uses: actions/checkout@v4 with: @@ -362,7 +432,6 @@ jobs: python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" python builder.pyz localhost-test -p ${{ env.PACKAGE_NAME }} downstream - GraalVM: runs-on: ${{ matrix.os }} strategy: @@ -371,17 +440,21 @@ jobs: os: [macos-latest, windows-latest, ubuntu-latest] java-version: ['22', '21', '17'] steps: - - name: Checkout Sources - uses: actions/checkout@v4 - with: - submodules: true - - name: Setup GraalVM - uses: graalvm/setup-graalvm@v1 - with: - java-version: ${{ matrix.java-version }} - distribution: 'graalvm' - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Build ${{ env.PACKAGE_NAME }} + consumers - run: | - python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" - python builder.pyz build -p ${{ env.PACKAGE_NAME }} --variant=graalvm downstream + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Checkout Sources + uses: actions/checkout@v4 + with: + submodules: true + - name: Setup GraalVM + uses: graalvm/setup-graalvm@v1 + with: + java-version: ${{ matrix.java-version }} + distribution: 'graalvm' + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" + python builder.pyz build -p ${{ env.PACKAGE_NAME }} --variant=graalvm downstream From f01fbf0a3030e13dec31063cd0a5d85c91ad3845 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 12 Dec 2024 09:10:17 -0800 Subject: [PATCH 2/5] latest submodles. (#854) --- crt/aws-c-common | 2 +- crt/aws-c-s3 | 2 +- src/main/java/software/amazon/awssdk/crt/s3/ChecksumConfig.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crt/aws-c-common b/crt/aws-c-common index fadfef492..7a6f5df20 160000 --- a/crt/aws-c-common +++ b/crt/aws-c-common @@ -1 +1 @@ -Subproject commit fadfef492042ae53387d4369a6de652c930a5be4 +Subproject commit 7a6f5df201cb4b1910932ea3221de83edaa39880 diff --git a/crt/aws-c-s3 b/crt/aws-c-s3 index 337155f6c..33b8cd07c 160000 --- a/crt/aws-c-s3 +++ b/crt/aws-c-s3 @@ -1 +1 @@ -Subproject commit 337155f6c07d39e61234e705ed6e58c31d4841eb +Subproject commit 33b8cd07c53068e1408f85ae956c44c9753df0f3 diff --git a/src/main/java/software/amazon/awssdk/crt/s3/ChecksumConfig.java b/src/main/java/software/amazon/awssdk/crt/s3/ChecksumConfig.java index 4e5f2f30d..d3d9e1dc2 100644 --- a/src/main/java/software/amazon/awssdk/crt/s3/ChecksumConfig.java +++ b/src/main/java/software/amazon/awssdk/crt/s3/ChecksumConfig.java @@ -114,7 +114,7 @@ public boolean getValidateChecksum() { * The list of algorithms for user to pick up when validate the checksum. Client * will pick up the algorithm from the list with the priority based on * performance, and the algorithm sent by server. The priority based on - * performance is [CRC32C, CRC32, SHA1, SHA256]. + * performance is [CRC64NVME, CRC32C, CRC32, SHA1, SHA256]. * * If the response checksum was validated by client, the result will indicate * which algorithm was picked. From 20dd76c45a4458e49c2a85c65b5c3b0d3fb777be Mon Sep 17 00:00:00 2001 From: Dmitriy Musatkin <63878209+DmitriyMusatkin@users.noreply.github.com> Date: Fri, 20 Dec 2024 21:01:19 -0800 Subject: [PATCH 3/5] bump crt versions (#857) --- crt/aws-c-s3 | 2 +- crt/s2n | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crt/aws-c-s3 b/crt/aws-c-s3 index 33b8cd07c..21a4ab4da 160000 --- a/crt/aws-c-s3 +++ b/crt/aws-c-s3 @@ -1 +1 @@ -Subproject commit 33b8cd07c53068e1408f85ae956c44c9753df0f3 +Subproject commit 21a4ab4dacd1eaf61e6c044c7a299d68fd4c1876 diff --git a/crt/s2n b/crt/s2n index 493b77167..2e79e7efe 160000 --- a/crt/s2n +++ b/crt/s2n @@ -1 +1 @@ -Subproject commit 493b77167dc367c394de23cfe78a029298e2a254 +Subproject commit 2e79e7efeb26f06eb59a1d4f3444ea63fc3e20c3 From 5593e88454b6106d1eca210e9a43404a06c0e530 Mon Sep 17 00:00:00 2001 From: Bret Ambrose Date: Mon, 6 Jan 2025 13:02:05 -0800 Subject: [PATCH 4/5] Flaky test pass1 (#855) * Update shared subscription test to remove invalid assertions on a property that does not necessarily hold (all subscriptions receive messages) * Update SelfPubSub test to not use counting variables that are pointless (they won't ever be wrong) as well as unsafe from race conditions (they are set and read from different threads with no explicit synchronizations, also the future that was waited upon strictly happens-before the setting of the counter which leads to consistency issues) * Update the double client id connection failure test to be more forgiving to potential race conditions between the two clients fighting over the client id * Adds a delay between subscribe and disconnect in the will test. Did not get an encouraging answer when the service team was queried about whether subscribe completion had any eventual consistency issues relative to a subsequent publish (connection will). Co-authored-by: Bret Ambrose --- .../awssdk/crt/test/Mqtt5ClientTest.java | 15 +++++++-------- .../awssdk/crt/test/SelfPubSubTest.java | 19 ------------------- 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/src/test/java/software/amazon/awssdk/crt/test/Mqtt5ClientTest.java b/src/test/java/software/amazon/awssdk/crt/test/Mqtt5ClientTest.java index fda1139f5..1fc36ea73 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Mqtt5ClientTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Mqtt5ClientTest.java @@ -1132,9 +1132,10 @@ public void onConnectionSuccess(Mqtt5Client client, OnConnectionSuccessReturn on @Override public void onConnectionFailure(Mqtt5Client client, OnConnectionFailureReturn onConnectionFailureReturn) { - connectedFuture.completeExceptionally(new Exception( - "[" + client_name + "] Could not connect! Error code is: " + onConnectionFailureReturn.getErrorCode() - )); + // failing the connected future here is not valid from a race condition standpoint. It is possible that + // the interrupting client itself gets interrupted and fails to fully connect due to the original client + // interrupting it. Eventually it will succeed (briefly) as the two clients fight over the client id + // with increasing reconnect backoff. } @Override @@ -2083,6 +2084,9 @@ public void Op_UC4() { eventsTwo.connectedFuture.get(OPERATION_TIMEOUT_TIME, TimeUnit.SECONDS); subscriber.subscribe(subscribePacketBuilder.build()).get(OPERATION_TIMEOUT_TIME, TimeUnit.SECONDS); + // Paranoid about service-side eventual consistency. Add a wait to reduce chances of a missed will publish. + Thread.sleep(2000); + publisher.stop(disconnectPacketBuilder.build()); publishEvents.publishReceivedFuture.get(OPERATION_TIMEOUT_TIME, TimeUnit.SECONDS); @@ -2171,11 +2175,6 @@ public void Op_SharedSubscription() { // Wait a little longer just to ensure that no packets beyond expectations are arrived. publishEvents.afterCompletionFuture.get(OPERATION_TIMEOUT_TIME, TimeUnit.SECONDS); - // Check that both clients received packets. - // PublishEvents_Futured_Counted also checks for duplicated packets, so this one assert is enough - // to ensure that AWS IoT Core sent different packets to different subscribers. - assertTrue(publishEvents.clientsReceived.size() == 2); - subscriberOneClient.stop(); subscriberTwoClient.stop(); publisherClient.stop(); diff --git a/src/test/java/software/amazon/awssdk/crt/test/SelfPubSubTest.java b/src/test/java/software/amazon/awssdk/crt/test/SelfPubSubTest.java index 18906a2f8..5de86f2ed 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/SelfPubSubTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/SelfPubSubTest.java @@ -34,9 +34,6 @@ public SelfPubSubTest() { static final String TEST_TOPIC = "publish/me/senpai/" + UUID.randomUUID().toString(); static final String TEST_PAYLOAD = "PUBLISH ME! SHINY AND CHROME!"; - int pubsAcked = 0; - int subsAcked = 0; - @Test public void testPubSub() { skipIfNetworkUnavailable(); @@ -65,27 +62,21 @@ public void testPubSub() { CompletableFuture subscribed = connection.subscribe(TEST_TOPIC, QualityOfService.AT_LEAST_ONCE, messageHandler); - subscribed.thenApply(unused -> subsAcked++); int packetId = subscribed.get(); assertNotSame(0, packetId); - assertEquals("Single subscription", 1, subsAcked); MqttMessage message = new MqttMessage(TEST_TOPIC, TEST_PAYLOAD.getBytes(), QualityOfService.AT_LEAST_ONCE, false); CompletableFuture published = connection.publish(message); - published.thenApply(unused -> pubsAcked++); packetId = published.get(); assertNotSame(0, packetId); - assertEquals("Published", 1, pubsAcked); published = connection.publish(message); - published.thenApply(unused -> pubsAcked++); packetId = published.get(); assertNotSame(0, packetId); - assertEquals("Published", 2, pubsAcked); MqttMessage received = receivedFuture.get(); assertEquals("Received", message.getTopic(), received.getTopic()); @@ -94,11 +85,9 @@ public void testPubSub() { assertEquals("Received", message.getRetain(), received.getRetain()); CompletableFuture unsubscribed = connection.unsubscribe(TEST_TOPIC); - unsubscribed.thenApply(unused -> subsAcked--); packetId = unsubscribed.get(); assertNotSame(0, packetId); - assertEquals("No Subscriptions", 0, subsAcked); } catch (Exception ex) { fail(ex.getMessage()); } @@ -142,33 +131,25 @@ public void testPubSubOnMessage() { try { CompletableFuture subscribed = connection.subscribe(TEST_TOPIC, QualityOfService.AT_LEAST_ONCE); - subscribed.thenApply(unused -> subsAcked++); int packetId = subscribed.get(); assertNotSame(0, packetId); - assertEquals("Single subscription", 1, subsAcked); MqttMessage message = new MqttMessage(TEST_TOPIC, TEST_PAYLOAD.getBytes(), QualityOfService.AT_LEAST_ONCE); CompletableFuture published = connection.publish(message); - published.thenApply(unused -> pubsAcked++); packetId = published.get(); assertNotSame(0, packetId); - assertEquals("Published", 1, pubsAcked); published = connection.publish(message); - published.thenApply(unused -> pubsAcked++); packetId = published.get(); assertNotSame(0, packetId); - assertEquals("Published", 2, pubsAcked); CompletableFuture unsubscribed = connection.unsubscribe(TEST_TOPIC); - unsubscribed.thenApply(unused -> subsAcked--); packetId = unsubscribed.get(); assertNotSame(0, packetId); - assertEquals("No Subscriptions", 0, subsAcked); } catch (Exception ex) { fail(ex.getMessage()); } From 0540e171fc3307ab0502eaca790cbcf27edcc8dc Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 9 Jan 2025 09:34:14 -0800 Subject: [PATCH 5/5] Fix connectionAcquisitionTimeoutInMilliseconds from Int to Long (#858) --- .../awssdk/crt/http/HttpClientConnectionManagerOptions.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java index 124d63a6f..f861569a7 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java @@ -303,7 +303,7 @@ public long getConnectionAcquisitionTimeoutInMilliseconds() { * @param connectionAcquisitionTimeoutInMilliseconds timeout in milliseconds. * @return this */ - public HttpClientConnectionManagerOptions withConnectionAcquisitionTimeoutInMilliseconds(int connectionAcquisitionTimeoutInMilliseconds) { + public HttpClientConnectionManagerOptions withConnectionAcquisitionTimeoutInMilliseconds(long connectionAcquisitionTimeoutInMilliseconds) { this.connectionAcquisitionTimeoutInMilliseconds = connectionAcquisitionTimeoutInMilliseconds; return this; } @@ -325,7 +325,7 @@ public long getMaxPendingConnectionAcquisitions() { * @param maxPendingConnectionAcquisitions maximum pending acquisitions allowed * @return this */ - public HttpClientConnectionManagerOptions withMaxPendingConnectionAcquisitions(int maxPendingConnectionAcquisitions) { + public HttpClientConnectionManagerOptions withMaxPendingConnectionAcquisitions(long maxPendingConnectionAcquisitions) { this.maxPendingConnectionAcquisitions = maxPendingConnectionAcquisitions; return this; }