Use Github workflows for CI and Release (#1064)

- Github Workflows for CI build, docker and release jobs - Remove docker from gradle - Build and test docker image in workflow --------- Co-authored-by: Usman Saleem <usman@usmans.info>
Consensys · Feb 26, 2025 · dbd20bf · dbd20bf
1 parent 332bc73
commit dbd20bf
Show file tree

Hide file tree

Showing 14 changed files with 404 additions and 704 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,6 @@
+# Ignore everything
+**
+
+# Whitelist specific directories
+!build/distributions/
+!docker/
diff --git a/.github/actions/build-test/action.yml b/.github/actions/build-test/action.yml
@@ -0,0 +1,47 @@
+---
+name: 'build-test'
+description: 'Composite action to build and test Web3Signer'
+inputs:
+  disable-test:
+    description: 'Disable tests'
+    required: false
+    default: 'false'
+runs:
+  using: "composite"
+  steps:
+    - name: Build with unit tests
+      if: ${{ inputs.disable-test  == 'false' }}
+      run: ./gradlew build
+      shell: bash
+
+    - name: Build without unit tests
+      if: ${{ inputs.disable-test == 'true' }}
+      run: ./gradlew build -x test
+      shell: bash
+
+    - name: Integration Tests
+      if: ${{ inputs.disable-test == 'false' }}
+      run: ./gradlew integrationTest
+      shell: bash
+
+    - name: Acceptance Tests
+      if: ${{ inputs.disable-test == 'false' }}
+      run: ./gradlew acceptanceTest
+      shell: bash
+
+    - name: Build Step Output
+      if: always()
+      run: echo "<h2>Test Results</h2>" >> $GITHUB_STEP_SUMMARY
+      shell: bash
+
+    - name: Summarize tests results
+      uses: jeantessier/test-summary-action@v1.0.7
+      if: always()
+
+    - name: Upload build reports
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: build-reports
+        path: '**/build/reports/'
+        retention-days: 7
diff --git a/.github/actions/project-version/action.yml b/.github/actions/project-version/action.yml
@@ -0,0 +1,25 @@
+---
+name: 'project-version'
+description: 'Composite action to determine Web3Signer version'
+outputs:
+  publish-version:
+    description: 'The publish version of the build'
+    value: ${{ steps.checks-version.outputs.publish-version }}
+  specific-version:
+    description: 'The specific version of the build'
+    value: ${{ steps.checks-version.outputs.specific-version }}
+runs:
+  using: "composite"
+  steps:
+    - name: Calculate Version
+      id: checks-version
+      run:  |
+        # Calculate Web3Signer version
+        output=$(./gradlew -q printVersion)
+        # Extract specific-version and publish-version from the output
+        specific_version=$(echo "$output" | grep -oP 'specific-version=\K.*')
+        publish_version=$(echo "$output" | grep -oP 'publish-version=\K.*')
+        # Set the outputs
+        echo "specific-version=$specific_version" >> $GITHUB_OUTPUT
+        echo "publish-version=$publish_version" >> $GITHUB_OUTPUT
+      shell: bash
diff --git a/.github/actions/setup-java-gradle/action.yml b/.github/actions/setup-java-gradle/action.yml
@@ -0,0 +1,14 @@
+---
+name: 'setup-java-gradle'
+description: 'Composite action to Setup Java and Gradle for Web3Signer'
+runs:
+  using: "composite"
+  steps:
+    - name: Set up JDK 21
+      uses: actions/setup-java@v4.2.1
+      with:
+        java-version: 21
+        distribution: 'temurin'
+
+    - name: Setup Gradle
+      uses: gradle/actions/setup-gradle@v4
diff --git a/.github/workflows/ci_main.yml b/.github/workflows/ci_main.yml
@@ -0,0 +1,220 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - '[0-9]+.[0-9]+.[0-9]+(-?.*)'
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  build:
+    # 4 cpu, 16G ram
+    runs-on: ubuntu-24.04
+    environment: dev
+    outputs:
+      publish-version: ${{ steps.project-version.outputs.publish-version }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Setup Java and Gradle
+        id: setup-java-gradle
+        uses: ./.github/actions/setup-java-gradle
+
+      - name: Determine Web3Signer version
+        id: project-version
+        uses: ./.github/actions/project-version
+
+      - name: Build and Test
+        id: build-test
+        uses: ./.github/actions/build-test
+        env:
+          AWS_REGION: ${{ secrets.AWS_REGION }}
+          RO_AWS_ACCESS_KEY_ID: ${{ secrets.RO_AWS_ACCESS_KEY_ID }}
+          RO_AWS_SECRET_ACCESS_KEY: ${{ secrets.RO_AWS_SECRET_ACCESS_KEY }}
+          RW_AWS_ACCESS_KEY_ID: ${{ secrets.RW_AWS_ACCESS_KEY_ID }}
+          RW_AWS_SECRET_ACCESS_KEY: ${{ secrets.RW_AWS_SECRET_ACCESS_KEY }}
+          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
+          AZURE_INVALID_KEY_VAULT_NAME: ${{ secrets.AZURE_INVALID_KEY_VAULT_NAME }}
+          AZURE_KEY_VAULT_NAME: ${{ secrets.AZURE_KEY_VAULT_NAME }}
+          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+        with:
+          disable-test: 'false'
+
+      - name: Create zip and tar distributions
+        run: ./gradlew distTar distZip
+
+      - name: Upload distribution artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: distribution
+          path: build/distributions/
+          retention-days: 1
+          if-no-files-found: error
+
+  docker:
+    runs-on: ubuntu-24.04
+    needs: build
+    environment: docker
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: distribution
+          path: build/distributions/
+
+      - name: Docker Metadata
+        id: docker-metadata
+        run: |
+          # Calculate Docker Tags
+          TAGS="consensys/web3signer:${{ needs.build.outputs.publish-version }}"
+          if [ "${{ needs.build.outputs.publish-version }}" != "develop" ]; then
+            TAGS="$TAGS,consensys/web3signer:latest"
+          fi
+          echo "tags=$TAGS" >> $GITHUB_OUTPUT
+          
+          # Determine Push Flag
+          if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "schedule" ]; then
+            echo "push=false" >> $GITHUB_OUTPUT
+          else
+            echo "push=true" >> $GITHUB_OUTPUT
+          fi
+          
+          # Set Build Args
+          echo "build-date=$(date --utc --rfc-3339=seconds)" >> $GITHUB_OUTPUT
+          echo "vcs-ref=${{ github.sha }}" >> $GITHUB_OUTPUT
+
+      - name: Build and export to docker
+        uses: docker/build-push-action@v6
+        env:
+          DOCKER_BUILD_SUMMARY: false
+        with:
+          file: docker/Dockerfile
+          context: .
+          build-args: |
+            TAR_FILE=./build/distributions/web3signer-${{ needs.build.outputs.publish-version }}.tar.gz
+            BUILD_DATE=${{ steps.docker-metadata.outputs.build-date }}
+            VCS_REF=${{ steps.docker-metadata.outputs.vcs-ref }}
+            VERSION=${{ needs.build.outputs.publish-version }}
+          no-cache: true
+          load: true
+          tags: consensys/web3signer:test
+
+      - name: Get absolute path of reports directory
+        id: get-reports-dir
+        run: echo "path=$(realpath ./build/reports)" >> $GITHUB_OUTPUT
+
+      - name: Run Docker tests
+        run: ./docker/test.sh 'consensys/web3signer:test' '${{ steps.get-reports-dir.outputs.path }}'
+
+      - name: Test Summary
+        if: always()
+        run: |
+          SUMMARY_CONTENT="<h2>Docker Test Summary</h2>\n"
+          SUMMARY_CONTENT+="<details><summary><strong>Details</strong></summary>\n"
+          SUMMARY_CONTENT+="<pre><code>\n"
+          SUMMARY_CONTENT+=$(cat ./build/reports/goss-report.txt)
+          SUMMARY_CONTENT+="\n</code></pre></details>\n"
+          echo -e "$SUMMARY_CONTENT" >> $GITHUB_STEP_SUMMARY  
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@0.29.0
+        with:
+          image-ref: 'consensys/web3signer:test'
+          format: 'sarif'
+          output: 'trivy-results.sarif'
+          vuln-type: 'os,library'
+          severity: 'CRITICAL,HIGH'
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v3
+        if: always()
+        with:
+          sarif_file: 'trivy-results.sarif'
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        if: steps.docker-metadata.outputs.push == 'true'
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push to registry
+        uses: docker/build-push-action@v6
+        with:
+          platforms: linux/amd64,linux/arm64
+          file: docker/Dockerfile
+          context: .
+          build-args: |
+            TAR_FILE=./build/distributions/web3signer-${{ needs.build.outputs.publish-version }}.tar.gz
+            BUILD_DATE=${{ steps.docker-metadata.outputs.build-date }}
+            VCS_REF=${{ steps.docker-metadata.outputs.vcs-ref }}
+            VERSION=${{ needs.build.outputs.publish-version }}
+          push: ${{ steps.docker-metadata.outputs.push }}
+          tags: ${{ steps.docker-metadata.outputs.tags }}
+
+  release:
+    runs-on: ubuntu-24.04
+    needs: [build, docker]
+    if: startsWith(github.ref, 'refs/tags/')
+    environment: release
+    permissions:
+        contents: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: distribution
+          path: build/distributions/
+
+      - name: Generate Checksum
+        working-directory: ./build/distributions
+        run: |
+          shasum -a 256 "web3signer-${{ needs.build.outputs.publish-version }}.tar.gz" > "web3signer-${{ needs.build.outputs.publish-version }}.tar.gz.sha256"
+          shasum -a 256 "web3signer-${{ needs.build.outputs.publish-version }}.zip" > "web3signer-${{ needs.build.outputs.publish-version }}.zip.sha256"
+
+      - name: Determine Prerelease
+        id: determine-prerelease
+        run: |
+          if [[ "${{ needs.build.outputs.publish-version }}" == *-RC* ]]; then
+            echo "prerelease=true" >> $GITHUB_OUTPUT
+          else
+            echo "prerelease=false" >> $GITHUB_OUTPUT
+          fi
+
+      # Create release tag and attach the distribution
+      - name: Web3Signer Release
+        id: release
+        uses: softprops/action-gh-release@v2.2.1
+        with:
+          files: |
+            build/distributions/web3signer-${{ needs.build.outputs.publish-version }}.tar.gz
+            build/distributions/web3signer-${{ needs.build.outputs.publish-version }}.tar.gz.sha256
+            build/distributions/web3signer-${{ needs.build.outputs.publish-version }}.zip
+            build/distributions/web3signer-${{ needs.build.outputs.publish-version }}.zip.sha256
+
+          tag_name: ${{ needs.build.outputs.publish-version }}
+          body: |
+            ** Web3Signer Release **
+          draft: true
+          prerelease: ${{ steps.determine-prerelease.outputs.prerelease }}
+          preserve_order: true
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -1,14 +1,3 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
 name: "CodeQL"
 
 on:
@@ -29,22 +18,15 @@ jobs:
       contents: read
       security-events: write
 
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'java' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
-
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
-          languages: ${{ matrix.language }}
+          languages: java-kotlin
           # If you wish to specify custom queries, you can do so here or in a config file.
           # By default, queries listed here will override any specified in a config file.
           # Prefix the list here with "+" to use these queries and those in the config file.
@@ -56,7 +38,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+        uses: github/codeql-action/autobuild@v3
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -70,4 +52,4 @@ jobs:
       #   make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3