Merge branch 'main' into chore/498-v5-techdebt-cleanup

.github/actions/compute-docker-images-hash/action.yml

@@ -0,0 +1,35 @@
+name: 'Compute Docker Images Hash'
+description: 'Compute Docker Images Hash for Cache Key'
+
+outputs:
+  docker_images_hash:
+    description: 'Docker Images Hash'
+    value: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}
+  docker_images_trimmed:
+    description: 'Trimmed List of Docker Images'
+    value: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_TRIMMED }}
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Compute docker images hash
+      id: compute-docker-images-hash
+      shell: bash
+      run: |
+        docker_images_traces_v1=$(docker compose -f docker/compose.yml -f docker/compose-local-dev.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
+        docker_images_traces_v2=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
+        docker_images=$(echo "$docker_images_traces_v1 $docker_images_traces_v2")
+        docker_images_unique=$(echo $docker_images | xargs -n1 | sort -u)
+        docker_images_trimmed=$(echo $docker_images_unique | xargs -n1 | grep -Ev "linea-postman|linea-coordinator|linea-transaction-exclusion-api|linea-traces-api-facade|linea-prover" | xargs)
+        echo DOCKER_IMAGES=$(echo $docker_images) >> $GITHUB_OUTPUT
+        echo DOCKER_IMAGES_UNIQUE=$(echo $docker_images_unique) >> $GITHUB_OUTPUT
+        echo DOCKER_IMAGES_TRIMMED=$(echo $docker_images_trimmed) >> $GITHUB_OUTPUT
+        echo DOCKER_IMAGES_HASH=$(echo -n $docker_images_trimmed | sha256sum | awk '{print $1}') >> $GITHUB_OUTPUT
+    - name: Show docker images hash
+      shell: bash
+      id: show-docker-images-hash
+      run: |
+        echo "DOCKER_IMAGES: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES }}"
+        echo "DOCKER_IMAGES_UNIQUE: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_UNIQUE }}"
+        echo "DOCKER_IMAGES_TRIMMED: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_TRIMMED }}"
+        echo "DOCKER_IMAGES_HASH: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}"

.github/actions/restore-docker-images/action.yml

@@ -0,0 +1,25 @@
+name: 'Restore Cached Docker Images'
+description: 'Restore Cached Docker Images'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Compute docker images hash
+      id: compute-docker-images-hash
+      uses: ./.github/actions/compute-docker-images-hash
+    - name: Create cached docker images folder
+      shell: bash
+      run: |
+        mkdir -p ~/docker-images-cached
+    - name: Cache common docker images
+      id: cache-common-docker-images
+      continue-on-error: true
+      uses: actions/cache/restore@v4.2.0
+      with:
+        path: ~/docker-images-cached
+        key: docker-images-cache-${{ steps.compute-docker-images-hash.outputs.docker_images_hash }}
+    - name: Load docker images
+      shell: bash
+      if: steps.cache-common-docker-images.outputs.cache-hit == 'true'
+      run: |
+        docker load -i ~/docker-images-cached/docker-images.tar

.github/workflows/all-tools.yml

@@ -16,7 +16,7 @@ on:
 
 jobs:
   changes:
-    runs-on: [self-hosted, ubuntu-20.04, X64, small]
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-small
     name: Filter commit changes
     outputs:
       all-tools: ${{ steps.filter.outputs['all-tools'] }}
@@ -39,7 +39,7 @@ jobs:
     uses: ./.github/workflows/reuse-store-image-name-and-tags.yml
 
   check_image_tags_exist:
-    runs-on: [self-hosted, ubuntu-20.04, X64, small]
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-small
     name: Check image tags exist
     needs: [ changes, store_image_name_and_tags ]
     if: ${{ needs.changes.outputs['all-tools'] == 'false' }}
@@ -53,7 +53,7 @@ jobs:
           image_name: consensys/linea-alltools
 
   all-tools-tag-only:
-    runs-on: [self-hosted, ubuntu-20.04, X64, small]
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-small
     name: All tools tag only
     needs: [ changes, store_image_name_and_tags, check_image_tags_exist ]
     if: ${{ github.event_name != 'pull_request' && needs.changes.outputs['all-tools'] == 'false' }}
@@ -77,7 +77,8 @@ jobs:
   build-and-publish:
     needs: [ changes, store_image_name_and_tags, all-tools-tag-only ]
     if: ${{ always() && (needs.changes.outputs['all-tools'] == 'true' || needs.all-tools-tag-only.result != 'success' || needs.all-tools-tag-only.outputs.image_tagged != 'true') }}
-    runs-on: [self-hosted, ubuntu-20.04, X64, small]
+    # ~0.5 mins saved vs small
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med
     env:
       COMMIT_TAG: ${{ needs.store_image_name_and_tags.outputs.commit_tag }}
       DEVELOP_TAG: ${{ needs.store_image_name_and_tags.outputs.develop_tag }}

.github/workflows/bridge-ui-e2e-tests.yml

@@ -16,7 +16,7 @@ on:
 
 jobs:
   run-e2e-tests:
-    runs-on: [self-hosted, ubuntu-20.04, X64, medium]
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med
     steps:
       - name: Checkout
         uses: actions/checkout@v4

.github/workflows/bridge-ui-publish.yml

@@ -5,18 +5,21 @@ on:
     branches:
       - main
     paths:
+      - '.github/workflows/bridge-ui-publish.yml'
       - 'bridge-ui/**'
       - '!bridge-ui/**/*.md'
   push:
     branches:
       - main
     paths:
+      - '.github/workflows/bridge-ui-publish.yml'
       - 'bridge-ui/**'
       - '!bridge-ui/**/*.md'
 
 jobs:
   publish:
-    runs-on: [self-hosted, ubuntu-20.04, X64, small]
+    # ~1 min saved vs small
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med
     env:
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/cache-docker-images.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   check-dockerhub-secrets-present:
-    runs-on: [self-hosted, ubuntu-20.04, X64, small]
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-small
     env:
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -24,7 +24,7 @@ jobs:
   changes:
     needs: [ check-dockerhub-secrets-present ]
     if: ${{ always() && needs.check-dockerhub-secrets-present.outputs.secrets_present == 'true' }}
-    runs-on: [self-hosted, ubuntu-20.04, X64, small]
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-small
     name: Filter commit changes
     outputs:
       cache_images: ${{ steps.filter.outputs.cache_images }}
@@ -46,31 +46,36 @@ jobs:
   pull-and-cache-images:
     needs: [ check-dockerhub-secrets-present, changes ]
     if: ${{ always() && needs.check-dockerhub-secrets-present.outputs.secrets_present == 'true' && needs.changes.outputs.cache_images == 'true' }}
-    runs-on: [self-hosted, ubuntu-20.04, X64, small]
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med
+    name: Pull and cache images
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Compute docker images hash
+        id: compute-docker-images-hash
+        uses: ./.github/actions/compute-docker-images-hash
+      - name: Create cached docker images folder
+        run: |
+          mkdir -p ~/docker-images-cached
+      - name: Cache common docker images
+        id: cache-common-docker-images
+        uses: actions/cache@v4.2.0
+        with:
+          path: ~/docker-images-cached
+          key: docker-images-cache-${{ steps.compute-docker-images-hash.outputs.docker_images_hash }}
+          lookup-only: true
       - name: Login to Docker Hub
+        if: steps.cache-common-docker-images.outputs.cache-hit != 'true'
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Pull docker images from compose
-        continue-on-error: true
+      - name: Pull docker images
+        if: steps.cache-common-docker-images.outputs.cache-hit != 'true'
         run: |
           docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 pull
-      - name: Save Docker images
+      - name: Cache docker images
+        if: steps.cache-common-docker-images.outputs.cache-hit != 'true'
         run: |
-          mkdir -p ~/docker-images
-          images=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config | grep "image:" | awk '{print $2}')
-          for image in $images; do
-            imageFileName=$(echo $image | sed -e 's|.*/||' -e 's|:|-|' -e 's/\./_/g')
-            echo $image - ${imageFileName}
-            docker save $image > ~/docker-images/${imageFileName}.tar
-          done
-      - name: Cache common docker images
-        continue-on-error: true
-        uses: actions/cache/save@v4.0.2
-        with:
-          path: ~/docker-images
-          key: cached-images
+          docker save ${{ steps.compute-docker-images-hash.outputs.docker_images_trimmed }} -o  ~/docker-images-cached/docker-images.tar
+          echo docker-images.tar=$(ls -lh ~/docker-images-cached/docker-images.tar)

.github/workflows/codeql.yml

@@ -25,7 +25,7 @@ on:
 jobs:
   analyze:
     name: Analyze
-    runs-on: [self-hosted, ubuntu-20.04, X64, medium]
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med
     permissions:
       actions: read
       contents: read

.github/workflows/coordinator-testing.yml

@@ -17,15 +17,13 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
 jobs:
-  cache-docker-images:
-    uses: ./.github/workflows/cache-docker-images.yml
-    secrets: inherit
   run-tests:
     env:
       COMMIT_TAG: ${{ inputs.commit_tag }}
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+    # ? Seems to fail more often on xl
     runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-large
     name: Coordinator tests
     steps:
@@ -39,14 +37,12 @@ jobs:
         # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies.
         # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md
         uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1
-      - name: Restore cached images
-        id: restore-cached-images
-        uses: actions/cache/restore@v4.0.2
-        with:
-          path: ~/docker-images
-          key: cached-images
-          restore-keys: |
-            cached-images
+      # Disable cache for pulling docker images as empirically found that this 
+      # (retrieving cache and loading docker images) actually increased test time-to-completion
+      # - name: Restore cached images
+      #   id: restore-cached-images
+      #   uses: ./.github/actions/restore-docker-images
+      #   continue-on-error: true
       # Install pnpm to compile smart contracts
       - name: Setup nodejs environment
         uses: ./.github/actions/setup-nodejs

.github/workflows/finalized-tag-updater-github-release.yml → .github/workflows/github-release-besu-plugin.yml

@@ -1,15 +1,37 @@
-name: Github Release for Finalized-Tag-Updater Besu Plugin
+name: Github Release for Besu Plugins
 on:
+  workflow_call:
+    inputs:
+      pluginName:
+        required: true
+        type: string
+        description: 'plugin name used for the tag name, e.g staterecovery'
+      workspaceModulePath:
+        required: true
+        type: string
+        description: 'Path to plugin to release, e.g state-recovery/besu-plugin'
+      version:
+        required: true
+        type: string
+        description: 'Release semantic version: e.g "1.0.0"'
   workflow_dispatch:
     inputs:
+      pluginName:
+        required: true
+        type: string
+        description: 'plugin name used for the tag name, e.g staterecovery'
+      workspaceModulePath:
+        required: true
+        type: string
+        description: 'Path to plugin to release, e.g state-recovery/besu-plugin'
       version:
         required: true
         type: string
         description: 'Release semantic version: e.g "1.0.0"'
 
 jobs:
   release:
-    runs-on: [self-hosted, ubuntu-20.04, X64, small]
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -34,16 +56,19 @@ jobs:
 
       - name: Build
         run: |
-          ./gradlew :finalized-tag-updater:shadowJar -Pversion=v${{inputs.version}}
+          GRAGLE_PATH=:$(echo "${{inputs.workspaceModulePath}}" | sed 's/\//:/g')
+          echo $GRAGLE_PATH
+          ./gradlew $GRAGLE_PATH:shadowJar -Pversion=v${{inputs.version}}
+          ls -lh ${{inputs.workspaceModulePath}}/build/libs
 
       - name: Release to GitHub
         uses: jreleaser/release-action@v2
         with:
-          arguments: full-release --git-root-search --basedir=${{ github.workspace }}/finalized-tag-updater
+          arguments: full-release --git-root-search --basedir=${{ github.workspace }}/${{inputs.workspaceModulePath}} -P=version=v${{inputs.version}}
         env:
           GITHUB_TOKEN: ${{ secrets._GITHUB_TOKEN_RELEASE_ACCESS }}
-          JRELEASER_TAG_NAME: finalized-tag-updater-v${{inputs.version}}
-          JRELEASER_PROJECT_VERSION: '0.0.1'
+          JRELEASER_TAG_NAME: ${{inputs.pluginName}}-v${{inputs.version}}
+          JRELEASER_PROJECT_VERSION: ${{inputs.version}}
           JRELEASER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           JRELEASER_GITHUB_USERNAME: ${{ github.actor }}
           JRELEASER_GITHUB_EMAIL: ${{ github.actor }}@users.noreply.github.com

.github/workflows/load-test.yml

@@ -27,7 +27,7 @@ concurrency:
 
 jobs:
   run-load-test:
-    runs-on: [self-hosted, ubuntu-20.04, X64, small]
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-small
     name: Run Load Test
     steps:
       - name: Checkout

.github/workflows/main.yml

@@ -11,7 +11,7 @@ jobs:
     uses: ./.github/workflows/reuse-store-image-name-and-tags.yml
 
   filter-commit-changes:
-    runs-on: [self-hosted, ubuntu-20.04, X64, small]
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-small
     name: Filter commit changes
     outputs:
       coordinator: ${{ steps.filter.outputs.coordinator }}
@@ -148,7 +148,7 @@ jobs:
     secrets: inherit
 
   manual-docker-build-and-e2e-tests:
-    runs-on: [self-hosted, ubuntu-20.04, X64, small]
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-small
     needs: [ store-image-name-and-tags, filter-commit-changes, check-and-tag-images ]
     if: ${{ needs.filter-commit-changes.outputs.has-changes-requiring-build == 'true' }}
     environment: ${{ github.ref != 'refs/heads/main' && 'docker-build-and-e2e' || '' }}
@@ -245,7 +245,7 @@ jobs:
   cleanup-deployments:
     needs: [ run-e2e-tests, run-e2e-tests-geth-tracing ]
     if: ${{ always() }}
-    runs-on: [self-hosted, ubuntu-20.04, X64, small]
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-small
     steps:
       - uses: strumwolf/delete-deployment-environment@v3
         with:

.github/workflows/postman-build-and-publish.yml

@@ -51,7 +51,8 @@ concurrency:
 
 jobs:
   build-and-publish:
-    runs-on: [self-hosted, ubuntu-20.04, X64, small]
+    # ~1 min saved vs small
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med
     name: Postman build
     env:
       COMMIT_TAG: ${{ inputs.commit_tag }}

.github/workflows/postman-testing.yml

@@ -9,7 +9,7 @@ concurrency:
 
 jobs:
   run-tests:
-    runs-on: [self-hosted, ubuntu-22.04, X64, large]
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-large
     name: Postman & SDK tests
     steps:
       - name: Checkout

.github/workflows/prover-build-and-publish.yml

@@ -54,7 +54,8 @@ env:
 
 jobs:
   build-and-publish:
-    runs-on: [self-hosted, ubuntu-20.04, X64, small]
+    # ~1 min saved vs small
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med
     name: Prover build
     env:
       COMMIT_TAG: ${{ inputs.commit_tag }}