Merge pull request #4687 from onflow/tarak/blst-merge-master

[Crypto] merge master branch

.github/workflows/actions/test-monitor-process-results/action.yml

@@ -0,0 +1,42 @@
+name: Test Monitor - Process Results
+
+description: Custom action that's used in multiple Flaky Test Monitor jobs to process test results and upload them to BigQuery
+
+inputs:
+  gcp_sa_key:
+    description: 'The GCP service account key for uploading to BigQuery'
+    required: true
+
+runs:
+  using : "composite"
+  steps:
+    - name: Get commit date
+      id: commit_date
+      run: echo "::set-output name=date::$(git show --no-patch --no-notes --pretty='%cI' $COMMIT_SHA)"
+      shell: bash
+    - name: Get job run date
+      id: job_run_date
+      run: echo "::set-output name=date::$(TZ=":UTC" date -Iseconds)"
+      shell: bash
+    - name: Process test results
+      run: cat test-output | go run tools/test_monitor/level1/process_summary1_results.go
+      env:
+        JOB_STARTED: ${{ steps.job_run_date.outputs.date }}
+        COMMIT_DATE: ${{ steps.commit_date.outputs.date }}
+      shell: bash
+    - name: Set up Cloud SDK
+      uses: google-github-actions/setup-gcloud@v0
+      with:
+        service_account_key: ${{ inputs.gcp_sa_key }}
+    - name: Upload results to BigQuery (skipped tests)
+      uses: nick-fields/retry@v2
+      with:
+        timeout_minutes: 1
+        max_attempts: 3
+        command: bq load --source_format=NEWLINE_DELIMITED_JSON $BIGQUERY_DATASET.$BIGQUERY_TABLE $SKIPPED_TESTS_FILE tools/test_monitor/schemas/skipped_tests_schema.json
+    - name: Upload results to BigQuery (test run)
+      uses: nick-fields/retry@v2
+      with:
+        timeout_minutes: 2
+        max_attempts: 3
+        command: bq load --source_format=NEWLINE_DELIMITED_JSON $BIGQUERY_DATASET.$BIGQUERY_TABLE2 $RESULTS_FILE tools/test_monitor/schemas/test_results_schema.json

.github/workflows/bench.yml

@@ -41,7 +41,7 @@ jobs:
       - name: Setup go
         uses: actions/setup-go@v3
         with:
-          go-version: "1.19"
+          go-version: "1.20"
           cache: true
 
       - name: Run benchmark on current branch

.github/workflows/builds.yml

@@ -51,6 +51,9 @@ jobs:
     outputs:
       matrix: ${{ steps.generate.outputs.matrix }}
     steps:
+    - name: Print all input variables
+      run: echo '${{ toJson(inputs) }}' | jq
+
     - id: generate
       run: |
         roles=()
@@ -74,6 +77,7 @@ jobs:
         fi
         rolesJSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${roles[@]}")
         echo "matrix={\"role\":$(echo $rolesJSON)}" >> $GITHUB_OUTPUT
+
   docker-push:
     name: ${{ matrix.role }} images
     runs-on: ubuntu-latest
@@ -89,10 +93,12 @@ jobs:
       uses: actions/setup-go@v2
       with:
         go-version: '1.19'
+
     - name: Checkout repo
       uses: actions/checkout@v2
       with:
         ref: ${{ inputs.tag }}
+
     # Provide Google Service Account credentials to Github Action, allowing interaction with the Google Container Registry
     # Logging in as github-actions@dl-flow.iam.gserviceaccount.com
     - name: Docker login

.github/workflows/cd.yml

@@ -14,7 +14,7 @@ jobs:
     - name: Setup Go
       uses: actions/setup-go@v2
       with:
-        go-version: '1.19'
+        go-version: "1.20"
     - name: Checkout repo
       uses: actions/checkout@v2
     # Provide Google Service Account credentials to Github Action, allowing interaction with the Google Container Registry

.github/workflows/ci.yml

@@ -14,11 +14,14 @@ on:
       - 'auto-cadence-upgrade/**'
       - 'feature/**'
       - 'v[0-9]+.[0-9]+'
+  merge_group:
+    branches:
+      - master
 
 env:
-  GO_VERSION: 1.19
+  GO_VERSION: "1.20"
 
-concurrency: 
+concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
   cancel-in-progress: true
 
@@ -49,7 +52,7 @@ jobs:
       uses: golangci/golangci-lint-action@v3
       with:
         # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-        version: v1.49
+        version: v1.51
         args: -v
         working-directory: ${{ matrix.dir }}
         # https://github.com/golangci/golangci-lint-action/issues/244
@@ -69,6 +72,8 @@ jobs:
           cache: true
       - name: Run tidy
         run: make tidy
+      - name: code sanity check
+        run: make code-sanity-check
 
   create-dynamic-test-matrix:
     name: Create Dynamic Test Matrix
@@ -85,7 +90,7 @@ jobs:
           cache: true
       - name: Set Test Matrix
         id: set-test-matrix
-        run: go run utils/test_matrix/test_matrix.go access admin cmd consensus engine fvm ledger module network utils
+        run: go run utils/test_matrix/test_matrix.go admin cmd consensus engine fvm ledger module network/test network/p2p utils
 
   unit-test:
     name: Unit Tests (${{ matrix.targets.name }})
@@ -104,7 +109,7 @@ jobs:
       with:
         go-version: ${{ env.GO_VERSION }}
         cache: true
-    - name: Setup tests (${{ matrix.targets.name }}
+    - name: Setup tests (${{ matrix.targets.name }})
       run: VERBOSE=1 make -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" install-tools
     - name: Run tests (${{ matrix.targets.name }})
       uses: nick-fields/retry@v2
@@ -136,7 +141,7 @@ jobs:
           - name: insecure
             setup: install-tools
             retries: 3
-            race: 1
+            race: 0
           - name: integration
             setup: install-tools
             retries: 3
@@ -175,10 +180,13 @@ jobs:
       matrix:
         make:
           - make -C integration access-tests
-          - make -C integration bft-tests
+          - make -C integration bft-framework-tests
+          - make -C integration bft-protocol-tests
+          - make -C integration bft-gossipsub-tests
           - make -C integration collection-tests
           - make -C integration consensus-tests
-          - make -C integration epochs-tests
+          - make -C integration epochs-cohort1-tests
+          - make -C integration epochs-cohort2-tests
           - make -C integration execution-tests
           - make -C integration ghost-tests
           - make -C integration mvp-tests
@@ -189,6 +197,9 @@ jobs:
     steps:
     - name: Checkout repo
       uses: actions/checkout@v3
+      with:
+          # all tags are needed for integration tests
+          fetch-depth: 0
     - name: Setup Go
       uses: actions/setup-go@v3
       with:
@@ -227,7 +238,7 @@ jobs:
     - name: Set up Localnet
       run: bash -c 'cd integration/localnet/ && make -e OBSERVER=2 bootstrap && make start-flow'
     - name: Ensure Observer is started
-      run: docker ps -f name=localnet_observer_1_1 | grep localnet_observer
+      run: docker ps -f name=localnet-observer_1-1 | grep localnet-observer
     - name: Get Client Version ensuring the client is provisioned
       run: docker run --network host localnet-client /go/flow -f /go/flow-localnet.json -n observer version
     - name: Wait for a default waiting period until a clean state