[CI][Testing] Add BuildJet CI runners for unit tests, integration tests; cache Docker images

.github/workflows/ci.yml

@@ -99,7 +99,7 @@ jobs:
           cache: true
       - name: Set Test Matrix
         id: set-test-matrix
-        run: go run utils/test_matrix/test_matrix.go admin cmd consensus engine fvm ledger module network/test network/p2p utils
+        run: go run utils/test_matrix/test_matrix.go admin cmd consensus engine/access engine/collection engine/common engine/consensus engine/execution/ingestion:buildjet-8vcpu-ubuntu-2204 engine/execution/computation engine/execution engine/verification engine:buildjet-4vcpu-ubuntu-2204 fvm ledger module/dkg module:buildjet-4vcpu-ubuntu-2204 network/alsp network/test/cohort1:buildjet-16vcpu-ubuntu-2204 network/test/cohort2:buildjet-4vcpu-ubuntu-2204 network/p2p/connection network/p2p/p2pnode:buildjet-4vcpu-ubuntu-2204 network/p2p/scoring network/p2p network state storage utils
 
   unit-test:
     name: Unit Tests (${{ matrix.targets.name }})
@@ -108,8 +108,8 @@ jobs:
       fail-fast: false
       matrix:
         targets: ${{ fromJSON(needs.create-dynamic-test-matrix.outputs.dynamic-matrix)}}
-    # need to set image explicitly due to GitHub logging issue as described in https://github.com/onflow/flow-go/pull/3087#issuecomment-1234383202
-    runs-on: ubuntu-20.04
+    ## need to set image explicitly due to GitHub logging issue as described in https://github.com/onflow/flow-go/pull/3087#issuecomment-1234383202
+    runs-on: ${{ matrix.targets.runner }}
     steps:
     - name: Checkout repo
       uses: actions/checkout@v3
@@ -123,8 +123,8 @@ jobs:
     - name: Run tests (${{ matrix.targets.name }})
       uses: nick-fields/retry@v2
       with:
-        timeout_minutes: 25
-        max_attempts: 3
+        timeout_minutes: 35
+        max_attempts: 5
         command: VERBOSE=1 make -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" test
 
       # TODO(rbtz): re-enable when we fix exisiting races.
@@ -148,17 +148,20 @@ jobs:
             make2: unittest
             retries: 1
             race: 1
+            runner: ubuntu-latest
           - name: insecure
             make1: install-tools
             make2: test
-            retries: 3
+            retries: 5
             race: 0
+            runner: buildjet-4vcpu-ubuntu-2204
           - name: integration
             make1: install-tools
             make2: test
-            retries: 3
+            retries: 5
             race: 0
-    runs-on: ubuntu-latest
+            runner: buildjet-4vcpu-ubuntu-2204
+    runs-on: ${{ matrix.runner }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
@@ -174,7 +177,7 @@ jobs:
           RACE_DETECTOR: ${{ matrix.race }}
         uses: nick-fields/retry@v2
         with:
-          timeout_minutes: 25
+          timeout_minutes: 35
           max_attempts: ${{ matrix.retries }}
           # run `make2` target inside each module's root
           command: VERBOSE=1 make -C ${{ matrix.name }} ${{ matrix.make2 }}
@@ -185,27 +188,101 @@ jobs:
           flags: unittests
           name: codecov-umbrella
 
+  docker-build:
+    name: Docker Build
+    runs-on: buildjet-16vcpu-ubuntu-2204
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          # all tags are needed for integration tests
+          fetch-depth: 0
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: true
+      - name: Build relic
+        run: make crypto_setup_gopath
+      - name: Docker build
+        run: make docker-build-flow docker-build-flow-corrupt
+      - name: Save Docker images
+        run: |
+          docker save \
+          gcr.io/flow-container-registry/access:latest \
+          gcr.io/flow-container-registry/collection:latest \
+          gcr.io/flow-container-registry/consensus:latest \
+          gcr.io/flow-container-registry/execution:latest \
+          gcr.io/flow-container-registry/ghost:latest \
+          gcr.io/flow-container-registry/observer:latest \
+          gcr.io/flow-container-registry/verification:latest \
+          gcr.io/flow-container-registry/access-corrupted:latest \
+          gcr.io/flow-container-registry/execution-corrupted:latest \
+          gcr.io/flow-container-registry/verification-corrupted:latest > flow-docker-images.tar
+      - name: Cache Docker images
+        uses: actions/cache@v3
+        with:
+          path: flow-docker-images.tar
+          # use the workflow run id as part of the cache key to ensure these docker images will only be used for a single workflow run
+          key: flow-docker-images-${{ hashFiles('**/Dockerfile') }}-${{ github.run_id }}
+
   integration-test:
     name: Integration Tests
+    needs: docker-build
     strategy:
       fail-fast: false
       matrix:
-        make:
-          - make -C integration access-tests
-          - make -C integration bft-framework-tests
-          - make -C integration bft-protocol-tests
-          - make -C integration bft-gossipsub-tests
-          - make -C integration collection-tests
-          - make -C integration consensus-tests
-          - make -C integration epochs-cohort1-tests
-          - make -C integration epochs-cohort2-tests
-          - make -C integration execution-tests
-          - make -C integration ghost-tests
-          - make -C integration mvp-tests
-          - make -C integration network-tests
-          - make -C integration verification-tests
-          - make -C integration upgrades-tests
-    runs-on: ubuntu-latest
+        include:
+          - name: Access Cohort1 Integration Tests
+            make: make -C integration access-cohort1-tests
+            runner: buildjet-4vcpu-ubuntu-2204
+          - name: Access Cohort2 Integration Tests
+            make: make -C integration access-cohort2-tests
+            runner: ubuntu-latest
+          - name: Access Cohort3 Integration Tests
+            make: make -C integration access-cohort3-tests
+            runner: ubuntu-latest
+            # test suite has single test which is flaky and needs to be fixed - reminder here to put it back when it's fixed
+#          - name: BFT (Framework) Integration Tests
+#            make: make -C integration bft-framework-tests
+#            runner: ubuntu-latest
+          - name: BFT (Protocol) Integration Tests
+            make: make -C integration bft-protocol-tests
+            runner: buildjet-8vcpu-ubuntu-2204
+          - name: BFT (Gossipsub) Integration Tests
+            make: make -C integration bft-gossipsub-tests
+            runner: ubuntu-latest
+          - name: Collection Integration Tests
+            make: make -C integration collection-tests
+            runner: ubuntu-latest
+          - name: Consensus Integration Tests
+            make: make -C integration consensus-tests
+            runner: ubuntu-latest
+          - name: Epoch Cohort1 Integration Tests
+            make: make -C integration epochs-cohort1-tests
+            runner: buildjet-8vcpu-ubuntu-2204
+          - name: Epoch Cohort2 Integration Tests
+            make: make -C integration epochs-cohort2-tests
+            runner: buildjet-4vcpu-ubuntu-2204
+          - name: Execution Integration Tests
+            make: make -C integration execution-tests
+            runner: ubuntu-latest
+          - name: Ghost Integration Tests
+            make: make -C integration ghost-tests
+            runner: ubuntu-latest
+          - name: MVP Integration Tests
+            make: make -C integration mvp-tests
+            runner: ubuntu-latest
+          - name: Network Integration Tests
+            make: make -C integration network-tests
+            runner: ubuntu-latest
+          - name: Verification Integration Tests
+            make: make -C integration verification-tests
+            runner: ubuntu-latest
+          - name: Upgrade Integration Tests
+            make: make -C integration upgrades-tests
+            runner: ubuntu-latest
+    runs-on: ${{ matrix.runner }}
     steps:
     - name: Checkout repo
       uses: actions/checkout@v3
@@ -219,51 +296,20 @@ jobs:
         cache: true
     - name: Build relic
       run: make crypto_setup_gopath
-    - name: Docker build
-      run: make docker-build-flow docker-build-flow-corrupt
-    - name: Run tests
+    - name: Load cached Docker images
+      uses: actions/cache@v3
+      with:
+        path: flow-docker-images.tar
+        # use the same cache key as the docker-build job
+        key: flow-docker-images-${{ hashFiles('**/Dockerfile') }}-${{ github.run_id }}
+    - name: Load Docker images
+      run: docker load -i flow-docker-images.tar
+    - name: Run tests (${{ matrix.name }})
       # TODO(rbtz): re-enable when we fix exisiting races.
       #env:
       #  RACE_DETECTOR: 1
       uses: nick-fields/retry@v2
       with:
-        timeout_minutes: 25
-        max_attempts: 3
+        timeout_minutes: 35
+        max_attempts: 5
         command: VERBOSE=1 ${{ matrix.make }}
-
-  localnet-test:
-    name: Localnet Compatibility Tests With Flow-CLI Client and Observer
-    strategy:
-      fail-fast: false
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout repo
-      uses: actions/checkout@v3
-    - name: Setup Go
-      uses: actions/setup-go@v3
-      with:
-        go-version: ${{ env.GO_VERSION }}
-        cache: true
-    - name: Build relic and other tools
-      run: make install-tools
-    - name: Install Flow Client In Docker
-      # This proved to be more reliable than installing it locally.
-      run: cd integration/localnet && sh client/client.sh
-    - name: Set up Localnet
-      run: bash -c 'cd integration/localnet/ && make -e OBSERVER=2 bootstrap && make start-flow'
-    - name: Ensure Observer is started
-      run: docker ps -f name=localnet-observer_1-1 | grep localnet-observer
-    - name: Get Client Version ensuring the client is provisioned
-      run: docker run --network host localnet-client /go/flow -f /go/flow-localnet.json -n observer version
-    - name: Wait for a default waiting period until a clean state
-      # This will not cause flakiness.
-      # The waiting time is a reasonable time to expect an observer to be responsive
-      run: sleep 10
-    - name: Get Status ensuring the access endpoint is online
-      run: docker run --network host localnet-client /go/flow -f /go/flow-localnet.json -n access status
-    - name: Wait for finalized blocks and check them
-      run: docker run --network host localnet-client /go/flow -f /go/flow-localnet.json -n observer blocks get latest
-    - name: Wait for finalized blocks and check them with Observer
-      run: sleep 5 && docker run --network host localnet-client /go/flow -f /go/flow-localnet.json -n access blocks get latest && docker run --network host localnet-client /go/flow -f /go/flow-localnet.json -n observer blocks get latest
-    - name: Stop localnet
-      run: bash -c 'cd integration/localnet/ && make stop'

.github/workflows/flaky-test-monitor.yml

@@ -5,10 +5,10 @@ name: Flaky Test Monitor
 on:
   schedule:
     - cron: '0 */2 * * *' # every 2 hours
-  push:
-    paths:
-      - '.github/workflows/flaky-test-monitor.yml'
-      - '.github/workflows/ci.yml'
+#  push:
+#    paths:
+#      - '.github/workflows/flaky-test-monitor.yml'
+#      - '.github/workflows/ci.yml'
 
 env:
   BIGQUERY_DATASET: production_src_flow_test_metrics

integration/Makefile

@@ -24,9 +24,18 @@ ci-integration-test: access-tests ghost-tests mvp-tests epochs-cohort1-tests epo
 test:
 	go test $(if $(VERBOSE),-v,) -tags relic -coverprofile=$(COVER_PROFILE) $(RACE_FLAG) $(if $(JSON_OUTPUT),-json,) $(if $(NUM_RUNS),-count $(NUM_RUNS),) `go list ./... | grep -v -e integration/tests`
 
-.PHONY: access-tests
-access-tests:
-	go test $(if $(VERBOSE),-v,) $(RACE_FLAG) $(if $(JSON_OUTPUT),-json,) $(if $(NUM_RUNS),-count $(NUM_RUNS),) -tags relic ./tests/access/...
+.PHONY: access-cohort1-tests
+access-cohort1-tests:
+	go test $(if $(VERBOSE),-v,) $(RACE_FLAG) $(if $(JSON_OUTPUT),-json,) $(if $(NUM_RUNS),-count $(NUM_RUNS),) -tags relic ./tests/access/cohort1/...
+
+.PHONY: access-cohort2-tests
+access-cohort2-tests:
+	go test $(if $(VERBOSE),-v,) $(RACE_FLAG) $(if $(JSON_OUTPUT),-json,) $(if $(NUM_RUNS),-count $(NUM_RUNS),) -tags relic ./tests/access/cohort2/...
+
+.PHONY: access-cohort3-tests
+access-cohort3-tests:
+	go test $(if $(VERBOSE),-v,) $(RACE_FLAG) $(if $(JSON_OUTPUT),-json,) $(if $(NUM_RUNS),-count $(NUM_RUNS),) -tags relic ./tests/access/cohort3/...
+
 
 .PHONY: collection-tests
 collection-tests:

integration/tests/access/access_api_test.go → integration/tests/access/cohort1/access_api_test.go

@@ -1,10 +1,12 @@
-package access
+package cohort1
 
 import (
 	"context"
 	"testing"
 	"time"
 
+	"github.com/onflow/flow-go/integration/tests/mvp"
+
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/suite"
 	"google.golang.org/grpc/codes"
@@ -18,7 +20,6 @@ import (
 	"github.com/onflow/flow-go/engine/access/rpc/backend"
 	"github.com/onflow/flow-go/integration/testnet"
 	"github.com/onflow/flow-go/integration/tests/lib"
-	"github.com/onflow/flow-go/integration/tests/mvp"
 	"github.com/onflow/flow-go/integration/utils"
 	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/utils/unittest"
@@ -143,30 +144,44 @@ func (s *AccessAPISuite) SetupTest() {
 	}, 30*time.Second, 1*time.Second)
 }
 
-// TestScriptExecutionAndGetAccounts test the Access API endpoints for executing scripts and getting
-// accounts using both local storage and execution nodes.
+// TestScriptExecutionAndGetAccountsAN1 test the Access API endpoints for executing scripts and getting
+// accounts using execution nodes.
 //
-// Note: combining all tests together to reduce setup/teardown time. test cases are read-only
+// Note: not combining AN1, AN2 tests together because that causes a drastic increase in test run times. test cases are read-only
 // and should not interfere with each other.
-func (s *AccessAPISuite) TestScriptExecutionAndGetAccounts() {
+func (s *AccessAPISuite) TestScriptExecutionAndGetAccountsAN1() {
 	// deploy the test contract
 	txResult := s.deployContract()
 	targetHeight := txResult.BlockHeight + 1
 	s.waitUntilIndexed(targetHeight)
 
-	// Run tests against Access 2, which uses local storage
-	s.testGetAccount(s.an2Client)
-	s.testExecuteScriptWithSimpleScript(s.an2Client)
-	s.testExecuteScriptWithSimpleContract(s.an2Client, targetHeight)
-
 	// Run tests against Access 1, which uses the execution node
 	s.testGetAccount(s.an1Client)
 	s.testExecuteScriptWithSimpleScript(s.an1Client)
 	s.testExecuteScriptWithSimpleContract(s.an1Client, targetHeight)
+}
+
+// TestScriptExecutionAndGetAccountsAN2 test the Access API endpoints for executing scripts and getting
+// accounts using local storage.
+//
+// Note: not combining AN1, AN2 tests together because that causes a drastic increase in test run times. test cases are read-only
+// and should not interfere with each other.
+func (s *AccessAPISuite) TestScriptExecutionAndGetAccountsAN2() {
+	// deploy the test contract
+	txResult := s.deployContract()
+	targetHeight := txResult.BlockHeight + 1
+	s.waitUntilIndexed(targetHeight)
+
+	// Run tests against Access 2, which uses local storage
+	s.testGetAccount(s.an2Client)
+	s.testExecuteScriptWithSimpleScript(s.an2Client)
+	s.testExecuteScriptWithSimpleContract(s.an2Client, targetHeight)
+}
 
+func (s *AccessAPISuite) TestMVPScriptExecutionLocalStorage() {
 	// this is a specialized test that creates accounts, deposits funds, deploys contracts, etc, and
 	// uses the provided access node to handle the Access API calls. there is an existing test that
-	// covers the default config, so we only need to test with local storage here.
+	// covers the default config, so we only need to test with local storage.
 	mvp.RunMVPTest(s.T(), s.ctx, s.net, s.accessNode2)
 }
 

integration/tests/access/observer_test.go → integration/tests/access/cohort2/observer_test.go

@@ -1,4 +1,4 @@
-package access
+package cohort2
 
 import (
 	"bytes"

integration/tests/access/access_circuit_breaker_test.go → integration/tests/access/cohort3/access_circuit_breaker_test.go

@@ -1,4 +1,4 @@
-package access
+package cohort3
 
 import (
 	"context"

integration/tests/access/access_test.go → integration/tests/access/cohort3/access_test.go

@@ -1,4 +1,4 @@
-package access
+package cohort3
 
 import (
 	"context"

integration/tests/access/consensus_follower_test.go → integration/tests/access/cohort3/consensus_follower_test.go

@@ -1,4 +1,4 @@
-package access
+package cohort3
 
 import (
 	"context"

integration/tests/access/execution_state_sync_test.go → integration/tests/access/cohort3/execution_state_sync_test.go

@@ -1,4 +1,4 @@
-package access
+package cohort3
 
 import (
 	"context"

integration/tests/access/grpc_compression_test.go → integration/tests/access/cohort3/grpc_compression_test.go

@@ -1,4 +1,4 @@
-package access
+package cohort3
 
 import (
 	"context"