From e37c9d87e7f4965ed177cc41ee218bbfae1bf9f0 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 18 Feb 2022 06:10:05 -0400
Subject: [PATCH 01/24] add(tests): full sync test

---
 .github/workflows/test.yml | 659 +++++++++++++++++++++----------------
 1 file changed, 373 insertions(+), 286 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index c987795c42c..1ce73684171 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -33,10 +33,364 @@ env:
   IMAGE_NAME: zebrad-test
 
 jobs:
-  build:
-    name: Build images
-    timeout-minutes: 210
+  # build:
+  #   name: Build images
+  #   timeout-minutes: 210
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v2.4.0
+  #       with:
+  #         persist-credentials: false
+
+  #     - name: Inject slug/short variables
+  #       uses: rlespinasse/github-slug-action@v4
+
+  #     # Setup Docker Buildx to allow use of docker cache layers from GH
+  #     - name: Set up Docker Buildx
+  #       uses: docker/setup-buildx-action@v1
+  #       with:
+  #         driver-opts: network=host
+
+  #     - name: Login to Google Artifact Registry
+  #       uses: docker/login-action@v1.12.0
+  #       with:
+  #         registry: us-docker.pkg.dev
+  #         username: _json_key
+  #         password: ${{ secrets.GOOGLE_CREDENTIALS }}
+
+  #     - name: Login to Google Container Registry
+  #       uses: docker/login-action@v1.12.0
+  #       with:
+  #         registry: gcr.io
+  #         username: _json_key
+  #         password: ${{ secrets.GOOGLE_CREDENTIALS }}
+
+  #     # Build and push image to Google Artifact Registry
+  #     - name: Build & push
+  #       id: docker_build
+  #       uses: docker/build-push-action@v2.8.0
+  #       with:
+  #         target: tester
+  #         context: .
+  #         file: ./docker/Dockerfile.build
+  #         tags: |
+  #           ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:latest
+  #           ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+  #           ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:latest
+  #           ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+  #         build-args: |
+  #           NETWORK=${{ github.event.inputs.network || env.NETWORK }}
+  #           SHORT_SHA=${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+  #           RUST_BACKTRACE=full
+  #           ZEBRA_SKIP_NETWORK_TESTS="1"
+  #           CHECKPOINT_SYNC=${{ github.event.inputs.checkpoint_sync || true }}
+  #           RUST_LOG=debug
+  #           SENTRY_DSN=${{ secrets.SENTRY_ENDPOINT }}
+  #         push: true
+  #         cache-from: type=gha
+  #         cache-to: type=gha,mode=max
+
+  # # Run all the zebra tests, including tests that are ignored by default
+  # test-all:
+  #   name: Test all
+  #   runs-on: ubuntu-latest
+  #   needs: build
+  #   if: ${{ github.event.inputs.regenerate-disks != 'true' }}
+  #   steps:
+  #     - uses: actions/checkout@v2.4.0
+  #       with:
+  #         persist-credentials: false
+
+  #     - name: Inject slug/short variables
+  #       uses: rlespinasse/github-slug-action@v4
+
+  #     - name: Run all zebrad tests
+  #       run: |
+  #         docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+  #         docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --workspace -- --include-ignored
+
+  # test-fake-activation-heights:
+  #   name: Test with fake activation heights
+  #   runs-on: ubuntu-latest
+  #   needs: build
+  #   if: ${{ github.event.inputs.regenerate-disks != 'true' }}
+  #   steps:
+  #     - uses: actions/checkout@v2.4.0
+  #       with:
+  #         persist-credentials: false
+
+  #     - name: Inject slug/short variables
+  #       uses: rlespinasse/github-slug-action@v4
+
+  #     - name: Run tests with fake activation heights
+  #       run: |
+  #         docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+  #         docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --package zebra-state --lib -- with_fake_activation_heights
+
+  # # Test that Zebra syncs and checkpoints a few thousand blocks from an empty state
+  # test-empty-sync:
+  #   name: Test checkpoint sync from empty state
+  #   runs-on: ubuntu-latest
+  #   needs: build
+  #   if: ${{ github.event.inputs.regenerate-disks != 'true' }}
+  #   steps:
+  #     - uses: actions/checkout@v2.4.0
+  #       with:
+  #         persist-credentials: false
+
+  #     - name: Inject slug/short variables
+  #       uses: rlespinasse/github-slug-action@v4
+
+  #     - name: Run zebrad large sync tests
+  #       run: |
+  #         docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+  #         docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance sync_large_checkpoints_ -- --ignored
+
+  # regenerate-stateful-disks:
+  #   name: Regenerate stateful disks
+  #   runs-on: ubuntu-latest
+  #   needs: build
+  #   outputs:
+  #     disk_short_sha: ${{ steps.disk-short-sha.outputs.disk_short_sha }}
+  #   steps:
+  #     - uses: actions/checkout@v2.4.0
+  #       with:
+  #         persist-credentials: false
+  #         fetch-depth: '2'
+
+  #     - name: Get specific changed files
+  #       id: changed-files-specific
+  #       uses: tj-actions/changed-files@v14.4
+  #       with:
+  #         files: |
+  #           /zebra-state/**/disk_format.rs
+  #           /zebra-state/**/disk_db.rs
+  #           /zebra-state/**/finalized_state.rs
+  #           /zebra-state/**/constants.rs
+
+  #     - name: Inject slug/short variables
+  #       uses: rlespinasse/github-slug-action@v4
+
+  #     - name: Downcase network name for disks
+  #       run: |
+  #         echo LOWER_NET_NAME="${{ github.event.inputs.network || env.NETWORK }}" | awk '{print tolower($0)}' >> $GITHUB_ENV
+
+  #     # Setup gcloud CLI
+  #     - name: Authenticate to Google Cloud
+  #       id: auth
+  #       uses: google-github-actions/auth@v0.5.0
+  #       with:
+  #         credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
+
+  #     - name: Create GCP compute instance
+  #       id: create-instance
+  #       if: ${{ steps.changed-files-specific.outputs.any_changed == 'true' || github.event.inputs.regenerate-disks == 'true' }}
+  #       run: |
+  #         gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" \
+  #         --boot-disk-size 100GB \
+  #         --boot-disk-type pd-ssd \
+  #         --create-disk name="zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy",size=100GB,type=pd-ssd \
+  #         --container-mount-disk mount-path='/zebrad-cache',name="zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy" \
+  #         --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
+  #         --container-restart-policy=never \
+  #         --container-stdin \
+  #         --container-tty \
+  #         --container-command="cargo" \
+  #         --container-arg="test" \
+  #         --container-arg="--locked" \
+  #         --container-arg="--release" \
+  #         --container-arg="--features" \
+  #         --container-arg="enable-sentry,test_sync_to_mandatory_checkpoint_${{ env.lower_net_name }}" \
+  #         --container-arg="--manifest-path" \
+  #         --container-arg="zebrad/Cargo.toml" \
+  #         --container-arg="sync_to_mandatory_checkpoint_${{ env.lower_net_name }}" \
+  #         --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
+  #         --machine-type ${{ env.MACHINE_TYPE }} \
+  #         --scopes cloud-platform \
+  #         --metadata=google-monitoring-enabled=true,google-logging-enabled=true \
+  #         --tags zebrad \
+  #         --zone "${{ env.ZONE }}"
+
+  #     # TODO: this approach is very mesy, but getting the just created container name is very error prone and GCP doesn't have a workaround for this without requiring a TTY
+  #     # This TODO relates to the following issues:
+  #     # https://github.com/actions/runner/issues/241
+  #     # https://www.googlecloudcommunity.com/gc/Infrastructure-Compute-Storage/SSH-into-Compute-Container-not-easily-possible/td-p/170915
+  #     - name: Get container name from logs
+  #       id: get-container-name
+  #       if: steps.create-instance.outcome == 'success'
+  #       run: |
+  #         INSTANCE_ID=$(gcloud compute instances describe zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --format='value(id)')
+  #         echo "Using instance: $INSTANCE_ID"
+  #         while [[ ${CONTAINER_NAME} != *"zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}"* ]]; do
+  #             CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
+  #             echo "Using container: ${CONTAINER_NAME} from instance: ${INSTANCE_ID}"
+  #             sleep 10
+  #         done
+  #         CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
+  #         echo "::set-output name=zebra_container::$CONTAINER_NAME"
+
+  #     - name: Regenerate stateful disks logs
+  #       id: sync-to-checkpoint
+  #       if: steps.create-instance.outcome == 'success'
+  #       run: |
+  #         gcloud compute ssh \
+  #         zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
+  #         --zone ${{ env.ZONE }} \
+  #         --quiet \
+  #         --ssh-flag="-o ServerAliveInterval=5" \
+  #         --command="docker logs --follow ${{ env.ZEBRA_CONTAINER }}"
+  #       env:
+  #         ZEBRA_CONTAINER: ${{ steps.get-container-name.outputs.zebra_container }}
+
+  #     # Create image from disk that will be used to sync past mandatory checkpoint test
+  #     # Force the image creation as the disk is still attached even though is not being used by the container
+  #     - name: Create image from state disk
+  #       # Only run if the earlier step succeeds
+  #       if: steps.sync-to-checkpoint.outcome == 'success'
+  #       run: |
+  #         gcloud compute images create zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy \
+  #         --force \
+  #         --source-disk=zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy \
+  #         --source-disk-zone=${{ env.ZONE }} \
+  #         --storage-location=us \
+  #         --description="Created from head branch ${{ env.GITHUB_HEAD_REF_SLUG_URL }} targeting ${{ env.GITHUB_BASE_REF_SLUG }} from PR ${{ env.GITHUB_REF_SLUG_URL }} with commit ${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA }}"
+
+  #     - name: Output and write the disk SHORT_SHA to a txt
+  #       id: disk-short-sha
+  #       if: steps.sync-to-checkpoint.outcome == 'success'
+  #       run: |
+  #         short_sha=$(echo "${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}")
+  #         echo "$short_sha" > latest-disk-state-sha.txt
+  #         echo "::set-output name=disk_short_sha::$short_sha"
+
+  #     - name: Upload the disk state txt
+  #       if: steps.sync-to-checkpoint.outcome == 'success'
+  #       uses: actions/upload-artifact@v2.3.1
+  #       with:
+  #         name: latest-disk-state-sha
+  #         path: latest-disk-state-sha.txt
+  #         retention-days: 1095
+
+  #     - name: Delete test instance
+  #       # Do not delete the instance if the sync timeouts in GitHub
+  #       if: ${{ steps.sync-to-checkpoint.outcome == 'success' }} || ${{ steps.sync-to-checkpoint.outcome == 'failure' }}
+  #       continue-on-error: true
+  #       run: |
+  #         gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
+
+  # # Test that Zebra syncs and fully validates a few thousand blocks from a cached post-checkpoint state
+  # test-stateful-sync:
+  #   name: Test full validation sync from cached state
+  #   runs-on: ubuntu-latest
+  #   needs: [ build, regenerate-stateful-disks]
+  #   steps:
+  #     - uses: actions/checkout@v2.4.0
+  #       with:
+  #         persist-credentials: false
+
+  #     - name: Inject slug/short variables
+  #       uses: rlespinasse/github-slug-action@v4
+
+  #     - name: Downcase network name for disks
+  #       run: |
+  #         echo LOWER_NET_NAME="${{ github.event.inputs.network || env.NETWORK }}" | awk '{print tolower($0)}' >> $GITHUB_ENV
+
+  #     # Get the latest uploaded txt with the disk SHORT_SHA from this workflow
+  #     - name: Download latest disk state SHORT_SHA
+  #       uses: dawidd6/action-download-artifact@v2.17.0
+  #       with:
+  #         github_token: ${{ secrets.GITHUB_TOKEN }}
+  #         workflow: test.yml
+  #         workflow_conclusion: ''
+  #         name: latest-disk-state-sha
+  #         check_artifacts:  true
+
+  #     - name: Get disk state SHA from txt
+  #       id: get-disk-sha
+  #       run: |
+  #         output=$(cat latest-disk-state-sha.txt)
+  #         echo "::set-output name=sha::$output"
+
+  #     # Setup gcloud CLI
+  #     - name: Authenticate to Google Cloud
+  #       id: auth
+  #       uses: google-github-actions/auth@v0.5.0
+  #       with:
+  #         credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
+
+  #     # Creates Compute Engine virtual machine instance w/ disks
+  #     - name: Create GCP compute instance
+  #       id: create-instance
+  #       run: |
+  #         gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" \
+  #         --boot-disk-size 100GB \
+  #         --boot-disk-type pd-ssd \
+  #         --create-disk=image=zebrad-cache-${{ env.DISK_SHORT_SHA }}-${{ env.lower_net_name }}-canopy,name=zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy,size=100GB,type=pd-ssd \
+  #         --container-mount-disk=mount-path='/zebrad-cache',name=zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy \
+  #         --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
+  #         --container-restart-policy=never \
+  #         --container-stdin \
+  #         --container-tty \
+  #         --container-command="cargo" \
+  #         --container-arg="test" \
+  #         --container-arg="--locked" \
+  #         --container-arg="--release" \
+  #         --container-arg="--features" \
+  #         --container-arg="enable-sentry,test_sync_past_mandatory_checkpoint_${{ env.lower_net_name }}" \
+  #         --container-arg="--manifest-path" \
+  #         --container-arg="zebrad/Cargo.toml" \
+  #         --container-arg="sync_past_mandatory_checkpoint_${{ env.lower_net_name }}" \
+  #         --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
+  #         --machine-type ${{ env.MACHINE_TYPE }} \
+  #         --scopes cloud-platform \
+  #         --metadata=google-monitoring-enabled=true,google-logging-enabled=true \
+  #         --tags zebrad \
+  #         --zone "${{ env.ZONE }}"
+  #       env:
+  #         DISK_SHORT_SHA: ${{ needs.regenerate-stateful-disks.outputs.disk_short_sha || steps.get-disk-sha.outputs.sha }}
+
+  #     # TODO: this approach is very mesy, but getting the just created container name is very error prone and GCP doesn't have a workaround for this without requiring a TTY
+  #     # This TODO relates to the following issues:
+  #     # https://github.com/actions/runner/issues/241
+  #     # https://www.googlecloudcommunity.com/gc/Infrastructure-Compute-Storage/SSH-into-Compute-Container-not-easily-possible/td-p/170915
+  #     - name: Get container name from logs
+  #       id: get-container-name
+  #       if: steps.create-instance.outcome == 'success'
+  #       run: |
+  #         INSTANCE_ID=$(gcloud compute instances describe zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --format='value(id)')
+  #         echo "Using instance: $INSTANCE_ID"
+  #         while [[ ${CONTAINER_NAME} != *"zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}"* ]]; do
+  #             CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
+  #             echo "Using container: ${CONTAINER_NAME} from instance: ${INSTANCE_ID}"
+  #             sleep 10
+  #         done
+  #         CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
+  #         echo "::set-output name=zebra_container::$CONTAINER_NAME"
+
+  #     - name: Sync past mandatory checkpoint logs
+  #       id: sync-past-checkpoint
+  #       run: |
+  #         gcloud compute ssh \
+  #         zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
+  #         --zone ${{ env.ZONE }} \
+  #         --quiet \
+  #         --ssh-flag="-o ServerAliveInterval=5" \
+  #         --command="docker logs --follow ${{ env.ZEBRA_CONTAINER }}"
+  #       env:
+  #         ZEBRA_CONTAINER: ${{ steps.get-container-name.outputs.zebra_container }}
+
+  #     - name: Delete test instance
+  #       # Do not delete the instance if the sync timeouts in GitHub
+  #       if: ${{ steps.sync-past-checkpoint.outcome == 'success' }} || ${{ steps.sync-past-checkpoint.outcome == 'failure' }}
+  #       continue-on-error: true
+  #       run: |
+  #         gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
+
+  # Test that Zebra can run a full mainnet sync
+  test-full-sync:
+    name: Test full Mainnet sync
     runs-on: ubuntu-latest
+    # needs: [ build]
     steps:
       - uses: actions/checkout@v2.4.0
         with:
@@ -45,272 +399,6 @@ jobs:
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v4
 
-      # Setup Docker Buildx to allow use of docker cache layers from GH
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          driver-opts: network=host
-
-      - name: Login to Google Artifact Registry
-        uses: docker/login-action@v1.12.0
-        with:
-          registry: us-docker.pkg.dev
-          username: _json_key
-          password: ${{ secrets.GOOGLE_CREDENTIALS }}
-
-      - name: Login to Google Container Registry
-        uses: docker/login-action@v1.12.0
-        with:
-          registry: gcr.io
-          username: _json_key
-          password: ${{ secrets.GOOGLE_CREDENTIALS }}
-
-      # Build and push image to Google Artifact Registry
-      - name: Build & push
-        id: docker_build
-        uses: docker/build-push-action@v2.8.0
-        with:
-          target: tester
-          context: .
-          file: ./docker/Dockerfile.build
-          tags: |
-            ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:latest
-            ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
-            ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:latest
-            ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
-          build-args: |
-            NETWORK=${{ github.event.inputs.network || env.NETWORK }}
-            SHORT_SHA=${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
-            RUST_BACKTRACE=full
-            ZEBRA_SKIP_NETWORK_TESTS="1"
-            CHECKPOINT_SYNC=${{ github.event.inputs.checkpoint_sync || true }}
-            RUST_LOG=debug
-            SENTRY_DSN=${{ secrets.SENTRY_ENDPOINT }}
-          push: true
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-  # Run all the zebra tests, including tests that are ignored by default
-  test-all:
-    name: Test all
-    runs-on: ubuntu-latest
-    needs: build
-    if: ${{ github.event.inputs.regenerate-disks != 'true' }}
-    steps:
-      - uses: actions/checkout@v2.4.0
-        with:
-          persist-credentials: false
-
-      - name: Inject slug/short variables
-        uses: rlespinasse/github-slug-action@v4
-
-      - name: Run all zebrad tests
-        run: |
-          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
-          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --workspace -- --include-ignored
-
-  test-fake-activation-heights:
-    name: Test with fake activation heights
-    runs-on: ubuntu-latest
-    needs: build
-    if: ${{ github.event.inputs.regenerate-disks != 'true' }}
-    steps:
-      - uses: actions/checkout@v2.4.0
-        with:
-          persist-credentials: false
-
-      - name: Inject slug/short variables
-        uses: rlespinasse/github-slug-action@v4
-
-      - name: Run tests with fake activation heights
-        run: |
-          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
-          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --package zebra-state --lib -- with_fake_activation_heights
-
-  # Test that Zebra syncs and checkpoints a few thousand blocks from an empty state
-  test-empty-sync:
-    name: Test checkpoint sync from empty state
-    runs-on: ubuntu-latest
-    needs: build
-    if: ${{ github.event.inputs.regenerate-disks != 'true' }}
-    steps:
-      - uses: actions/checkout@v2.4.0
-        with:
-          persist-credentials: false
-
-      - name: Inject slug/short variables
-        uses: rlespinasse/github-slug-action@v4
-
-      - name: Run zebrad large sync tests
-        run: |
-          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
-          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance sync_large_checkpoints_ -- --ignored
-
-  regenerate-stateful-disks:
-    name: Regenerate stateful disks
-    runs-on: ubuntu-latest
-    needs: build
-    outputs:
-      disk_short_sha: ${{ steps.disk-short-sha.outputs.disk_short_sha }}
-    steps:
-      - uses: actions/checkout@v2.4.0
-        with:
-          persist-credentials: false
-          fetch-depth: '2'
-
-      - name: Get specific changed files
-        id: changed-files-specific
-        uses: tj-actions/changed-files@v14.4
-        with:
-          files: |
-            /zebra-state/**/disk_format.rs
-            /zebra-state/**/disk_db.rs
-            /zebra-state/**/finalized_state.rs
-            /zebra-state/**/constants.rs
-
-      - name: Inject slug/short variables
-        uses: rlespinasse/github-slug-action@v4
-
-      - name: Downcase network name for disks
-        run: |
-          echo LOWER_NET_NAME="${{ github.event.inputs.network || env.NETWORK }}" | awk '{print tolower($0)}' >> $GITHUB_ENV
-
-      # Setup gcloud CLI
-      - name: Authenticate to Google Cloud
-        id: auth
-        uses: google-github-actions/auth@v0.5.0
-        with:
-          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
-
-      - name: Create GCP compute instance
-        id: create-instance
-        if: ${{ steps.changed-files-specific.outputs.any_changed == 'true' || github.event.inputs.regenerate-disks == 'true' }}
-        run: |
-          gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" \
-          --boot-disk-size 100GB \
-          --boot-disk-type pd-ssd \
-          --create-disk name="zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy",size=100GB,type=pd-ssd \
-          --container-mount-disk mount-path='/zebrad-cache',name="zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy" \
-          --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
-          --container-restart-policy=never \
-          --container-stdin \
-          --container-tty \
-          --container-command="cargo" \
-          --container-arg="test" \
-          --container-arg="--locked" \
-          --container-arg="--release" \
-          --container-arg="--features" \
-          --container-arg="enable-sentry,test_sync_to_mandatory_checkpoint_${{ env.lower_net_name }}" \
-          --container-arg="--manifest-path" \
-          --container-arg="zebrad/Cargo.toml" \
-          --container-arg="sync_to_mandatory_checkpoint_${{ env.lower_net_name }}" \
-          --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
-          --machine-type ${{ env.MACHINE_TYPE }} \
-          --scopes cloud-platform \
-          --metadata=google-monitoring-enabled=true,google-logging-enabled=true \
-          --tags zebrad \
-          --zone "${{ env.ZONE }}"
-
-      # TODO: this approach is very mesy, but getting the just created container name is very error prone and GCP doesn't have a workaround for this without requiring a TTY
-      # This TODO relates to the following issues:
-      # https://github.com/actions/runner/issues/241
-      # https://www.googlecloudcommunity.com/gc/Infrastructure-Compute-Storage/SSH-into-Compute-Container-not-easily-possible/td-p/170915
-      - name: Get container name from logs
-        id: get-container-name
-        if: steps.create-instance.outcome == 'success'
-        run: |
-          INSTANCE_ID=$(gcloud compute instances describe zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --format='value(id)')
-          echo "Using instance: $INSTANCE_ID"
-          while [[ ${CONTAINER_NAME} != *"zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}"* ]]; do
-              CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
-              echo "Using container: ${CONTAINER_NAME} from instance: ${INSTANCE_ID}"
-              sleep 10
-          done
-          CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
-          echo "::set-output name=zebra_container::$CONTAINER_NAME"
-
-      - name: Regenerate stateful disks logs
-        id: sync-to-checkpoint
-        if: steps.create-instance.outcome == 'success'
-        run: |
-          gcloud compute ssh \
-          zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
-          --zone ${{ env.ZONE }} \
-          --quiet \
-          --ssh-flag="-o ServerAliveInterval=5" \
-          --command="docker logs --follow ${{ env.ZEBRA_CONTAINER }}"
-        env:
-          ZEBRA_CONTAINER: ${{ steps.get-container-name.outputs.zebra_container }}
-
-      # Create image from disk that will be used to sync past mandatory checkpoint test
-      # Force the image creation as the disk is still attached even though is not being used by the container
-      - name: Create image from state disk
-        # Only run if the earlier step succeeds
-        if: steps.sync-to-checkpoint.outcome == 'success'
-        run: |
-          gcloud compute images create zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy \
-          --force \
-          --source-disk=zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy \
-          --source-disk-zone=${{ env.ZONE }} \
-          --storage-location=us \
-          --description="Created from head branch ${{ env.GITHUB_HEAD_REF_SLUG_URL }} targeting ${{ env.GITHUB_BASE_REF_SLUG }} from PR ${{ env.GITHUB_REF_SLUG_URL }} with commit ${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA }}"
-
-      - name: Output and write the disk SHORT_SHA to a txt
-        id: disk-short-sha
-        if: steps.sync-to-checkpoint.outcome == 'success'
-        run: |
-          short_sha=$(echo "${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}")
-          echo "$short_sha" > latest-disk-state-sha.txt
-          echo "::set-output name=disk_short_sha::$short_sha"
-
-      - name: Upload the disk state txt
-        if: steps.sync-to-checkpoint.outcome == 'success'
-        uses: actions/upload-artifact@v2.3.1
-        with:
-          name: latest-disk-state-sha
-          path: latest-disk-state-sha.txt
-          retention-days: 1095
-
-      - name: Delete test instance
-        # Do not delete the instance if the sync timeouts in GitHub
-        if: ${{ steps.sync-to-checkpoint.outcome == 'success' }} || ${{ steps.sync-to-checkpoint.outcome == 'failure' }}
-        continue-on-error: true
-        run: |
-          gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
-
-  # Test that Zebra syncs and fully validates a few thousand blocks from a cached post-checkpoint state
-  test-stateful-sync:
-    name: Test full validation sync from cached state
-    runs-on: ubuntu-latest
-    needs: [ build, regenerate-stateful-disks]
-    steps:
-      - uses: actions/checkout@v2.4.0
-        with:
-          persist-credentials: false
-
-      - name: Inject slug/short variables
-        uses: rlespinasse/github-slug-action@v4
-
-      - name: Downcase network name for disks
-        run: |
-          echo LOWER_NET_NAME="${{ github.event.inputs.network || env.NETWORK }}" | awk '{print tolower($0)}' >> $GITHUB_ENV
-
-      # Get the latest uploaded txt with the disk SHORT_SHA from this workflow
-      - name: Download latest disk state SHORT_SHA
-        uses: dawidd6/action-download-artifact@v2.17.0
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          workflow: test.yml
-          workflow_conclusion: ''
-          name: latest-disk-state-sha
-          check_artifacts:  true
-
-      - name: Get disk state SHA from txt
-        id: get-disk-sha
-        run: |
-          output=$(cat latest-disk-state-sha.txt)
-          echo "::set-output name=sha::$output"
-
       # Setup gcloud CLI
       - name: Authenticate to Google Cloud
         id: auth
@@ -322,12 +410,10 @@ jobs:
       - name: Create GCP compute instance
         id: create-instance
         run: |
-          gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" \
+          gcloud compute instances create-with-container "sync-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 100GB \
           --boot-disk-type pd-ssd \
-          --create-disk=image=zebrad-cache-${{ env.DISK_SHORT_SHA }}-${{ env.lower_net_name }}-canopy,name=zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy,size=100GB,type=pd-ssd \
-          --container-mount-disk=mount-path='/zebrad-cache',name=zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy \
-          --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
+          --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }} \
           --container-restart-policy=never \
           --container-stdin \
           --container-tty \
@@ -336,18 +422,19 @@ jobs:
           --container-arg="--locked" \
           --container-arg="--release" \
           --container-arg="--features" \
-          --container-arg="enable-sentry,test_sync_past_mandatory_checkpoint_${{ env.lower_net_name }}" \
-          --container-arg="--manifest-path" \
-          --container-arg="zebrad/Cargo.toml" \
-          --container-arg="sync_past_mandatory_checkpoint_${{ env.lower_net_name }}" \
-          --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
+          --container-arg="enable-sentry" \
+          --container-arg="--test" \
+          --container-arg="acceptance" \
+          --container-arg=-- \
+          --container-arg="--nocapture" \
+          --container-arg="--ignored" \
+          --container-arg="full_sync_mainnet" \
+          --container-env=ZEBRA_SKIP_IPV6_TESTS=1,FULL_SYNC_MAINNET_TIMEOUT_MINUTES=600 \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --scopes cloud-platform \
           --metadata=google-monitoring-enabled=true,google-logging-enabled=true \
           --tags zebrad \
           --zone "${{ env.ZONE }}"
-        env:
-          DISK_SHORT_SHA: ${{ needs.regenerate-stateful-disks.outputs.disk_short_sha || steps.get-disk-sha.outputs.sha }}
 
       # TODO: this approach is very mesy, but getting the just created container name is very error prone and GCP doesn't have a workaround for this without requiring a TTY
       # This TODO relates to the following issues:
@@ -357,21 +444,21 @@ jobs:
         id: get-container-name
         if: steps.create-instance.outcome == 'success'
         run: |
-          INSTANCE_ID=$(gcloud compute instances describe zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --format='value(id)')
+          INSTANCE_ID=$(gcloud compute instances describe sync-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --format='value(id)')
           echo "Using instance: $INSTANCE_ID"
-          while [[ ${CONTAINER_NAME} != *"zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}"* ]]; do
-              CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
+          while [[ ${CONTAINER_NAME} != *"sync-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}"* ]]; do
+              CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:sync-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-sync-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
               echo "Using container: ${CONTAINER_NAME} from instance: ${INSTANCE_ID}"
               sleep 10
           done
-          CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
+          CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:sync-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-sync-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
           echo "::set-output name=zebra_container::$CONTAINER_NAME"
 
       - name: Sync past mandatory checkpoint logs
         id: sync-past-checkpoint
         run: |
           gcloud compute ssh \
-          zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
+          sync-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
           --zone ${{ env.ZONE }} \
           --quiet \
           --ssh-flag="-o ServerAliveInterval=5" \
@@ -384,4 +471,4 @@ jobs:
         if: ${{ steps.sync-past-checkpoint.outcome == 'success' }} || ${{ steps.sync-past-checkpoint.outcome == 'failure' }}
         continue-on-error: true
         run: |
-          gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
+          gcloud compute instances delete "sync-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"

From 3bc3455ca51e899d6468d5f76b022a994143bf6c Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 18 Feb 2022 06:17:17 -0400
Subject: [PATCH 02/24] fix(test): add build

---
 .github/workflows/test.yml | 106 ++++++++++++++++++-------------------
 1 file changed, 53 insertions(+), 53 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 1ce73684171..cbdeda929da 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -33,62 +33,62 @@ env:
   IMAGE_NAME: zebrad-test
 
 jobs:
-  # build:
-  #   name: Build images
-  #   timeout-minutes: 210
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - uses: actions/checkout@v2.4.0
-  #       with:
-  #         persist-credentials: false
+  build:
+    name: Build images
+    timeout-minutes: 210
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2.4.0
+        with:
+          persist-credentials: false
 
-  #     - name: Inject slug/short variables
-  #       uses: rlespinasse/github-slug-action@v4
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v4
 
-  #     # Setup Docker Buildx to allow use of docker cache layers from GH
-  #     - name: Set up Docker Buildx
-  #       uses: docker/setup-buildx-action@v1
-  #       with:
-  #         driver-opts: network=host
+      # Setup Docker Buildx to allow use of docker cache layers from GH
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          driver-opts: network=host
 
-  #     - name: Login to Google Artifact Registry
-  #       uses: docker/login-action@v1.12.0
-  #       with:
-  #         registry: us-docker.pkg.dev
-  #         username: _json_key
-  #         password: ${{ secrets.GOOGLE_CREDENTIALS }}
+      - name: Login to Google Artifact Registry
+        uses: docker/login-action@v1.12.0
+        with:
+          registry: us-docker.pkg.dev
+          username: _json_key
+          password: ${{ secrets.GOOGLE_CREDENTIALS }}
 
-  #     - name: Login to Google Container Registry
-  #       uses: docker/login-action@v1.12.0
-  #       with:
-  #         registry: gcr.io
-  #         username: _json_key
-  #         password: ${{ secrets.GOOGLE_CREDENTIALS }}
-
-  #     # Build and push image to Google Artifact Registry
-  #     - name: Build & push
-  #       id: docker_build
-  #       uses: docker/build-push-action@v2.8.0
-  #       with:
-  #         target: tester
-  #         context: .
-  #         file: ./docker/Dockerfile.build
-  #         tags: |
-  #           ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:latest
-  #           ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
-  #           ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:latest
-  #           ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
-  #         build-args: |
-  #           NETWORK=${{ github.event.inputs.network || env.NETWORK }}
-  #           SHORT_SHA=${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
-  #           RUST_BACKTRACE=full
-  #           ZEBRA_SKIP_NETWORK_TESTS="1"
-  #           CHECKPOINT_SYNC=${{ github.event.inputs.checkpoint_sync || true }}
-  #           RUST_LOG=debug
-  #           SENTRY_DSN=${{ secrets.SENTRY_ENDPOINT }}
-  #         push: true
-  #         cache-from: type=gha
-  #         cache-to: type=gha,mode=max
+      - name: Login to Google Container Registry
+        uses: docker/login-action@v1.12.0
+        with:
+          registry: gcr.io
+          username: _json_key
+          password: ${{ secrets.GOOGLE_CREDENTIALS }}
+
+      # Build and push image to Google Artifact Registry
+      - name: Build & push
+        id: docker_build
+        uses: docker/build-push-action@v2.8.0
+        with:
+          target: tester
+          context: .
+          file: ./docker/Dockerfile.build
+          tags: |
+            ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:latest
+            ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+            ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:latest
+            ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+          build-args: |
+            NETWORK=${{ github.event.inputs.network || env.NETWORK }}
+            SHORT_SHA=${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+            RUST_BACKTRACE=full
+            ZEBRA_SKIP_NETWORK_TESTS="1"
+            CHECKPOINT_SYNC=${{ github.event.inputs.checkpoint_sync || true }}
+            RUST_LOG=debug
+            SENTRY_DSN=${{ secrets.SENTRY_ENDPOINT }}
+          push: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
 
   # # Run all the zebra tests, including tests that are ignored by default
   # test-all:
@@ -390,7 +390,7 @@ jobs:
   test-full-sync:
     name: Test full Mainnet sync
     runs-on: ubuntu-latest
-    # needs: [ build]
+    needs: [ build]
     steps:
       - uses: actions/checkout@v2.4.0
         with:

From f5d710c5f276166c32ba825888f7a18b84b1e68d Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Sun, 20 Feb 2022 23:12:21 -0400
Subject: [PATCH 03/24] fix(deploy): escape double dashes '--' correctly

---
 .github/workflows/test.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index cbdeda929da..342856316a3 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -29,7 +29,7 @@ env:
   GCR_BASE: gcr.io/${{ secrets.GCP_PROJECT_ID }}
   REGION: us-central1
   ZONE: us-central1-a
-  MACHINE_TYPE: c2-standard-8
+  MACHINE_TYPE: c2-standard-16
   IMAGE_NAME: zebrad-test
 
 jobs:
@@ -412,7 +412,7 @@ jobs:
         run: |
           gcloud compute instances create-with-container "sync-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 100GB \
-          --boot-disk-type pd-ssd \
+          --boot-disk-type pd-extreme \
           --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }} \
           --container-restart-policy=never \
           --container-stdin \
@@ -425,7 +425,7 @@ jobs:
           --container-arg="enable-sentry" \
           --container-arg="--test" \
           --container-arg="acceptance" \
-          --container-arg=-- \
+          --container-arg="'--'" \
           --container-arg="--nocapture" \
           --container-arg="--ignored" \
           --container-arg="full_sync_mainnet" \

From b38a1a5a9f9432ea40149089a08d0e9286475637 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Sun, 20 Feb 2022 23:16:43 -0400
Subject: [PATCH 04/24] fix(test): remove unexpected --no-capture arg

error: Found argument '--nocapture' which wasn't expected, or isn't valid in this context
---
 .github/workflows/test.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 342856316a3..c5588d5006e 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -426,7 +426,6 @@ jobs:
           --container-arg="--test" \
           --container-arg="acceptance" \
           --container-arg="'--'" \
-          --container-arg="--nocapture" \
           --container-arg="--ignored" \
           --container-arg="full_sync_mainnet" \
           --container-env=ZEBRA_SKIP_IPV6_TESTS=1,FULL_SYNC_MAINNET_TIMEOUT_MINUTES=600 \

From 3f42944f10e9ef9c7d0edf1328528b2f947adb37 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Mon, 21 Feb 2022 07:37:08 -0400
Subject: [PATCH 05/24] refactor(docker): use default executable as entrypoint

---
 .github/workflows/test.yml | 11 -----------
 docker/Dockerfile.build    |  3 ++-
 2 files changed, 2 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index c5588d5006e..b9206a22c75 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -417,17 +417,6 @@ jobs:
           --container-restart-policy=never \
           --container-stdin \
           --container-tty \
-          --container-command="cargo" \
-          --container-arg="test" \
-          --container-arg="--locked" \
-          --container-arg="--release" \
-          --container-arg="--features" \
-          --container-arg="enable-sentry" \
-          --container-arg="--test" \
-          --container-arg="acceptance" \
-          --container-arg="'--'" \
-          --container-arg="--ignored" \
-          --container-arg="full_sync_mainnet" \
           --container-env=ZEBRA_SKIP_IPV6_TESTS=1,FULL_SYNC_MAINNET_TIMEOUT_MINUTES=600 \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --scopes cloud-platform \
diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index b599474afe5..202ccd578f1 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -63,7 +63,8 @@ ENV ZEBRA_SKIP_IPV6_TESTS ${ZEBRA_SKIP_IPV6_TESTS}
 
 RUN cargo test --locked --release --features enable-sentry --workspace --no-run
 
-CMD ["cargo" "test" "--locked" "--release" "--features" "enable-sentry" "--workspace"]
+ENTRYPOINT ["cargo" "test" "--locked" "--release" "--features" "enable-sentry"]
+CMD [ "--test" "acceptance" "--" "--nocapture" "--ignored" "full_sync_mainnet"]
 
 # Runner image
 FROM debian:bullseye-slim AS runtime

From 6ae180bec6932c387d8578bd55cbf4f2d8cd485a Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Mon, 21 Feb 2022 14:48:55 -0400
Subject: [PATCH 06/24] refactor(startup): add a custom entrypoint

---
 docker/Dockerfile.build | 10 ++++++++--
 docker/entrypoint.sh    | 37 +++++++++++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+), 2 deletions(-)
 create mode 100755 docker/entrypoint.sh

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 202ccd578f1..158231ddb0f 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -61,10 +61,16 @@ COPY --from=us-docker.pkg.dev/zealous-zebra/zebra/zcash-params /root/.zcash-para
 ARG ZEBRA_SKIP_IPV6_TESTS=1
 ENV ZEBRA_SKIP_IPV6_TESTS ${ZEBRA_SKIP_IPV6_TESTS}
 
+ARG RUN_TESTS=1
+ENV RUN_TESTS ${RUN_TESTS}
+
 RUN cargo test --locked --release --features enable-sentry --workspace --no-run
 
-ENTRYPOINT ["cargo" "test" "--locked" "--release" "--features" "enable-sentry"]
-CMD [ "--test" "acceptance" "--" "--nocapture" "--ignored" "full_sync_mainnet"]
+COPY ./docker/entrypoint.sh /
+RUN chmod u+x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD [ "cargo"]
 
 # Runner image
 FROM debian:bullseye-slim AS runtime
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
new file mode 100755
index 00000000000..eb8ff6d8f63
--- /dev/null
+++ b/docker/entrypoint.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+set -x
+
+if [ ! -f zebrad.toml ]; then
+echo "
+[consensus]
+checkpoint_sync = ${CHECKPOINT_SYNC}
+[metrics]
+endpoint_addr = '0.0.0.0:9999'
+[network]
+network = ${NETWORK}
+[state]
+cache_dir = '/zebrad-cache'
+[tracing]
+endpoint_addr = '0.0.0.0:3000'
+" > $zebrad.toml
+fi
+
+case "$1" in
+    -- | cargo)
+        if [[ "$RUN_TESTS" -eq "1" ]]; then
+            if [[ "$TEST_FULL_SYNC" -eq "1" ]]; then
+                exec cargo "test" "--locked" "--release" "--features" "enable-sentry" "--test" "acceptance" "--" "--nocapture" "--ignored" "full_sync_mainnet"
+            else
+                exec cargo "test" "--locked" "--release" "--features" "enable-sentry" "--workspace" "--" "--include-ignored"
+            fi
+        fi
+        ;;
+    zebrad)
+        exec zebrad "$@"
+        ;;
+    *)
+        exec "$@"
+esac
+
+exit 1
\ No newline at end of file

From f7014c10962f0af8232195940e7bcc3d9f13e28b Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Mon, 21 Feb 2022 14:54:19 -0400
Subject: [PATCH 07/24] fix(test): add missing TEST_FULL_SYNC variable

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index b9206a22c75..0edaef6c921 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -417,7 +417,7 @@ jobs:
           --container-restart-policy=never \
           --container-stdin \
           --container-tty \
-          --container-env=ZEBRA_SKIP_IPV6_TESTS=1,FULL_SYNC_MAINNET_TIMEOUT_MINUTES=600 \
+          --container-env=ZEBRA_SKIP_IPV6_TESTS=1,TEST_FULL_SYNC=1,FULL_SYNC_MAINNET_TIMEOUT_MINUTES=600 \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --scopes cloud-platform \
           --metadata=google-monitoring-enabled=true,google-logging-enabled=true \

From 445a3dd472de704a54f347278a9cf9f12dd190f4 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Mon, 21 Feb 2022 23:10:29 -0400
Subject: [PATCH 08/24] test(timeout): use the biggest machine

---
 .github/workflows/test.yml | 106 ++++++++++++++++++-------------------
 1 file changed, 53 insertions(+), 53 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0edaef6c921..230de841c2f 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -29,66 +29,66 @@ env:
   GCR_BASE: gcr.io/${{ secrets.GCP_PROJECT_ID }}
   REGION: us-central1
   ZONE: us-central1-a
-  MACHINE_TYPE: c2-standard-16
+  MACHINE_TYPE: c2d-standard-112
   IMAGE_NAME: zebrad-test
 
 jobs:
-  build:
-    name: Build images
-    timeout-minutes: 210
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2.4.0
-        with:
-          persist-credentials: false
+  # build:
+  #   name: Build images
+  #   timeout-minutes: 210
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v2.4.0
+  #       with:
+  #         persist-credentials: false
 
-      - name: Inject slug/short variables
-        uses: rlespinasse/github-slug-action@v4
+  #     - name: Inject slug/short variables
+  #       uses: rlespinasse/github-slug-action@v4
 
-      # Setup Docker Buildx to allow use of docker cache layers from GH
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          driver-opts: network=host
+  #     # Setup Docker Buildx to allow use of docker cache layers from GH
+  #     - name: Set up Docker Buildx
+  #       uses: docker/setup-buildx-action@v1
+  #       with:
+  #         driver-opts: network=host
 
-      - name: Login to Google Artifact Registry
-        uses: docker/login-action@v1.12.0
-        with:
-          registry: us-docker.pkg.dev
-          username: _json_key
-          password: ${{ secrets.GOOGLE_CREDENTIALS }}
+  #     - name: Login to Google Artifact Registry
+  #       uses: docker/login-action@v1.12.0
+  #       with:
+  #         registry: us-docker.pkg.dev
+  #         username: _json_key
+  #         password: ${{ secrets.GOOGLE_CREDENTIALS }}
 
-      - name: Login to Google Container Registry
-        uses: docker/login-action@v1.12.0
-        with:
-          registry: gcr.io
-          username: _json_key
-          password: ${{ secrets.GOOGLE_CREDENTIALS }}
-
-      # Build and push image to Google Artifact Registry
-      - name: Build & push
-        id: docker_build
-        uses: docker/build-push-action@v2.8.0
-        with:
-          target: tester
-          context: .
-          file: ./docker/Dockerfile.build
-          tags: |
-            ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:latest
-            ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
-            ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:latest
-            ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
-          build-args: |
-            NETWORK=${{ github.event.inputs.network || env.NETWORK }}
-            SHORT_SHA=${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
-            RUST_BACKTRACE=full
-            ZEBRA_SKIP_NETWORK_TESTS="1"
-            CHECKPOINT_SYNC=${{ github.event.inputs.checkpoint_sync || true }}
-            RUST_LOG=debug
-            SENTRY_DSN=${{ secrets.SENTRY_ENDPOINT }}
-          push: true
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+  #     - name: Login to Google Container Registry
+  #       uses: docker/login-action@v1.12.0
+  #       with:
+  #         registry: gcr.io
+  #         username: _json_key
+  #         password: ${{ secrets.GOOGLE_CREDENTIALS }}
+
+  #     # Build and push image to Google Artifact Registry
+  #     - name: Build & push
+  #       id: docker_build
+  #       uses: docker/build-push-action@v2.8.0
+  #       with:
+  #         target: tester
+  #         context: .
+  #         file: ./docker/Dockerfile.build
+  #         tags: |
+  #           ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:latest
+  #           ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+  #           ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:latest
+  #           ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+  #         build-args: |
+  #           NETWORK=${{ github.event.inputs.network || env.NETWORK }}
+  #           SHORT_SHA=${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+  #           RUST_BACKTRACE=full
+  #           ZEBRA_SKIP_NETWORK_TESTS="1"
+  #           CHECKPOINT_SYNC=${{ github.event.inputs.checkpoint_sync || true }}
+  #           RUST_LOG=debug
+  #           SENTRY_DSN=${{ secrets.SENTRY_ENDPOINT }}
+  #         push: true
+  #         cache-from: type=gha
+  #         cache-to: type=gha,mode=max
 
   # # Run all the zebra tests, including tests that are ignored by default
   # test-all:

From 563f8caececc8f16cbcd437c2732c09bce9ec19f Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Mon, 21 Feb 2022 23:17:20 -0400
Subject: [PATCH 09/24] fix

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 230de841c2f..0a7eb8941c2 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -390,7 +390,7 @@ jobs:
   test-full-sync:
     name: Test full Mainnet sync
     runs-on: ubuntu-latest
-    needs: [ build]
+    # needs: [ build]
     steps:
       - uses: actions/checkout@v2.4.0
         with:

From 882da522c7c7c555564fef52fe30cc4166969d4f Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Mon, 21 Feb 2022 23:32:01 -0400
Subject: [PATCH 10/24] fix(deploy): use latest successful image

---
 .github/workflows/test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0a7eb8941c2..93101c50150 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -390,7 +390,7 @@ jobs:
   test-full-sync:
     name: Test full Mainnet sync
     runs-on: ubuntu-latest
-    # needs: [ build]
+    needs: [ build]
     steps:
       - uses: actions/checkout@v2.4.0
         with:
@@ -413,7 +413,7 @@ jobs:
           gcloud compute instances create-with-container "sync-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 100GB \
           --boot-disk-type pd-extreme \
-          --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }} \
+          --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:f7014c1 \
           --container-restart-policy=never \
           --container-stdin \
           --container-tty \

From f37f04439418de4599fd94e9fbb7b14025e841dd Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Mon, 21 Feb 2022 23:34:20 -0400
Subject: [PATCH 11/24] typo

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 93101c50150..1812ed5faeb 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -390,7 +390,7 @@ jobs:
   test-full-sync:
     name: Test full Mainnet sync
     runs-on: ubuntu-latest
-    needs: [ build]
+    # needs: [ build]
     steps:
       - uses: actions/checkout@v2.4.0
         with:

From 56e77c1a6acf09f06a851d5ee1ee687da07bd0f1 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 22 Feb 2022 10:16:09 -0400
Subject: [PATCH 12/24] refactor(docker): generate config file at startup

---
 docker/Dockerfile.build | 25 +++++++++++++------------
 docker/entrypoint.sh    | 10 ++++++++--
 2 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 158231ddb0f..a658adcefbe 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -45,8 +45,19 @@ ENV CARGO_HOME /app/.cargo/
 # Build dependencies - this is the caching Docker layer!
 RUN cargo chef cook --release --features enable-sentry --recipe-path recipe.json
 
-ARG RUST_BACKTRACE=1
-ENV RUST_BACKTRACE ${RUST_BACKTRACE}
+ARG RUST_BACKTRACE
+ENV RUST_BACKTRACE ${RUST_BACKTRACE:-1}
+
+# Skip IPv6 tests by default, as some CI environment don't have IPv6 available
+ARG ZEBRA_SKIP_IPV6_TESTS
+ENV ZEBRA_SKIP_IPV6_TESTS ${ZEBRA_SKIP_IPV6_TESTS:-1}
+
+# Use default checkpoint sync and network values if none is provided
+ARG CHECKPOINT_SYNC
+ENV CHECKPOINT_SYNC ${CHECKPOINT_SYNC:-true}
+
+ARG NETWORK
+ENV NETWORK ${NETWORK:-Mainnet}
 
 COPY . .
 # Build zebra
@@ -57,13 +68,6 @@ FROM builder AS tester
 # TODO: do not hardcode the user /root/ even though is a safe assumption
 COPY --from=us-docker.pkg.dev/zealous-zebra/zebra/zcash-params /root/.zcash-params /root/.zcash-params
 
-# Skip IPv6 tests by default, as some CI environment don't have IPv6 available
-ARG ZEBRA_SKIP_IPV6_TESTS=1
-ENV ZEBRA_SKIP_IPV6_TESTS ${ZEBRA_SKIP_IPV6_TESTS}
-
-ARG RUN_TESTS=1
-ENV RUN_TESTS ${RUN_TESTS}
-
 RUN cargo test --locked --release --features enable-sentry --workspace --no-run
 
 COPY ./docker/entrypoint.sh /
@@ -81,9 +85,6 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     ca-certificates
 
-ARG CHECKPOINT_SYNC=true
-ARG NETWORK=Mainnet
-
 RUN set -ex; \
   { \
     echo "[consensus]"; \
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index eb8ff6d8f63..a9143b10768 100755
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -9,18 +9,24 @@ checkpoint_sync = ${CHECKPOINT_SYNC}
 [metrics]
 endpoint_addr = '0.0.0.0:9999'
 [network]
-network = ${NETWORK}
+network = '${NETWORK}'
 [state]
 cache_dir = '/zebrad-cache'
 [tracing]
+force_use_color = true
 endpoint_addr = '0.0.0.0:3000'
-" > $zebrad.toml
+" > zebrad.toml
 fi
 
 case "$1" in
     -- | cargo)
         if [[ "$RUN_TESTS" -eq "1" ]]; then
             if [[ "$TEST_FULL_SYNC" -eq "1" ]]; then
+                # exec cargo "build" "--release" "--bin" "zebra-checkpoints"
+                # LAST_CHECKPOINT=$(tail -1 ./zebra-consensus/src/checkpoint/main-checkpoints.txt | cut -d" " -f1)
+                # echo ${LAST_CHECKPOINT}
+                # ./target/release/zebra-checkpoints --last-checkpoint ${LAST_CHECKPOINT}
+                # ./target/release/zebra-checkpoints --last-checkpoint ${LAST_CHECKPOINT} -- -testnet
                 exec cargo "test" "--locked" "--release" "--features" "enable-sentry" "--test" "acceptance" "--" "--nocapture" "--ignored" "full_sync_mainnet"
             else
                 exec cargo "test" "--locked" "--release" "--features" "enable-sentry" "--workspace" "--" "--include-ignored"

From 707cb4365726d28cfa6fc00892d5b7efe77eb474 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 22 Feb 2022 11:46:41 -0400
Subject: [PATCH 13/24] revert(build): changes were made to docker

---
 .github/workflows/test.yml | 108 ++++++++++++++++++-------------------
 1 file changed, 54 insertions(+), 54 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 1812ed5faeb..aa11a880e64 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -33,62 +33,62 @@ env:
   IMAGE_NAME: zebrad-test
 
 jobs:
-  # build:
-  #   name: Build images
-  #   timeout-minutes: 210
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - uses: actions/checkout@v2.4.0
-  #       with:
-  #         persist-credentials: false
+  build:
+    name: Build images
+    timeout-minutes: 210
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2.4.0
+        with:
+          persist-credentials: false
 
-  #     - name: Inject slug/short variables
-  #       uses: rlespinasse/github-slug-action@v4
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v4
 
-  #     # Setup Docker Buildx to allow use of docker cache layers from GH
-  #     - name: Set up Docker Buildx
-  #       uses: docker/setup-buildx-action@v1
-  #       with:
-  #         driver-opts: network=host
+      # Setup Docker Buildx to allow use of docker cache layers from GH
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          driver-opts: network=host
 
-  #     - name: Login to Google Artifact Registry
-  #       uses: docker/login-action@v1.12.0
-  #       with:
-  #         registry: us-docker.pkg.dev
-  #         username: _json_key
-  #         password: ${{ secrets.GOOGLE_CREDENTIALS }}
+      - name: Login to Google Artifact Registry
+        uses: docker/login-action@v1.12.0
+        with:
+          registry: us-docker.pkg.dev
+          username: _json_key
+          password: ${{ secrets.GOOGLE_CREDENTIALS }}
 
-  #     - name: Login to Google Container Registry
-  #       uses: docker/login-action@v1.12.0
-  #       with:
-  #         registry: gcr.io
-  #         username: _json_key
-  #         password: ${{ secrets.GOOGLE_CREDENTIALS }}
-
-  #     # Build and push image to Google Artifact Registry
-  #     - name: Build & push
-  #       id: docker_build
-  #       uses: docker/build-push-action@v2.8.0
-  #       with:
-  #         target: tester
-  #         context: .
-  #         file: ./docker/Dockerfile.build
-  #         tags: |
-  #           ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:latest
-  #           ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
-  #           ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:latest
-  #           ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
-  #         build-args: |
-  #           NETWORK=${{ github.event.inputs.network || env.NETWORK }}
-  #           SHORT_SHA=${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
-  #           RUST_BACKTRACE=full
-  #           ZEBRA_SKIP_NETWORK_TESTS="1"
-  #           CHECKPOINT_SYNC=${{ github.event.inputs.checkpoint_sync || true }}
-  #           RUST_LOG=debug
-  #           SENTRY_DSN=${{ secrets.SENTRY_ENDPOINT }}
-  #         push: true
-  #         cache-from: type=gha
-  #         cache-to: type=gha,mode=max
+      - name: Login to Google Container Registry
+        uses: docker/login-action@v1.12.0
+        with:
+          registry: gcr.io
+          username: _json_key
+          password: ${{ secrets.GOOGLE_CREDENTIALS }}
+
+      # Build and push image to Google Artifact Registry
+      - name: Build & push
+        id: docker_build
+        uses: docker/build-push-action@v2.8.0
+        with:
+          target: tester
+          context: .
+          file: ./docker/Dockerfile.build
+          tags: |
+            ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:latest
+            ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+            ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:latest
+            ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+          build-args: |
+            NETWORK=${{ github.event.inputs.network || env.NETWORK }}
+            SHORT_SHA=${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+            RUST_BACKTRACE=full
+            ZEBRA_SKIP_NETWORK_TESTS="1"
+            CHECKPOINT_SYNC=${{ github.event.inputs.checkpoint_sync || true }}
+            RUST_LOG=debug
+            SENTRY_DSN=${{ secrets.SENTRY_ENDPOINT }}
+          push: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
 
   # # Run all the zebra tests, including tests that are ignored by default
   # test-all:
@@ -390,7 +390,7 @@ jobs:
   test-full-sync:
     name: Test full Mainnet sync
     runs-on: ubuntu-latest
-    # needs: [ build]
+    needs: [ build]
     steps:
       - uses: actions/checkout@v2.4.0
         with:
@@ -413,7 +413,7 @@ jobs:
           gcloud compute instances create-with-container "sync-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 100GB \
           --boot-disk-type pd-extreme \
-          --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:f7014c1 \
+          --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
           --container-restart-policy=never \
           --container-stdin \
           --container-tty \

From ec076a0f50713feb147a22ce28644a9225f05d80 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 22 Feb 2022 12:53:30 -0400
Subject: [PATCH 14/24] fix(docker): send variables correctly to the entrypoint

---
 docker/Dockerfile.build |  3 +++
 docker/entrypoint.sh    | 11 +++++------
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index a658adcefbe..f3b9e3244e7 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -59,6 +59,9 @@ ENV CHECKPOINT_SYNC ${CHECKPOINT_SYNC:-true}
 ARG NETWORK
 ENV NETWORK ${NETWORK:-Mainnet}
 
+ARG TEST_FULL_SYNC
+ENV TEST_FULL_SYNC ${TEST_FULL_SYNC:-1}
+
 COPY . .
 # Build zebra
 RUN cargo build --locked --release --features enable-sentry --bin zebrad
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index a9143b10768..52fcac505b6 100755
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -2,20 +2,19 @@
 
 set -x
 
-if [ ! -f zebrad.toml ]; then
+if [ ! -f /app/zebrad.toml ]; then
 echo "
 [consensus]
 checkpoint_sync = ${CHECKPOINT_SYNC}
 [metrics]
-endpoint_addr = '0.0.0.0:9999'
+endpoint_addr = 0.0.0.0:9999
 [network]
-network = '${NETWORK}'
+network = ${NETWORK}
 [state]
-cache_dir = '/zebrad-cache'
+cache_dir = /zebrad-cache
 [tracing]
 force_use_color = true
-endpoint_addr = '0.0.0.0:3000'
-" > zebrad.toml
+endpoint_addr = 0.0.0.0:3000" > /app/zebrad.toml
 fi
 
 case "$1" in

From 3ded9fa36a73425955a0a6a648b1183b96e5ce63 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 22 Feb 2022 13:57:16 -0400
Subject: [PATCH 15/24] test different conf file approach

---
 docker/Dockerfile.build | 20 ++++++++++++++++++++
 docker/entrypoint.sh    | 33 ++++++++++++++-------------------
 2 files changed, 34 insertions(+), 19 deletions(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index f3b9e3244e7..166c17dd35f 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -76,6 +76,26 @@ RUN cargo test --locked --release --features enable-sentry --workspace --no-run
 COPY ./docker/entrypoint.sh /
 RUN chmod u+x /entrypoint.sh
 
+ARG CHECKPOINT_SYNC=true
+ARG NETWORK=Mainnet
+ARG TEST_FULL_SYNC
+ENV TEST_FULL_SYNC ${TEST_FULL_SYNC:-1}
+
+RUN set -ex; \
+  { \
+    echo "[consensus]"; \
+    echo "checkpoint_sync = ${CHECKPOINT_SYNC}"; \
+    echo "[metrics]"; \
+    echo "endpoint_addr = '0.0.0.0:9999'"; \
+    echo "[network]"; \
+    echo "network = '${NETWORK}'"; \
+    echo "[state]"; \
+    echo "cache_dir = '/zebrad-cache'"; \
+    echo "[tracing]"; \
+    echo "force_use_color = true"; \
+    echo "endpoint_addr = '0.0.0.0:3000'"; \
+  } > "zebrad.toml"
+
 ENTRYPOINT ["/entrypoint.sh"]
 CMD [ "cargo"]
 
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index 52fcac505b6..148f3338757 100755
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -2,30 +2,25 @@
 
 set -x
 
-if [ ! -f /app/zebrad.toml ]; then
-echo "
-[consensus]
-checkpoint_sync = ${CHECKPOINT_SYNC}
-[metrics]
-endpoint_addr = 0.0.0.0:9999
-[network]
-network = ${NETWORK}
-[state]
-cache_dir = /zebrad-cache
-[tracing]
-force_use_color = true
-endpoint_addr = 0.0.0.0:3000" > /app/zebrad.toml
-fi
+# if [ ! -f /app/zebrad.toml ]; then
+# echo "
+# [consensus]
+# checkpoint_sync = ${CHECKPOINT_SYNC}
+# [metrics]
+# endpoint_addr = 0.0.0.0:9999
+# [network]
+# network = ${NETWORK}
+# [state]
+# cache_dir = /zebrad-cache
+# [tracing]
+# force_use_color = true
+# endpoint_addr = 0.0.0.0:3000" > /app/zebrad.toml
+# fi
 
 case "$1" in
     -- | cargo)
         if [[ "$RUN_TESTS" -eq "1" ]]; then
             if [[ "$TEST_FULL_SYNC" -eq "1" ]]; then
-                # exec cargo "build" "--release" "--bin" "zebra-checkpoints"
-                # LAST_CHECKPOINT=$(tail -1 ./zebra-consensus/src/checkpoint/main-checkpoints.txt | cut -d" " -f1)
-                # echo ${LAST_CHECKPOINT}
-                # ./target/release/zebra-checkpoints --last-checkpoint ${LAST_CHECKPOINT}
-                # ./target/release/zebra-checkpoints --last-checkpoint ${LAST_CHECKPOINT} -- -testnet
                 exec cargo "test" "--locked" "--release" "--features" "enable-sentry" "--test" "acceptance" "--" "--nocapture" "--ignored" "full_sync_mainnet"
             else
                 exec cargo "test" "--locked" "--release" "--features" "enable-sentry" "--workspace" "--" "--include-ignored"

From 38c6a5b1a5ad80dd514536354763ffee5e3bd36c Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 22 Feb 2022 15:55:52 -0400
Subject: [PATCH 16/24] fix(env): add RUN_TEST env variable

---
 docker/Dockerfile.build | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 166c17dd35f..1d1fdbe2cfa 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -81,6 +81,9 @@ ARG NETWORK=Mainnet
 ARG TEST_FULL_SYNC
 ENV TEST_FULL_SYNC ${TEST_FULL_SYNC:-1}
 
+ARG RUN_TESTS
+ENV RUN_TESTS ${RUN_TESTS:-1}
+
 RUN set -ex; \
   { \
     echo "[consensus]"; \

From 06ccd6ea90456411b0606ae749dc8c18bfb0a661 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 22 Feb 2022 15:56:38 -0400
Subject: [PATCH 17/24] ref: use previous approach

---
 docker/Dockerfile.build | 15 ---------------
 docker/entrypoint.sh    | 28 ++++++++++++++--------------
 2 files changed, 14 insertions(+), 29 deletions(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 1d1fdbe2cfa..d1c63b6b06d 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -84,21 +84,6 @@ ENV TEST_FULL_SYNC ${TEST_FULL_SYNC:-1}
 ARG RUN_TESTS
 ENV RUN_TESTS ${RUN_TESTS:-1}
 
-RUN set -ex; \
-  { \
-    echo "[consensus]"; \
-    echo "checkpoint_sync = ${CHECKPOINT_SYNC}"; \
-    echo "[metrics]"; \
-    echo "endpoint_addr = '0.0.0.0:9999'"; \
-    echo "[network]"; \
-    echo "network = '${NETWORK}'"; \
-    echo "[state]"; \
-    echo "cache_dir = '/zebrad-cache'"; \
-    echo "[tracing]"; \
-    echo "force_use_color = true"; \
-    echo "endpoint_addr = '0.0.0.0:3000'"; \
-  } > "zebrad.toml"
-
 ENTRYPOINT ["/entrypoint.sh"]
 CMD [ "cargo"]
 
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index 148f3338757..c668a6b16c3 100755
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -2,20 +2,20 @@
 
 set -x
 
-# if [ ! -f /app/zebrad.toml ]; then
-# echo "
-# [consensus]
-# checkpoint_sync = ${CHECKPOINT_SYNC}
-# [metrics]
-# endpoint_addr = 0.0.0.0:9999
-# [network]
-# network = ${NETWORK}
-# [state]
-# cache_dir = /zebrad-cache
-# [tracing]
-# force_use_color = true
-# endpoint_addr = 0.0.0.0:3000" > /app/zebrad.toml
-# fi
+if [ ! -f /app/zebrad.toml ]; then
+echo "
+[consensus]
+checkpoint_sync = ${CHECKPOINT_SYNC}
+[metrics]
+endpoint_addr = 0.0.0.0:9999
+[network]
+network = ${NETWORK}
+[state]
+cache_dir = /zebrad-cache
+[tracing]
+force_use_color = true
+endpoint_addr = 0.0.0.0:3000" > /app/zebrad.toml
+fi
 
 case "$1" in
     -- | cargo)

From c46dd50aa6dd766abae6d7c0f68607e184e26808 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 22 Feb 2022 20:10:59 -0400
Subject: [PATCH 18/24] fix(color): use environment variable

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index aa11a880e64..f5761f4ca02 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -417,7 +417,7 @@ jobs:
           --container-restart-policy=never \
           --container-stdin \
           --container-tty \
-          --container-env=ZEBRA_SKIP_IPV6_TESTS=1,TEST_FULL_SYNC=1,FULL_SYNC_MAINNET_TIMEOUT_MINUTES=600 \
+          --container-env=ZEBRA_SKIP_IPV6_TESTS=1,TEST_FULL_SYNC=1,ZEBRA_FORCE_USE_COLOR=1,FULL_SYNC_MAINNET_TIMEOUT_MINUTES=600 \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --scopes cloud-platform \
           --metadata=google-monitoring-enabled=true,google-logging-enabled=true \

From 2452df2c06aa7209d7fdca3928a17f66d311b1fd Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Wed, 23 Feb 2022 18:09:50 -0400
Subject: [PATCH 19/24] fix(resources): use our normal machine size

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index f5761f4ca02..6ceb78492d3 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -29,7 +29,7 @@ env:
   GCR_BASE: gcr.io/${{ secrets.GCP_PROJECT_ID }}
   REGION: us-central1
   ZONE: us-central1-a
-  MACHINE_TYPE: c2d-standard-112
+  MACHINE_TYPE: c2d-standard-8
   IMAGE_NAME: zebrad-test
 
 jobs:

From 48b201db50cd61f63c39ed9a5cb1f213c500947d Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Fri, 25 Feb 2022 06:47:12 +1000
Subject: [PATCH 20/24] fix(ci): double CPU and RAM for full sync test

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6ceb78492d3..502c2774e9c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -29,7 +29,7 @@ env:
   GCR_BASE: gcr.io/${{ secrets.GCP_PROJECT_ID }}
   REGION: us-central1
   ZONE: us-central1-a
-  MACHINE_TYPE: c2d-standard-8
+  MACHINE_TYPE: c2d-standard-16
   IMAGE_NAME: zebrad-test
 
 jobs:

From 2669338267d9e8ee71694e55cbb98f2ffec267af Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Fri, 25 Feb 2022 14:56:46 +1000
Subject: [PATCH 21/24] fix(test): check for zebrad test output in the correct
 order

The mempool is only activated once, so we must check for that log first.
After mempool activation, the stop regex is logged at least once.
(It might be logged before as well, but we can't rely on that.)

When checking that the mempool didn't activate,
wait for the `zebrad` command to exit,
then check the entire log.
---
 zebra-test/src/command.rs  |  12 +++-
 zebrad/tests/acceptance.rs | 128 +++++++++++++++++++++++++------------
 2 files changed, 96 insertions(+), 44 deletions(-)

diff --git a/zebra-test/src/command.rs b/zebra-test/src/command.rs
index e54301d0ffc..623d9f391f1 100644
--- a/zebra-test/src/command.rs
+++ b/zebra-test/src/command.rs
@@ -382,11 +382,17 @@ impl<T> TestChild<T> {
 }
 
 pub struct TestOutput<T> {
+    /// The test directory for this test output.
+    ///
+    /// Keeps the test dir around from `TestChild`,
+    /// so it doesn't get deleted during `wait_with_output`.
     #[allow(dead_code)]
-    // this just keeps the test dir around from `TestChild` so it doesn't get
-    // deleted during `wait_with_output`
-    dir: Option<T>,
+    pub dir: Option<T>,
+
+    /// The test command for this test output.
     pub cmd: String,
+
+    /// The test exit status, standard out, and standard error.
     pub output: Output,
 }
 
diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs
index f89a9045f7b..b888e5e1997 100644
--- a/zebrad/tests/acceptance.rs
+++ b/zebrad/tests/acceptance.rs
@@ -1066,56 +1066,71 @@ fn sync_until(
     let mut child = tempdir.spawn_child(&["start"])?.with_timeout(timeout);
 
     let network = format!("network: {},", network);
-    child.expect_stdout_line_matches(&network)?;
 
-    if check_legacy_chain {
-        child.expect_stdout_line_matches("starting legacy chain check")?;
-        child.expect_stdout_line_matches("no legacy chain found")?;
-    }
+    if mempool_behavior.require_activation() {
+        // require that the mempool activated,
+        // checking logs as they arrive
 
-    if mempool_behavior.is_forced_activation() {
-        child.expect_stdout_line_matches("enabling mempool for debugging")?;
+        child.expect_stdout_line_matches(&network)?;
+
+        if check_legacy_chain {
+            child.expect_stdout_line_matches("starting legacy chain check")?;
+            child.expect_stdout_line_matches("no legacy chain found")?;
+        }
+
+        // before the stop regex, expect mempool activation
+        if mempool_behavior.require_forced_activation() {
+            child.expect_stdout_line_matches("enabling mempool for debugging")?;
+        }
         child.expect_stdout_line_matches("activating mempool")?;
 
-        // make sure zebra is running with the mempool
-        child.expect_stdout_line_matches("verified checkpoint range")?;
-    }
+        // then wait for the stop log, which must happen after the mempool becomes active
+        child.expect_stdout_line_matches(stop_regex)?;
 
-    child.expect_stdout_line_matches(stop_regex)?;
+        // make sure the child process is dead
+        // if it has already exited, ignore that error
+        let _ = child.kill();
 
-    // make sure mempool behaves as expected when we don't explicitly enable it
-    if !mempool_behavior.is_forced_activation() {
-        // if there is no matching line, the `expect_stdout_line_matches` error kills the `zebrad` child.
-        // the error is delayed until the test timeout, or until the child reaches the stop height and exits.
-        let mempool_is_activated = child
-            .expect_stdout_line_matches("activating mempool")
-            .is_ok();
+        Ok(child.dir)
+    } else {
+        // Require that the mempool didn't activate,
+        // checking the entire `zebrad` output after it exits.
+        //
+        // # Correctness
+        //
+        // Unlike the other mempool behaviours, `zebrad` must stop after logging the stop regex,
+        // without being killed by [`sync_until`] test harness.
+        //
+        // Since it needs to collect all the output,
+        // the test harness can't kill `zebrad` after it logs the `stop_regex`.
+        assert!(
+            height.0 < 2_000_000,
+            "zebrad must exit by itself, so we can collect all the output",
+        );
+        let output = child.wait_with_output()?;
 
-        let mempool_check = match mempool_behavior {
-            MempoolBehavior::ShouldAutomaticallyActivate if !mempool_is_activated => {
-                Some("mempool did not activate as expected")
-            }
-            MempoolBehavior::ShouldNotActivate if mempool_is_activated => Some(
-                "unexpected mempool activation: \
-                mempool should not activate while syncing lots of blocks",
-            ),
-            MempoolBehavior::ForceActivationAt(_) => unreachable!("checked by outer if condition"),
-            _ => None,
-        };
-
-        if let Some(error) = mempool_check {
-            // if the mempool does not behave as expected, we panic and kill the test process.
-            // but we also need to kill the `zebrad` child before the test panics.
-            child.kill()?;
-            panic!("{error}")
+        output.stdout_line_contains(&network)?;
+
+        if check_legacy_chain {
+            output.stdout_line_contains("starting legacy chain check")?;
+            output.stdout_line_contains("no legacy chain found")?;
         }
-    }
 
-    // make sure the child process is dead
-    // if it has already exited, ignore that error
-    let _ = child.kill();
+        // check it did not activate or use the mempool
+        assert!(output.stdout_line_contains("activating mempool").is_err());
+        assert!(output
+            .stdout_line_contains("sending mempool transaction broadcast")
+            .is_err());
+
+        // check it logged the stop regex before exiting
+        output.stdout_line_matches(stop_regex)?;
 
-    Ok(child.dir)
+        // check exited by itself, successfully
+        output.assert_was_not_killed()?;
+        let output = output.assert_success()?;
+
+        Ok(output.dir.expect("wait_with_output sets dir"))
+    }
 }
 
 fn cached_mandatory_checkpoint_test_config() -> Result<ZebradConfig> {
@@ -1803,12 +1818,26 @@ where
 /// What the expected behavior of the mempool is for a test that uses [`sync_until`].
 enum MempoolBehavior {
     /// The mempool should be forced to activate at a certain height, for debug purposes.
+    ///
+    /// [`sync_until`] will kill `zebrad` after it logs mempool activation,
+    /// then the `stop_regex`.
     ForceActivationAt(Height),
 
     /// The mempool should be automatically activated.
+    ///
+    /// [`sync_until`] will kill `zebrad` after it logs mempool activation,
+    /// then the `stop_regex`.
     ShouldAutomaticallyActivate,
 
     /// The mempool should not become active during the test.
+    ///
+    /// # Correctness
+    ///
+    /// Unlike the other mempool behaviours, `zebrad` must stop after logging the stop regex,
+    /// without being killed by [`sync_until`] test harness.
+    ///
+    /// Since it needs to collect all the output,
+    /// the test harness can't kill `zebrad` after it logs the `stop_regex`.
     ShouldNotActivate,
 }
 
@@ -1823,8 +1852,25 @@ impl MempoolBehavior {
         }
     }
 
+    /// Returns `true` if the mempool should activate,
+    /// either by forced or automatic activation.
+    pub fn require_activation(&self) -> bool {
+        self.require_forced_activation() || self.require_automatic_activation()
+    }
+
     /// Returns `true` if the mempool should be forcefully activated at a specified height.
-    pub fn is_forced_activation(&self) -> bool {
+    pub fn require_forced_activation(&self) -> bool {
         matches!(self, MempoolBehavior::ForceActivationAt(_))
     }
+
+    /// Returns `true` if the mempool should automatically activate.
+    pub fn require_automatic_activation(&self) -> bool {
+        matches!(self, MempoolBehavior::ShouldAutomaticallyActivate)
+    }
+
+    /// Returns `true` if the mempool should not activate.
+    #[allow(dead_code)]
+    pub fn require_no_activation(&self) -> bool {
+        matches!(self, MempoolBehavior::ShouldNotActivate)
+    }
 }

From 023ea40a381d3e06047e953cf8cd396b84ef1bd3 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Fri, 25 Feb 2022 16:22:43 +1000
Subject: [PATCH 22/24] fix(ci): run full sync test with full compiler
 optimisations

---
 docker/Dockerfile.build | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index d1c63b6b06d..a51334a23a2 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -40,7 +40,10 @@ RUN apt-get -qq update && \
 
 # Optimize builds. In particular, regenerate-stateful-test-disks.yml was reaching the
 # GitHub Actions time limit (6 hours), so we needed to make it faster.
-ENV RUSTFLAGS -O
+#
+# TODO: apply this optimisation level to all release builds in .cargo/config.toml
+ENV RUSTFLAGS "-C opt-level=3"
+
 ENV CARGO_HOME /app/.cargo/
 # Build dependencies - this is the caching Docker layer!
 RUN cargo chef cook --release --features enable-sentry --recipe-path recipe.json

From 90a8e361ea0a7458ee5cc803d3d842d940c09918 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Mar 2022 06:45:12 -0400
Subject: [PATCH 23/24] fix(tests): reintroduce tests and run full sync on
 approval

---
 .github/workflows/test.yml | 606 +++++++++++++++++++------------------
 1 file changed, 309 insertions(+), 297 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 502c2774e9c..770fc9364f1 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -19,6 +19,17 @@ on:
       - '**/Cargo.lock'
       - 'docker/**'
       - '.github/workflows/test.yml'
+  pull_request_review:
+    branches:
+      - main
+    paths:
+      - '**/*.rs'
+      - '**/*.txt'
+      - '**/Cargo.toml'
+      - '**/Cargo.lock'
+      - 'docker/**'
+      - '.github/workflows/test.yml'
+    types: [submitted]
 
 env:
   CARGO_INCREMENTAL: '1'
@@ -90,307 +101,308 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
-  # # Run all the zebra tests, including tests that are ignored by default
-  # test-all:
-  #   name: Test all
-  #   runs-on: ubuntu-latest
-  #   needs: build
-  #   if: ${{ github.event.inputs.regenerate-disks != 'true' }}
-  #   steps:
-  #     - uses: actions/checkout@v2.4.0
-  #       with:
-  #         persist-credentials: false
-
-  #     - name: Inject slug/short variables
-  #       uses: rlespinasse/github-slug-action@v4
-
-  #     - name: Run all zebrad tests
-  #       run: |
-  #         docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
-  #         docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --workspace -- --include-ignored
-
-  # test-fake-activation-heights:
-  #   name: Test with fake activation heights
-  #   runs-on: ubuntu-latest
-  #   needs: build
-  #   if: ${{ github.event.inputs.regenerate-disks != 'true' }}
-  #   steps:
-  #     - uses: actions/checkout@v2.4.0
-  #       with:
-  #         persist-credentials: false
-
-  #     - name: Inject slug/short variables
-  #       uses: rlespinasse/github-slug-action@v4
-
-  #     - name: Run tests with fake activation heights
-  #       run: |
-  #         docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
-  #         docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --package zebra-state --lib -- with_fake_activation_heights
-
-  # # Test that Zebra syncs and checkpoints a few thousand blocks from an empty state
-  # test-empty-sync:
-  #   name: Test checkpoint sync from empty state
-  #   runs-on: ubuntu-latest
-  #   needs: build
-  #   if: ${{ github.event.inputs.regenerate-disks != 'true' }}
-  #   steps:
-  #     - uses: actions/checkout@v2.4.0
-  #       with:
-  #         persist-credentials: false
-
-  #     - name: Inject slug/short variables
-  #       uses: rlespinasse/github-slug-action@v4
-
-  #     - name: Run zebrad large sync tests
-  #       run: |
-  #         docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
-  #         docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance sync_large_checkpoints_ -- --ignored
-
-  # regenerate-stateful-disks:
-  #   name: Regenerate stateful disks
-  #   runs-on: ubuntu-latest
-  #   needs: build
-  #   outputs:
-  #     disk_short_sha: ${{ steps.disk-short-sha.outputs.disk_short_sha }}
-  #   steps:
-  #     - uses: actions/checkout@v2.4.0
-  #       with:
-  #         persist-credentials: false
-  #         fetch-depth: '2'
-
-  #     - name: Get specific changed files
-  #       id: changed-files-specific
-  #       uses: tj-actions/changed-files@v14.4
-  #       with:
-  #         files: |
-  #           /zebra-state/**/disk_format.rs
-  #           /zebra-state/**/disk_db.rs
-  #           /zebra-state/**/finalized_state.rs
-  #           /zebra-state/**/constants.rs
-
-  #     - name: Inject slug/short variables
-  #       uses: rlespinasse/github-slug-action@v4
-
-  #     - name: Downcase network name for disks
-  #       run: |
-  #         echo LOWER_NET_NAME="${{ github.event.inputs.network || env.NETWORK }}" | awk '{print tolower($0)}' >> $GITHUB_ENV
-
-  #     # Setup gcloud CLI
-  #     - name: Authenticate to Google Cloud
-  #       id: auth
-  #       uses: google-github-actions/auth@v0.5.0
-  #       with:
-  #         credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
-
-  #     - name: Create GCP compute instance
-  #       id: create-instance
-  #       if: ${{ steps.changed-files-specific.outputs.any_changed == 'true' || github.event.inputs.regenerate-disks == 'true' }}
-  #       run: |
-  #         gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" \
-  #         --boot-disk-size 100GB \
-  #         --boot-disk-type pd-ssd \
-  #         --create-disk name="zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy",size=100GB,type=pd-ssd \
-  #         --container-mount-disk mount-path='/zebrad-cache',name="zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy" \
-  #         --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
-  #         --container-restart-policy=never \
-  #         --container-stdin \
-  #         --container-tty \
-  #         --container-command="cargo" \
-  #         --container-arg="test" \
-  #         --container-arg="--locked" \
-  #         --container-arg="--release" \
-  #         --container-arg="--features" \
-  #         --container-arg="enable-sentry,test_sync_to_mandatory_checkpoint_${{ env.lower_net_name }}" \
-  #         --container-arg="--manifest-path" \
-  #         --container-arg="zebrad/Cargo.toml" \
-  #         --container-arg="sync_to_mandatory_checkpoint_${{ env.lower_net_name }}" \
-  #         --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
-  #         --machine-type ${{ env.MACHINE_TYPE }} \
-  #         --scopes cloud-platform \
-  #         --metadata=google-monitoring-enabled=true,google-logging-enabled=true \
-  #         --tags zebrad \
-  #         --zone "${{ env.ZONE }}"
-
-  #     # TODO: this approach is very mesy, but getting the just created container name is very error prone and GCP doesn't have a workaround for this without requiring a TTY
-  #     # This TODO relates to the following issues:
-  #     # https://github.com/actions/runner/issues/241
-  #     # https://www.googlecloudcommunity.com/gc/Infrastructure-Compute-Storage/SSH-into-Compute-Container-not-easily-possible/td-p/170915
-  #     - name: Get container name from logs
-  #       id: get-container-name
-  #       if: steps.create-instance.outcome == 'success'
-  #       run: |
-  #         INSTANCE_ID=$(gcloud compute instances describe zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --format='value(id)')
-  #         echo "Using instance: $INSTANCE_ID"
-  #         while [[ ${CONTAINER_NAME} != *"zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}"* ]]; do
-  #             CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
-  #             echo "Using container: ${CONTAINER_NAME} from instance: ${INSTANCE_ID}"
-  #             sleep 10
-  #         done
-  #         CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
-  #         echo "::set-output name=zebra_container::$CONTAINER_NAME"
-
-  #     - name: Regenerate stateful disks logs
-  #       id: sync-to-checkpoint
-  #       if: steps.create-instance.outcome == 'success'
-  #       run: |
-  #         gcloud compute ssh \
-  #         zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
-  #         --zone ${{ env.ZONE }} \
-  #         --quiet \
-  #         --ssh-flag="-o ServerAliveInterval=5" \
-  #         --command="docker logs --follow ${{ env.ZEBRA_CONTAINER }}"
-  #       env:
-  #         ZEBRA_CONTAINER: ${{ steps.get-container-name.outputs.zebra_container }}
-
-  #     # Create image from disk that will be used to sync past mandatory checkpoint test
-  #     # Force the image creation as the disk is still attached even though is not being used by the container
-  #     - name: Create image from state disk
-  #       # Only run if the earlier step succeeds
-  #       if: steps.sync-to-checkpoint.outcome == 'success'
-  #       run: |
-  #         gcloud compute images create zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy \
-  #         --force \
-  #         --source-disk=zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy \
-  #         --source-disk-zone=${{ env.ZONE }} \
-  #         --storage-location=us \
-  #         --description="Created from head branch ${{ env.GITHUB_HEAD_REF_SLUG_URL }} targeting ${{ env.GITHUB_BASE_REF_SLUG }} from PR ${{ env.GITHUB_REF_SLUG_URL }} with commit ${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA }}"
-
-  #     - name: Output and write the disk SHORT_SHA to a txt
-  #       id: disk-short-sha
-  #       if: steps.sync-to-checkpoint.outcome == 'success'
-  #       run: |
-  #         short_sha=$(echo "${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}")
-  #         echo "$short_sha" > latest-disk-state-sha.txt
-  #         echo "::set-output name=disk_short_sha::$short_sha"
-
-  #     - name: Upload the disk state txt
-  #       if: steps.sync-to-checkpoint.outcome == 'success'
-  #       uses: actions/upload-artifact@v2.3.1
-  #       with:
-  #         name: latest-disk-state-sha
-  #         path: latest-disk-state-sha.txt
-  #         retention-days: 1095
-
-  #     - name: Delete test instance
-  #       # Do not delete the instance if the sync timeouts in GitHub
-  #       if: ${{ steps.sync-to-checkpoint.outcome == 'success' }} || ${{ steps.sync-to-checkpoint.outcome == 'failure' }}
-  #       continue-on-error: true
-  #       run: |
-  #         gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
-
-  # # Test that Zebra syncs and fully validates a few thousand blocks from a cached post-checkpoint state
-  # test-stateful-sync:
-  #   name: Test full validation sync from cached state
-  #   runs-on: ubuntu-latest
-  #   needs: [ build, regenerate-stateful-disks]
-  #   steps:
-  #     - uses: actions/checkout@v2.4.0
-  #       with:
-  #         persist-credentials: false
-
-  #     - name: Inject slug/short variables
-  #       uses: rlespinasse/github-slug-action@v4
-
-  #     - name: Downcase network name for disks
-  #       run: |
-  #         echo LOWER_NET_NAME="${{ github.event.inputs.network || env.NETWORK }}" | awk '{print tolower($0)}' >> $GITHUB_ENV
-
-  #     # Get the latest uploaded txt with the disk SHORT_SHA from this workflow
-  #     - name: Download latest disk state SHORT_SHA
-  #       uses: dawidd6/action-download-artifact@v2.17.0
-  #       with:
-  #         github_token: ${{ secrets.GITHUB_TOKEN }}
-  #         workflow: test.yml
-  #         workflow_conclusion: ''
-  #         name: latest-disk-state-sha
-  #         check_artifacts:  true
-
-  #     - name: Get disk state SHA from txt
-  #       id: get-disk-sha
-  #       run: |
-  #         output=$(cat latest-disk-state-sha.txt)
-  #         echo "::set-output name=sha::$output"
-
-  #     # Setup gcloud CLI
-  #     - name: Authenticate to Google Cloud
-  #       id: auth
-  #       uses: google-github-actions/auth@v0.5.0
-  #       with:
-  #         credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
-
-  #     # Creates Compute Engine virtual machine instance w/ disks
-  #     - name: Create GCP compute instance
-  #       id: create-instance
-  #       run: |
-  #         gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" \
-  #         --boot-disk-size 100GB \
-  #         --boot-disk-type pd-ssd \
-  #         --create-disk=image=zebrad-cache-${{ env.DISK_SHORT_SHA }}-${{ env.lower_net_name }}-canopy,name=zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy,size=100GB,type=pd-ssd \
-  #         --container-mount-disk=mount-path='/zebrad-cache',name=zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy \
-  #         --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
-  #         --container-restart-policy=never \
-  #         --container-stdin \
-  #         --container-tty \
-  #         --container-command="cargo" \
-  #         --container-arg="test" \
-  #         --container-arg="--locked" \
-  #         --container-arg="--release" \
-  #         --container-arg="--features" \
-  #         --container-arg="enable-sentry,test_sync_past_mandatory_checkpoint_${{ env.lower_net_name }}" \
-  #         --container-arg="--manifest-path" \
-  #         --container-arg="zebrad/Cargo.toml" \
-  #         --container-arg="sync_past_mandatory_checkpoint_${{ env.lower_net_name }}" \
-  #         --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
-  #         --machine-type ${{ env.MACHINE_TYPE }} \
-  #         --scopes cloud-platform \
-  #         --metadata=google-monitoring-enabled=true,google-logging-enabled=true \
-  #         --tags zebrad \
-  #         --zone "${{ env.ZONE }}"
-  #       env:
-  #         DISK_SHORT_SHA: ${{ needs.regenerate-stateful-disks.outputs.disk_short_sha || steps.get-disk-sha.outputs.sha }}
-
-  #     # TODO: this approach is very mesy, but getting the just created container name is very error prone and GCP doesn't have a workaround for this without requiring a TTY
-  #     # This TODO relates to the following issues:
-  #     # https://github.com/actions/runner/issues/241
-  #     # https://www.googlecloudcommunity.com/gc/Infrastructure-Compute-Storage/SSH-into-Compute-Container-not-easily-possible/td-p/170915
-  #     - name: Get container name from logs
-  #       id: get-container-name
-  #       if: steps.create-instance.outcome == 'success'
-  #       run: |
-  #         INSTANCE_ID=$(gcloud compute instances describe zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --format='value(id)')
-  #         echo "Using instance: $INSTANCE_ID"
-  #         while [[ ${CONTAINER_NAME} != *"zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}"* ]]; do
-  #             CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
-  #             echo "Using container: ${CONTAINER_NAME} from instance: ${INSTANCE_ID}"
-  #             sleep 10
-  #         done
-  #         CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
-  #         echo "::set-output name=zebra_container::$CONTAINER_NAME"
-
-  #     - name: Sync past mandatory checkpoint logs
-  #       id: sync-past-checkpoint
-  #       run: |
-  #         gcloud compute ssh \
-  #         zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
-  #         --zone ${{ env.ZONE }} \
-  #         --quiet \
-  #         --ssh-flag="-o ServerAliveInterval=5" \
-  #         --command="docker logs --follow ${{ env.ZEBRA_CONTAINER }}"
-  #       env:
-  #         ZEBRA_CONTAINER: ${{ steps.get-container-name.outputs.zebra_container }}
-
-  #     - name: Delete test instance
-  #       # Do not delete the instance if the sync timeouts in GitHub
-  #       if: ${{ steps.sync-past-checkpoint.outcome == 'success' }} || ${{ steps.sync-past-checkpoint.outcome == 'failure' }}
-  #       continue-on-error: true
-  #       run: |
-  #         gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
-
-  # Test that Zebra can run a full mainnet sync
+  # Run all the zebra tests, including tests that are ignored by default
+  test-all:
+    name: Test all
+    runs-on: ubuntu-latest
+    needs: build
+    if: ${{ github.event.inputs.regenerate-disks != 'true' }}
+    steps:
+      - uses: actions/checkout@v2.4.0
+        with:
+          persist-credentials: false
+
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v4
+
+      - name: Run all zebrad tests
+        run: |
+          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --workspace -- --include-ignored
+
+  test-fake-activation-heights:
+    name: Test with fake activation heights
+    runs-on: ubuntu-latest
+    needs: build
+    if: ${{ github.event.inputs.regenerate-disks != 'true' }}
+    steps:
+      - uses: actions/checkout@v2.4.0
+        with:
+          persist-credentials: false
+
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v4
+
+      - name: Run tests with fake activation heights
+        run: |
+          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --package zebra-state --lib -- with_fake_activation_heights
+
+  # Test that Zebra syncs and checkpoints a few thousand blocks from an empty state
+  test-empty-sync:
+    name: Test checkpoint sync from empty state
+    runs-on: ubuntu-latest
+    needs: build
+    if: ${{ github.event.inputs.regenerate-disks != 'true' }}
+    steps:
+      - uses: actions/checkout@v2.4.0
+        with:
+          persist-credentials: false
+
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v4
+
+      - name: Run zebrad large sync tests
+        run: |
+          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance sync_large_checkpoints_ -- --ignored
+
+  regenerate-stateful-disks:
+    name: Regenerate stateful disks
+    runs-on: ubuntu-latest
+    needs: build
+    outputs:
+      disk_short_sha: ${{ steps.disk-short-sha.outputs.disk_short_sha }}
+    steps:
+      - uses: actions/checkout@v2.4.0
+        with:
+          persist-credentials: false
+          fetch-depth: '2'
+
+      - name: Get specific changed files
+        id: changed-files-specific
+        uses: tj-actions/changed-files@v14.4
+        with:
+          files: |
+            /zebra-state/**/disk_format.rs
+            /zebra-state/**/disk_db.rs
+            /zebra-state/**/finalized_state.rs
+            /zebra-state/**/constants.rs
+
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v4
+
+      - name: Downcase network name for disks
+        run: |
+          echo LOWER_NET_NAME="${{ github.event.inputs.network || env.NETWORK }}" | awk '{print tolower($0)}' >> $GITHUB_ENV
+
+      # Setup gcloud CLI
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@v0.5.0
+        with:
+          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
+
+      - name: Create GCP compute instance
+        id: create-instance
+        if: ${{ steps.changed-files-specific.outputs.any_changed == 'true' || github.event.inputs.regenerate-disks == 'true' }}
+        run: |
+          gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" \
+          --boot-disk-size 100GB \
+          --boot-disk-type pd-ssd \
+          --create-disk name="zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy",size=100GB,type=pd-ssd \
+          --container-mount-disk mount-path='/zebrad-cache',name="zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy" \
+          --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
+          --container-restart-policy=never \
+          --container-stdin \
+          --container-tty \
+          --container-command="cargo" \
+          --container-arg="test" \
+          --container-arg="--locked" \
+          --container-arg="--release" \
+          --container-arg="--features" \
+          --container-arg="enable-sentry,test_sync_to_mandatory_checkpoint_${{ env.lower_net_name }}" \
+          --container-arg="--manifest-path" \
+          --container-arg="zebrad/Cargo.toml" \
+          --container-arg="sync_to_mandatory_checkpoint_${{ env.lower_net_name }}" \
+          --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
+          --machine-type ${{ env.MACHINE_TYPE }} \
+          --scopes cloud-platform \
+          --metadata=google-monitoring-enabled=true,google-logging-enabled=true \
+          --tags zebrad \
+          --zone "${{ env.ZONE }}"
+
+      # TODO: this approach is very mesy, but getting the just created container name is very error prone and GCP doesn't have a workaround for this without requiring a TTY
+      # This TODO relates to the following issues:
+      # https://github.com/actions/runner/issues/241
+      # https://www.googlecloudcommunity.com/gc/Infrastructure-Compute-Storage/SSH-into-Compute-Container-not-easily-possible/td-p/170915
+      - name: Get container name from logs
+        id: get-container-name
+        if: steps.create-instance.outcome == 'success'
+        run: |
+          INSTANCE_ID=$(gcloud compute instances describe zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --format='value(id)')
+          echo "Using instance: $INSTANCE_ID"
+          while [[ ${CONTAINER_NAME} != *"zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}"* ]]; do
+              CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
+              echo "Using container: ${CONTAINER_NAME} from instance: ${INSTANCE_ID}"
+              sleep 10
+          done
+          CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
+          echo "::set-output name=zebra_container::$CONTAINER_NAME"
+
+      - name: Regenerate stateful disks logs
+        id: sync-to-checkpoint
+        if: steps.create-instance.outcome == 'success'
+        run: |
+          gcloud compute ssh \
+          zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
+          --zone ${{ env.ZONE }} \
+          --quiet \
+          --ssh-flag="-o ServerAliveInterval=5" \
+          --command="docker logs --follow ${{ env.ZEBRA_CONTAINER }}"
+        env:
+          ZEBRA_CONTAINER: ${{ steps.get-container-name.outputs.zebra_container }}
+
+      # Create image from disk that will be used to sync past mandatory checkpoint test
+      # Force the image creation as the disk is still attached even though is not being used by the container
+      - name: Create image from state disk
+        # Only run if the earlier step succeeds
+        if: steps.sync-to-checkpoint.outcome == 'success'
+        run: |
+          gcloud compute images create zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy \
+          --force \
+          --source-disk=zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy \
+          --source-disk-zone=${{ env.ZONE }} \
+          --storage-location=us \
+          --description="Created from head branch ${{ env.GITHUB_HEAD_REF_SLUG_URL }} targeting ${{ env.GITHUB_BASE_REF_SLUG }} from PR ${{ env.GITHUB_REF_SLUG_URL }} with commit ${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA }}"
+
+      - name: Output and write the disk SHORT_SHA to a txt
+        id: disk-short-sha
+        if: steps.sync-to-checkpoint.outcome == 'success'
+        run: |
+          short_sha=$(echo "${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}")
+          echo "$short_sha" > latest-disk-state-sha.txt
+          echo "::set-output name=disk_short_sha::$short_sha"
+
+      - name: Upload the disk state txt
+        if: steps.sync-to-checkpoint.outcome == 'success'
+        uses: actions/upload-artifact@v2.3.1
+        with:
+          name: latest-disk-state-sha
+          path: latest-disk-state-sha.txt
+          retention-days: 1095
+
+      - name: Delete test instance
+        # Do not delete the instance if the sync timeouts in GitHub
+        if: ${{ steps.sync-to-checkpoint.outcome == 'success' }} || ${{ steps.sync-to-checkpoint.outcome == 'failure' }}
+        continue-on-error: true
+        run: |
+          gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
+
+  # Test that Zebra syncs and fully validates a few thousand blocks from a cached post-checkpoint state
+  test-stateful-sync:
+    name: Test full validation sync from cached state
+    runs-on: ubuntu-latest
+    needs: [ build, regenerate-stateful-disks]
+    steps:
+      - uses: actions/checkout@v2.4.0
+        with:
+          persist-credentials: false
+
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v4
+
+      - name: Downcase network name for disks
+        run: |
+          echo LOWER_NET_NAME="${{ github.event.inputs.network || env.NETWORK }}" | awk '{print tolower($0)}' >> $GITHUB_ENV
+
+      # Get the latest uploaded txt with the disk SHORT_SHA from this workflow
+      - name: Download latest disk state SHORT_SHA
+        uses: dawidd6/action-download-artifact@v2.17.0
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          workflow: test.yml
+          workflow_conclusion: ''
+          name: latest-disk-state-sha
+          check_artifacts:  true
+
+      - name: Get disk state SHA from txt
+        id: get-disk-sha
+        run: |
+          output=$(cat latest-disk-state-sha.txt)
+          echo "::set-output name=sha::$output"
+
+      # Setup gcloud CLI
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@v0.5.0
+        with:
+          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
+
+      # Creates Compute Engine virtual machine instance w/ disks
+      - name: Create GCP compute instance
+        id: create-instance
+        run: |
+          gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" \
+          --boot-disk-size 100GB \
+          --boot-disk-type pd-ssd \
+          --create-disk=image=zebrad-cache-${{ env.DISK_SHORT_SHA }}-${{ env.lower_net_name }}-canopy,name=zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy,size=100GB,type=pd-ssd \
+          --container-mount-disk=mount-path='/zebrad-cache',name=zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy \
+          --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
+          --container-restart-policy=never \
+          --container-stdin \
+          --container-tty \
+          --container-command="cargo" \
+          --container-arg="test" \
+          --container-arg="--locked" \
+          --container-arg="--release" \
+          --container-arg="--features" \
+          --container-arg="enable-sentry,test_sync_past_mandatory_checkpoint_${{ env.lower_net_name }}" \
+          --container-arg="--manifest-path" \
+          --container-arg="zebrad/Cargo.toml" \
+          --container-arg="sync_past_mandatory_checkpoint_${{ env.lower_net_name }}" \
+          --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
+          --machine-type ${{ env.MACHINE_TYPE }} \
+          --scopes cloud-platform \
+          --metadata=google-monitoring-enabled=true,google-logging-enabled=true \
+          --tags zebrad \
+          --zone "${{ env.ZONE }}"
+        env:
+          DISK_SHORT_SHA: ${{ needs.regenerate-stateful-disks.outputs.disk_short_sha || steps.get-disk-sha.outputs.sha }}
+
+      # TODO: this approach is very mesy, but getting the just created container name is very error prone and GCP doesn't have a workaround for this without requiring a TTY
+      # This TODO relates to the following issues:
+      # https://github.com/actions/runner/issues/241
+      # https://www.googlecloudcommunity.com/gc/Infrastructure-Compute-Storage/SSH-into-Compute-Container-not-easily-possible/td-p/170915
+      - name: Get container name from logs
+        id: get-container-name
+        if: steps.create-instance.outcome == 'success'
+        run: |
+          INSTANCE_ID=$(gcloud compute instances describe zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --format='value(id)')
+          echo "Using instance: $INSTANCE_ID"
+          while [[ ${CONTAINER_NAME} != *"zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}"* ]]; do
+              CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
+              echo "Using container: ${CONTAINER_NAME} from instance: ${INSTANCE_ID}"
+              sleep 10
+          done
+          CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
+          echo "::set-output name=zebra_container::$CONTAINER_NAME"
+
+      - name: Sync past mandatory checkpoint logs
+        id: sync-past-checkpoint
+        run: |
+          gcloud compute ssh \
+          zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
+          --zone ${{ env.ZONE }} \
+          --quiet \
+          --ssh-flag="-o ServerAliveInterval=5" \
+          --command="docker logs --follow ${{ env.ZEBRA_CONTAINER }}"
+        env:
+          ZEBRA_CONTAINER: ${{ steps.get-container-name.outputs.zebra_container }}
+
+      - name: Delete test instance
+        # Do not delete the instance if the sync timeouts in GitHub
+        if: ${{ steps.sync-past-checkpoint.outcome == 'success' }} || ${{ steps.sync-past-checkpoint.outcome == 'failure' }}
+        continue-on-error: true
+        run: |
+          gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
+
+  # Test that Zebra can run a full mainnet sync after a PR is approved
   test-full-sync:
     name: Test full Mainnet sync
     runs-on: ubuntu-latest
     needs: [ build]
+    if: github.event.review.state == 'approved'
     steps:
       - uses: actions/checkout@v2.4.0
         with:

From 794d3fe207331473b6b17823788bbaca485b476c Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Mar 2022 06:49:23 -0400
Subject: [PATCH 24/24] fix(tests): reduce the changelog

---
 .github/workflows/test.yml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 770fc9364f1..af7c2e29da0 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -169,15 +169,17 @@ jobs:
           persist-credentials: false
           fetch-depth: '2'
 
+      # only run this job if the database format might have changed
       - name: Get specific changed files
         id: changed-files-specific
-        uses: tj-actions/changed-files@v14.4
+        uses: tj-actions/changed-files@v17.2
         with:
           files: |
+            /zebra-state/**/constants.rs
+            /zebra-state/**/finalized_state.rs
             /zebra-state/**/disk_format.rs
             /zebra-state/**/disk_db.rs
-            /zebra-state/**/finalized_state.rs
-            /zebra-state/**/constants.rs
+            /zebra-state/**/zebra_db.rs
 
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v4