chore: e2e on spots [ci rebuild]

build-system/scripts/build

@@ -105,7 +105,7 @@ for PARENT_REPO in $(query_manifest dependencies $REPOSITORY); do
   fetch_image $PARENT_IMAGE_URI
   # Tag it to look like an official release as that's what we use in Dockerfiles.
   TAG=$ECR_DEPLOY_URL/$PARENT_REPO
-  retry docker tag $PARENT_IMAGE_URI $TAG
+  docker tag $PARENT_IMAGE_URI $TAG
 done
 
 COMMIT_TAG_VERSION=$(extract_tag_version $REPOSITORY false)

build-system/scripts/cond_run_compose

@@ -0,0 +1,35 @@
+#!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
+set -eu
+
+REPOSITORY=$1
+COMPOSE_FILE=$2
+shift 2
+
+CONTENT_HASH=$(calculate_content_hash $REPOSITORY)
+BASE_TAG=cache-$CONTENT_HASH
+SUCCESS_TAG=$BASE_TAG-$JOB_NAME
+
+echo "Content hash: $CONTENT_HASH"
+
+if ! check_rebuild $SUCCESS_TAG $REPOSITORY; then
+  # Login to pull our ecr images with docker.
+  ecr_login
+
+  # For each dependency and self, pull in the latest image and give it correct tag.
+  for REPO in $(query_manifest runDependencies $REPOSITORY $JOB_NAME) $REPOSITORY; do
+    echo "Pulling $REPO..."
+    REPO_IMAGE_URI=$(calculate_image_uri $REPO)
+    retry docker pull $REPO_IMAGE_URI
+    docker tag $REPO_IMAGE_URI aztecprotocol/$REPO
+  done
+
+  cd $(query_manifest projectDir $REPOSITORY)
+
+  export $@
+  docker-compose -f $COMPOSE_FILE run $REPOSITORY
+
+  upload_logs_to_s3 log
+
+  retry tag_remote_image $REPOSITORY $BASE_TAG $SUCCESS_TAG
+fi

build-system/scripts/cond_spot_run_compose

@@ -0,0 +1,10 @@
+#!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
+set -eu
+
+REPOSITORY=$1
+CPUS=$2
+shift 2
+
+export TAG_POSTFIX=$JOB_NAME
+cond_spot_run_script $REPOSITORY $CPUS cond_run_compose $REPOSITORY $@

build-system/scripts/query_manifest

@@ -75,6 +75,12 @@ case "$CMD" in
   dependencies)
     yq -r ".\"$REPO\".dependencies // [] | .[]" $MANIFEST
     ;;
+  runDependencies)
+    echo $REPO
+    RUN_NAME=$3
+    yq -r ".\"$REPO\".runDependencies // [] | .[]" $MANIFEST
+    yq -r ".\"$REPO\".run.\"$RUN_NAME\".dependencies // [] | .[]" $MANIFEST
+    ;;
   allDependencies)
     # Get dependencies for a given repo. Inclusive of repo itself.
     # If a string, attempt to execute <projectDir>/<string> if exists to get dependencies, else error.
@@ -94,8 +100,6 @@ case "$CMD" in
       ALL_DEPS+=("$1")
     }
     add_deps $REPO
-    # Remove ourself as a dependency.
-    # unset ALL_DEPS[-1]
     for DEP in "${ALL_DEPS[@]}"; do
       echo $DEP
     done

build-system/scripts/remote_run_script

@@ -24,6 +24,8 @@ ssh -A -F $SSH_CONFIG_PATH $IP "
   export COMMIT_TAG=$COMMIT_TAG
   export JOB_NAME=$JOB_NAME
   export GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL
+  export BRANCH=${BRANCH:-}
+  export PULL_REQUEST=${PULL_REQUEST:-}
   export DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD
   export ECR_DEPLOY_URL=$ECR_DEPLOY_URL
   export ECR_URL=$ECR_URL

build-system/scripts/remote_runner

@@ -22,6 +22,6 @@ echo "Git checkout completed."
 
 BASH_ENV=/tmp/bash_env
 echo "Calling setup env..."
-source ./build-system/scripts/setup_env "$COMMIT_HASH" "$COMMIT_TAG" "$JOB_NAME" "$GIT_REPOSITORY_URL"
+source ./build-system/scripts/setup_env "$COMMIT_HASH" "$COMMIT_TAG" "$JOB_NAME" "$GIT_REPOSITORY_URL" "$BRANCH" "$PULL_REQUEST"
 echo "Calling $@..."
 $@

build-system/scripts/setup_env

@@ -13,7 +13,7 @@ COMMIT_TAG=${2##*aztec-packages-}
 JOB_NAME=$3
 GIT_REPOSITORY_URL=${4:-}
 BRANCH=${5:-}
-COMMIT_MESSAGE=${6:-}
+PULL_REQUEST=${6:-}
 
 BASH_ENV=${BASH_ENV:-}
 BUILD_SYSTEM_PATH=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
@@ -87,6 +87,7 @@ echo export GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL >> $BASH_ENV
 echo export VERSION_TAG=$VERSION_TAG >> $BASH_ENV
 echo export DEPLOY_TAG=$DEPLOY_TAG >> $BASH_ENV
 echo export BRANCH=$BRANCH >> $BASH_ENV
+echo export PULL_REQUEST=$PULL_REQUEST >> $BASH_ENV
 
 # Only run the following if we're the result of a commit (i.e. not being run manually).
 if [ -n "$COMMIT_HASH" ]; then

yarn-project/end-to-end/scripts/upload_logs_to_s3.sh → build-system/scripts/upload_logs_to_s3

@@ -8,8 +8,8 @@
 [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
+LOG_FOLDER=$1
 BUCKET_NAME="aztec-ci-artifacts"
-LOG_FOLDER="${LOG_FOLDER:-log}"
 COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}"
 
 if [ ! -d "$LOG_FOLDER" ] || [ -z "$(ls -A "$LOG_FOLDER")" ]; then
@@ -18,10 +18,10 @@ if [ ! -d "$LOG_FOLDER" ] || [ -z "$(ls -A "$LOG_FOLDER")" ]; then
 fi
 
 # Duplicated in scripts/ci/assemble_e2e_benchmark.sh
-if [ "${CIRCLE_BRANCH:-}" = "master" ]; then
+if [ "${BRANCH:-}" = "master" ]; then
   TARGET_FOLDER="logs-v1/master/$COMMIT_HASH/"
-elif [ -n "${CIRCLE_PULL_REQUEST:-}" ]; then
-  TARGET_FOLDER="logs-v1/pulls/${CIRCLE_PULL_REQUEST##*/}"
+elif [ -n "${PULL_REQUEST:-}" ]; then
+  TARGET_FOLDER="logs-v1/pulls/${PULL_REQUEST##*/}"
 fi
 
 if [ -n "${TARGET_FOLDER:-}" ]; then

build_manifest.yml

@@ -166,6 +166,8 @@ end-to-end:
   projectDir: yarn-project/end-to-end
   dependencies:
     - yarn-project
+  runDependencies:
+    - aztec-sandbox
 
 aztec-node:
   buildDir: yarn-project

cspell.json

@@ -27,6 +27,8 @@
     "camelcase",
     "cbind",
     "cbinds",
+    "cimg",
+    "cpus",
     "chainsafe",
     "cheatcode",
     "cheatcodes",
@@ -37,6 +39,7 @@
     "comlink",
     "composability",
     "concat",
+    "cond",
     "customizability",
     "danlee",
     "Daos",
@@ -73,6 +76,7 @@
     "Kademlia",
     "keccak",
     "keypairs",
+    "keyscan",
     "leveldb",
     "leveldown",
     "leveljs",
@@ -132,6 +136,7 @@
     "sload",
     "snakecase",
     "solhint",
+    "stdlib",
     "struct",
     "structs",
     "subrepo",
@@ -178,7 +183,5 @@
     "*.snap",
     "package.json"
   ],
-  "flagWords": [
-    "anonymous"
-  ]
+  "flagWords": ["anonymous"]
 }

yarn-project/end-to-end/scripts/docker-compose-browser.yml

@@ -30,11 +30,14 @@ services:
   end-to-end:
     image: aztecprotocol/end-to-end:latest
     environment:
-      BENCHMARK: true
+      BENCHMARK: 'true'
       DEBUG: ${DEBUG:-'aztec:*'}
       ETHEREUM_HOST: http://fork:8545
       CHAIN_ID: 31337
       PXE_URL: http://sandbox:8080
     entrypoint: ['./scripts/start_e2e_ci_browser.sh', './src/e2e_aztec_js_browser.test.ts']
     volumes:
       - ../log:/usr/src/yarn-project/end-to-end/log:rw
+    depends_on:
+      - sandbox
+      - fork

yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml

@@ -15,7 +15,7 @@ services:
   end-to-end:
     image: aztecprotocol/end-to-end:latest
     environment:
-      BENCHMARK: true
+      BENCHMARK: 'true'
       DEBUG: ${DEBUG:-'aztec:*'}
       ETHEREUM_HOST: http://fork:8545
       CHAIN_ID: 31337
@@ -28,3 +28,5 @@ services:
     command: ${TEST:-./src/e2e_deploy_contract.test.ts}
     volumes:
       - ../log:/usr/src/yarn-project/end-to-end/log:rw
+    depends_on:
+      - fork

yarn-project/end-to-end/scripts/docker-compose.yml

@@ -30,7 +30,7 @@ services:
   end-to-end:
     image: aztecprotocol/end-to-end:latest
     environment:
-      BENCHMARK: true
+      BENCHMARK: 'true'
       LOG_LEVL: 'debug'
       DEBUG: ${DEBUG:-'aztec:*'}
       ETHEREUM_HOST: http://fork:8545
@@ -39,3 +39,6 @@ services:
     command: ${TEST:-./src/e2e_deploy_contract.test.ts}
     volumes:
       - ../log:/usr/src/yarn-project/end-to-end/log:rw
+    depends_on:
+      - sandbox
+      - fork