feat: Bootstrap cache v2 (#3876)

Builds on top of @charlielye's #3866. Tested by doing `bootstrap_cache`
+ `l1-contracts/bootstrap` + `yarn-project/bootstrap` on an fresh clone.

- Extracts missing folders
- Removes global bootstrap_cache in favor of individual ones on each
root folder
- Manually builds the ts for bb.js (it's pretty fast, the main
bottleneck was getting the wasm)
- Adds missing foundation build step in yarn project bootstrap
- Fixes `query_manifest` to handle alternate version of yq that return
different strings for `type`
- Installs foundry only if needed

---------

Co-authored-by: Charlie Lye <karl.lye@gmail.com>

README.md

@@ -23,6 +23,8 @@ All issues being worked on are tracked on the [Aztec Github Project](https://git
 
 Run `bootstrap.sh` in the project root to set up your environment. This will update git submodules, download ignition transcripts, install Foundry, compile Solidity contracts, install the current node version via nvm, and build all typescript packages.
 
+Alternatively, to just hack on Noir contracts and Typescript, run `BOOTSTRAP_USE_REMOTE_CACHE=1 ./bootstrap.sh`, which will download existing builds for barretenberg and nargo from the CI cache. Note that this only works on Ubuntu.
+
 To build Typescript code, make sure to have [`nvm`](https://github.com/nvm-sh/nvm) (node version manager) installed.
 
 To build noir code, make sure that you are using the version from `yarn-project/noir-compiler/src/noir-version.json`.

barretenberg/bootstrap_cache.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -eu
+
+cd "$(dirname "$0")"
+source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
+
+echo -e "\033[1mRetrieving bb.wasm from remote cache...\033[0m"
+extract_repo bb.js \
+  /usr/src/barretenberg/cpp/build-wasm/bin ./cpp/build-wasm \
+  /usr/src/barretenberg/cpp/build-wasm-threads/bin ./cpp/build-wasm-threads
+
+echo -e "\033[1mBuilding ESM bb.ts...\033[0m"
+(cd ts && ./bootstrap.sh esm)

barretenberg/cpp/dockerfiles/Dockerfile.wasm-linux-clang

@@ -10,7 +10,6 @@ RUN ./scripts/strip-wasm.sh
 
 FROM scratch
 WORKDIR /usr/src/barretenberg/cpp
-COPY . .
 COPY --from=builder /usr/src/barretenberg/cpp/srs_db /usr/src/barretenberg/cpp/srs_db
 COPY --from=builder /usr/src/barretenberg/cpp/build-wasm/bin/barretenberg.wasm /usr/src/barretenberg/cpp/build-wasm/bin/barretenberg.wasm
 COPY --from=builder /usr/src/barretenberg/cpp/build-wasm-threads/bin/barretenberg.wasm /usr/src/barretenberg/cpp/build-wasm-threads/bin/barretenberg.wasm

barretenberg/ts/bootstrap.sh

@@ -4,19 +4,24 @@ set -eu
 cd "$(dirname "$0")"
 
 CMD=${1:-}
+BUILD_CMD="build"
 
 if [ -n "$CMD" ]; then
   if [ "$CMD" = "clean" ]; then
     git clean -fdx
     exit 0
+  elif [ "$CMD" = "esm" ]; then
+    BUILD_CMD="build:esm"
   else
     echo "Unknown command: $CMD"
     exit 1
   fi
 fi
 
 yarn install --immutable
-yarn build
+echo "Building with command 'yarn $BUILD_CMD'..."
+yarn $BUILD_CMD
 
 # Make bin globally available.
 npm link
+echo "Barretenberg ts build successful"

bootstrap.sh

@@ -57,12 +57,21 @@ PROJECTS=(
   yarn-project
 )
 
+# Build projects locally
 for P in "${PROJECTS[@]}"; do
-  echo "**************************************"
-  echo -e "\033[1mBootstrapping $P...\033[0m"
-  echo "**************************************"
-  echo
-  $P/bootstrap.sh
+  if [ -n "${BOOTSTRAP_USE_REMOTE_CACHE:-}" ] && [ -f "$P/bootstrap_cache.sh" ]; then
+    echo "**************************************"
+    echo -e "\033[1mBootstrapping $P from remote cache...\033[0m"
+    echo "**************************************"
+    echo
+    $P/bootstrap_cache.sh
+  else  
+    echo "**************************************"
+    echo -e "\033[1mBootstrapping $P...\033[0m"
+    echo "**************************************"
+    echo
+    $P/bootstrap.sh
+  fi
   echo
   echo
 done

build-system/scripts/extract_repo

@@ -1,22 +1,36 @@
 #!/usr/bin/env bash
-# Given a repository, extracts the builds entire /usr/src dir to the given path.
+# Given a repository, extracts the builds entire /usr/src dir to the current path.
+# Can be given any number of pairs to extract the first path from the repo to the second path locally.
 [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 REPOSITORY=$1
-EXTRACT_FROM=${2:-/usr/src}
-EXTRACT_TO=${3:-./}
-IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY)
+shift
 
+IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY)
 echo "Pulling $IMAGE_COMMIT_URI..."
 ecr_login
 retry docker pull $IMAGE_COMMIT_URI
-TEMP_CONTAINER=$(docker create $IMAGE_COMMIT_URI)
+TEMP_CONTAINER=$(docker create $IMAGE_COMMIT_URI dummy_cmd)
 
-echo "Extracting $EXTRACT_FROM from $REPOSITORY to $EXTRACT_TO..."
-mkdir -p $EXTRACT_TO
-docker cp $TEMP_CONTAINER:$EXTRACT_FROM $EXTRACT_TO
-docker rm -v $TEMP_CONTAINER >/dev/null
 
-echo "Extracted contents:"
-ls -al $EXTRACT_TO
+function extract_from_temp_container {
+  EXTRACT_FROM=$1
+  EXTRACT_TO=$2
+  echo "Extracting $EXTRACT_FROM from $REPOSITORY to $EXTRACT_TO..."
+  mkdir -p $EXTRACT_TO
+  docker cp $TEMP_CONTAINER:$EXTRACT_FROM $EXTRACT_TO
+}
+
+# Default to extracting the entire /usr/src dir
+if [ $# -eq 0 ]; then 
+  extract_from_temp_container /usr/src ./
+else 
+  while [ $# -gt 0 ]; do
+    extract_from_temp_container $1 $2
+    shift 2
+  done
+fi
+
+echo "Cleaning up temp container"
+docker rm -v $TEMP_CONTAINER >/dev/null

build-system/scripts/query_manifest

@@ -29,7 +29,7 @@ function get_deps {
   elif [ "$TYPE" == "!!seq" ]; then
     DEPS=($(yq -r ".\"$1\".dependencies // [] | .[]" $MANIFEST))
   else
-    >&2 echo "dependencies must be a array, string or null."
+    >&2 echo "Build manifest 'dependencies' must be array, string, or null (got $TYPE)."
     exit 1
   fi
 }
@@ -46,7 +46,7 @@ function add_rebuild_patterns {
     local PROJECT_DIR=$($0 relativeProjectDir $1)
     PATTERNS=(${PATTERNS[@]} "^$PROJECT_DIR/")
   else
-    >&2 echo "rebuildPatterns must be array, string, or null."
+    >&2 echo "Build manifest 'rebuildPatterns' must be array, string, or null (got $TYPE)."
     exit 1
   fi
 }

build-system/scripts/setup_env

@@ -6,6 +6,7 @@
 # The script should be sourced from the root of the repository, e.g:
 #   source ./build-system/scripts/setup_env
 # This ensures the resultant variables are set in the calling shell.
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 COMMIT_HASH=$1

noir/bootstrap_cache.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -eu
+
+cd "$(dirname "$0")"
+source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
+
+echo -e "\033[1mRetrieving noir packages from remote cache...\033[0m"
+extract_repo noir-packages /usr/src/noir/packages ./noir
+echo -e "\033[1mRetrieving nargo from remote cache...\033[0m"
+extract_repo noir /usr/src/noir/target/release ./noir/target

yarn-project/bootstrap.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 # Check node version.
@@ -28,17 +29,28 @@ fi
 
 yarn install --immutable
 
+echo -e "\033[1mGenerating constants files...\033[0m"
+# Required to run remake-constants.
+yarn workspace @aztec/foundation build
 # Run remake constants before building Aztec.nr contracts or l1 contracts as they depend on files created by it.
 yarn workspace @aztec/circuits.js remake-constants
+
+echo -e "\033[1mSetting up compiler and building contracts...\033[0m"
 # This is actually our code generation tool. Needed to build contract typescript wrappers.
+echo "Building noir compiler..."
 yarn workspace @aztec/noir-compiler build
 # Builds noir contracts (TODO: move this stage pre yarn-project). Generates typescript wrappers.
+echo "Building contracts from noir-contracts..."
 yarn workspace @aztec/noir-contracts build:contracts
+# Bundle compiled account contracts into accounts package
+echo "Copying account contracts..."
 yarn workspace @aztec/accounts build:copy-contracts
 # Build protocol circuits. TODO: move pre yarn-project.
+echo "Building contracts from noir-protocol-circuits..."
 yarn workspace @aztec/noir-protocol-circuits build
 
+echo -e "\033[1mBuilding all packages...\033[0m"
 yarn build
 
 echo
-echo "Success! You can now e.g. run anvil and end-to-end tests"
+echo "Yarn project successfully built."