From 58e40c9125e6d7b30abf7a4cbb170bbfc15e2037 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Sun, 5 May 2024 15:05:47 +0300 Subject: [PATCH] feat: run noir-packages-test in Earthly (#6174) --- .circleci/config.yml | 19 ------ .github/workflows/ci.yml | 18 +++++- barretenberg/ts/Earthfile | 2 +- noir/.earthlyignore | 87 +++++++++++++++++++++++++++ noir/Earthfile | 121 +++++++++++++++++++++++++++++++++----- 5 files changed, 211 insertions(+), 36 deletions(-) create mode 100644 noir/.earthlyignore diff --git a/.circleci/config.yml b/.circleci/config.yml index 374aff1dd79..73655966bf3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -254,18 +254,6 @@ jobs: command: cond_spot_run_build noir-packages 32 aztec_manifest_key: noir-packages - noir-packages-tests: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small - steps: - - *checkout - - *setup_env - - run: - name: "Build" - command: cond_spot_run_build noir-packages-tests 32 - aztec_manifest_key: noir-packages-tests - avm-transpiler: docker: - image: aztecprotocol/alpine-build-image @@ -521,12 +509,6 @@ workflows: requires: - bb-js <<: *defaults - - noir-packages-tests: - requires: - - bb-js - - noir-ecr-manifest - - noir-packages - <<: *defaults # Transpiler - avm-transpiler: *defaults @@ -588,7 +570,6 @@ workflows: - barretenberg-x86_64-linux-clang-fuzzing - barretenberg-wasm-linux-clang - barretenberg-docs - - noir-packages-tests - e2e-join - aztec-builder <<: *defaults diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f6189ce0b1..63ba605aca4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,6 +145,21 @@ jobs: timeout-minutes: 25 run: earthly-ci --no-output ./+test + noir-packages-test: + needs: setup + runs-on: ${{ inputs.username || github.actor }}-x86 + steps: + - { + uses: actions/checkout@v4, + with: { ref: "${{ github.event.pull_request.head.sha }}" }, + } + - uses: ./.github/ci-setup-action + with: + dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + concurrency_key: noir-${{ inputs.username || github.actor }}-x86 + - name: "Test Noir JS packages" + run: earthly-ci --no-output ./noir+packages-test + noir-projects: needs: setup runs-on: ${{ inputs.username || github.actor }}-x86 @@ -348,7 +363,7 @@ jobs: working-directory: ./barretenberg/cpp/ timeout-minutes: 15 run: earthly-ci --no-output +bench-ultra-honk --bench_mode=cache - + protocol-circuits-gates-report: needs: setup runs-on: ${{ inputs.username || github.actor }}-x86 @@ -393,6 +408,7 @@ jobs: - yarn-project-formatting - yarn-project-test - prover-client-test + - noir-packages-test if: always() steps: - run: | diff --git a/barretenberg/ts/Earthfile b/barretenberg/ts/Earthfile index 981dda905d7..3cebf04974c 100644 --- a/barretenberg/ts/Earthfile +++ b/barretenberg/ts/Earthfile @@ -16,7 +16,7 @@ COPY --dir src *.json *.js *.cjs . COPY ../cpp/+preset-wasm-threads/bin/barretenberg.wasm src/barretenberg_wasm/barretenberg-threads.wasm COPY ../cpp/+preset-wasm/bin/barretenberg.wasm src/barretenberg_wasm/barretenberg.wasm COPY ../cpp/+preset-wasm-threads/bin/barretenberg.wasm dest/node/barretenberg_wasm/barretenberg-threads.wasm -COPY ../cpp/+preset-wasm/bin/barretenberg.wasm dest/node-cjs/barretenberg_wasm/barretenberg-threads.wasm +COPY ../cpp/+preset-wasm-threads/bin/barretenberg.wasm dest/node-cjs/barretenberg_wasm/barretenberg-threads.wasm esm: RUN yarn build:esm diff --git a/noir/.earthlyignore b/noir/.earthlyignore new file mode 100644 index 00000000000..a90b738fcb2 --- /dev/null +++ b/noir/.earthlyignore @@ -0,0 +1,87 @@ +# Must include the .gitignore for all child projects as this is used by Earthly +# Note due to how we use Eartlhy each .gitignore MUST accompany any earthfile that might actually copy these artifacts +**/Earthfile +**/Readme.md +**/Dockerfile* +**/docker-compose*.yml + +# root .gitignore contents +dest +node_modules +.cache +scripts/.earthly +.pnp.cjs +.pnp.loader.mjs +build/ +.idea +cmake-build-debug +.terraform* +.bootstrapped +.tsbuildinfo + +# Local Netlify folder +.netlify + +.graphite* +.DS_Store + +**/*.dockerignore + +# Earthly +.arg +.secret + +# ./ .gitignore contents: +**/package.tgz +packages + +# ./noir-repo/ .gitignore contents: +/target +.DS_Store +examples/**/target/ +examples/9 +node_modules +pkg/ +.idea + +# Yarn +.pnp.* +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/sdks +!.yarn/versions + +# Noir.js +tooling/noir_js/lib + +# Nargo output +*.proof +*.acir +*.acir.sha256 +*.tr +*.pk +*.vk +**/Verifier.toml +**/contract +**/target +!test_programs/acir_artifacts/*/target +!test_programs/acir_artifacts/*/target/witness.gz +!compiler/wasm/noir-script/target + +gates_report.json + +# Github Actions scratch space +# This gives a location to download artifacts into the repository in CI without making git dirty. +libbarretenberg-wasm32 + +# Wasm build atifacts +compiler/wasm/nodejs +compiler/wasm/web +tooling/noirc_abi_wasm/nodejs +tooling/noirc_abi_wasm/web +tooling/noir_js/lib + +# docs autogen build +/docs/docs/noir_js/reference/ diff --git a/noir/Earthfile b/noir/Earthfile index 8a4db6aee43..82b19984d49 100644 --- a/noir/Earthfile +++ b/noir/Earthfile @@ -1,6 +1,7 @@ VERSION 0.8 -nargo: +nargo-src: + FROM ../build-images/+build WORKDIR /usr/src # Relevant source (TODO finer-grained 'tooling') @@ -14,26 +15,58 @@ nargo: noir-repo/utils \ noir-repo/Cargo.lock \ noir-repo/Cargo.toml \ + noir-repo/.github \ noir-repo - # TODO(AD) is this OK as a content hash? + # NOTE: we use a fake commit hash here + # we don't want Noir to rebuild everytime the parent repo changes + # just only when it changes + # the commit hash gets injected into version strings ENV COMMIT_HASH=$(find . -type f -exec sha256sum {} ';' | sort | sha256sum | awk '{print $1}') + RUN echo $COMMIT_HASH > .content-hash + + # # borrow Nix's approach to build everything in 1970 + ENV SOURCE_TIMESTAMP=1 + ENV SOURCE_DATE_EPOCH=1 + +nargo: + FROM +nargo-src + RUN ./noir-repo/.github/scripts/wasm-bindgen-install.sh COPY ./scripts/bootstrap_native.sh ./scripts/bootstrap_native.sh RUN ./scripts/bootstrap_native.sh - RUN echo CONTENT HASH $COMMIT_HASH | tee .content-hash SAVE ARTIFACT /usr/src/noir-repo/target/release/nargo nargo SAVE ARTIFACT /usr/src/noir-repo/target/release/acvm acvm SAVE IMAGE aztecprotocol/nargo -packages: +packages-deps: BUILD ../barretenberg/ts/+build # prefetch - FROM ../build-images/+build - # `noir-repo` is nested inside of `noir` so we copy `bb.js` as such to account - # for the extra nested folder specified in portalled package paths - COPY ../barretenberg/ts/+build/build /usr/src/../barretenberg/ts + LOCALLY + LET packages = $(git ls-files "**/package*.json" package*.json) + LET tsconfigs = $(git ls-files "**/tsconfig*.json" tsconfig*.json) - WORKDIR /usr/src + FROM +nargo-src + + COPY ../barretenberg/ts/+build/build /usr/src/barretenberg/ts + + WORKDIR /usr/src/noir + + COPY --dir \ + ./noir-repo/package.json \ + ./noir-repo/yarn.lock \ + ./noir-repo/.yarnrc.yml \ + ./noir-repo/.yarn \ + ./noir-repo + + FOR file IN $packages + COPY $file $file + END + + RUN cd noir-repo && yarn install --immutable && cd ../ + + FOR file IN $tsconfigs + COPY $file $file + END # Relevant source (TODO finer-grained) COPY --dir \ @@ -45,7 +78,7 @@ packages: noir-repo/scripts \ noir-repo/test_programs \ noir-repo/tooling \ - noir-repo/utils \ + noir-repo/utils \ noir-repo/Cargo.lock \ noir-repo/.yarnrc.yml \ noir-repo/.yarn \ @@ -58,16 +91,74 @@ packages: noir-repo/.github \ noir-repo - COPY noir-repo/.github/scripts noir-repo/.github/scripts - COPY ./scripts/bootstrap_packages.sh ./scripts/bootstrap_packages.sh +packages: + FROM +packages-deps - # TODO(AD) is this OK as a content hash? - ENV COMMIT_HASH=$(find . -type f -exec sha256sum {} ';' | sort | sha256sum | awk '{print $1}') - RUN echo CONTENT HASH $COMMIT_HASH | tee .content-hash + COPY ./scripts/bootstrap_packages.sh ./scripts/bootstrap_packages.sh RUN ./scripts/bootstrap_packages.sh SAVE ARTIFACT packages SAVE IMAGE --cache-hint +packages-test-build: + FROM +packages-deps + + COPY +nargo/nargo /usr/src/noir/noir-repo/target/release/nargo + COPY +nargo/acvm /usr/src/noir/noir-repo/target/release/acvm + + ENV NARGO_BACKEND_PATH=/usr/src/barretenberg/ts/dest/node/main.js + ENV PATH=$PATH:/usr/src/noir/noir-repo/target/release + + WORKDIR /usr/src/barretenberg/ts + RUN yarn --immutable + + WORKDIR /usr/src/noir/noir-repo + COPY --dir noir-repo/.github/scripts/wasm-bindgen-install.sh ./.github/scripts/wasm-bindgen-install.sh + RUN ./.github/scripts/wasm-bindgen-install.sh + + ENV SOURCE_DATE_EPOCH=$(date +%s) + ENV GIT_DIRTY=false + ENV GIT_COMMIT=$COMMIT_HASH + RUN yarn build + # this builds text fixtures to be used in tests + RUN yarn workspace @noir-lang/noir_wasm run test:build_fixtures + + SAVE ARTIFACT /usr/src /usr/src + +packages-test-node: + FROM +packages-test-build + ENV NODE_OPTIONS=--max_old_space_size=8192 + WORKDIR /usr/src/noir/noir-repo + RUN yarn workspaces foreach \ + --parallel \ + --verbose \ + --exclude @noir-lang/root \ # foreach includes the root workspace, ignore it + --exclude @noir-lang/noir_js \ # noir_js OOMs + --exclude integration-tests \ # separate node and browser tests + --exclude @noir-lang/noir_wasm \ + run test + RUN yarn workspaces foreach \ + --parallel \ + --verbose \ + --include integration-tests \ + --include @noir-lang/noir_wasm \ + run test:node + +packages-test-browser: + FROM node:18 + COPY --dir +packages-test-build/usr/src /usr + WORKDIR /usr/src/noir/noir-repo + RUN ./.github/scripts/playwright-install.sh + RUN yarn workspaces foreach \ + --parallel \ + --verbose \ + --include integration-tests \ + --include @noir-lang/noir_wasm \ + run test:browser + +packages-test: + BUILD +packages-test-node + BUILD +packages-test-browser + run: # When running the container, mount the users home directory to same location. FROM ubuntu:noble