Merge branch 'master' into gd/noir_issue_4330

AztecProtocol · May 17, 2024 · b1bd2cb · b1bd2cb
2 parents fcd3480 + 0ac83dc
commit b1bd2cb
Show file tree

Hide file tree

Showing 371 changed files with 6,807 additions and 6,579 deletions.
diff --git a/.github/spot-runner-action/dist/index.js b/.github/spot-runner-action/dist/index.js
@@ -127,6 +127,9 @@ class Ec2Instance {
         this.client = new aws_sdk_1.default.EC2({
             credentials: this.credentials,
             region: this.config.awsRegion,
+            maxRetries: 3,
+            // base 10 seconds for the exponential backoff, up to 3 times
+            retryDelayOptions: { base: 10000 },
         });
         this.tags = this.getTags();
     }
@@ -138,6 +141,9 @@ class Ec2Instance {
                 this.client = new aws_sdk_1.default.EC2({
                     credentials: credentials,
                     region: this.config.awsRegion,
+                    maxRetries: 3,
+                    // base 10 seconds for the exponential backoff, up to 3 times
+                    retryDelayOptions: { base: 10000 }
                 });
             }
             return this.client;
@@ -748,8 +754,10 @@ function requestAndWaitForSpot(config) {
                 // wait 10 seconds
                 yield new Promise((r) => setTimeout(r, 5000 * Math.pow(2, backoff)));
                 backoff += 1;
-                core.info("Polling to see if we somehow have an instance up");
-                instanceId = yield ((_a = ec2Client.getInstancesForTags("running")[0]) === null || _a === void 0 ? void 0 : _a.instanceId);
+                if (config.githubActionRunnerConcurrency > 0) {
+                    core.info("Polling to see if we somehow have an instance up");
+                    instanceId = yield ((_a = ec2Client.getInstancesForTags("running")[0]) === null || _a === void 0 ? void 0 : _a.instanceId);
+                }
             }
             if (instanceId) {
                 core.info("Successfully requested/found instance with ID " + instanceId);

diff --git a/.github/spot-runner-action/src/ec2.ts b/.github/spot-runner-action/src/ec2.ts
@@ -39,6 +39,9 @@ export class Ec2Instance {
     this.client = new AWS.EC2({
       credentials: this.credentials,
       region: this.config.awsRegion,
+      maxRetries: 3,
+      // base 10 seconds for the exponential backoff, up to 3 times
+      retryDelayOptions: { base: 10000 },
     });
 
     this.tags = this.getTags();
@@ -51,6 +54,9 @@ export class Ec2Instance {
       this.client = new AWS.EC2({
         credentials: credentials,
         region: this.config.awsRegion,
+        maxRetries: 3,
+        // base 10 seconds for the exponential backoff, up to 3 times
+        retryDelayOptions: {base: 10000}
       });
     }
     return this.client;

diff --git a/.github/spot-runner-action/src/main.ts b/.github/spot-runner-action/src/main.ts
@@ -88,8 +88,10 @@ async function requestAndWaitForSpot(config: ActionConfig): Promise<string> {
       // wait 10 seconds
       await new Promise((r) => setTimeout(r, 5000 * 2 ** backoff));
       backoff += 1;
-      core.info("Polling to see if we somehow have an instance up");
-      instanceId = await ec2Client.getInstancesForTags("running")[0]?.instanceId;
+      if (config.githubActionRunnerConcurrency > 0) {
+        core.info("Polling to see if we somehow have an instance up");
+        instanceId = await ec2Client.getInstancesForTags("running")[0]?.instanceId;
+      }
     }
     if (instanceId) {
       core.info("Successfully requested/found instance with ID " + instanceId);

diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml
@@ -1,7 +1,7 @@
 name: CI (ARM)
 on:
   push:
-    branches: [disabled] # [master]
+    branches: [master]
   workflow_dispatch:
     inputs: {}
 concurrency:
@@ -20,14 +20,10 @@ jobs:
     needs: setup
     runs-on: master-arm
     steps:
-      - {
-          uses: actions/checkout@v4,
-          with: { ref: "${{ github.event.pull_request.head.sha }}" },
-        }
+      - uses: actions/checkout@v4
+        with: { ref: "${{ github.event.pull_request.head.sha }}" }
       - uses: ./.github/ci-setup-action
         with:
-          dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
-          # must be globally unique for build x runner
           concurrency_key: build-master-arm
       # prepare images locally, tagged by commit hash
       - name: "Build E2E Image"
@@ -39,13 +35,10 @@ jobs:
     needs: build
     runs-on: master-arm
     steps:
-      - {
-          uses: actions/checkout@v4,
-          with: { ref: "${{ github.event.pull_request.head.sha }}" },
-        }
+      - uses: actions/checkout@v4
+        with: { ref: "${{ github.event.pull_request.head.sha }}" }
       - uses: ./.github/ci-setup-action
         with:
-          dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
           concurrency_key: e2e-master-arm-e2e-tests
       - name: Test
         working-directory: ./yarn-project/end-to-end/

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -32,6 +32,39 @@ jobs:
       username: ${{ github.event.pull_request.user.login || github.actor }}
       runner_type: builder-x86
     secrets: inherit
+  # job just to alert that you *might* hit missing CI files, but if not you can continue
+
+  ci-consistency:
+    runs-on: ubuntu-latest
+    # Only check PRs for consistency (not master)
+    if: ${{ github.event.pull_request.head.sha != '' }}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Checkout Merge Pipeline Files
+        uses: actions/checkout@v4
+        with:
+          path: merge-commit-pipeline-files
+          sparse-checkout: |
+            .github/workflows/ci.yml
+            .github/workflows/setup-runner.yml
+
+      - name: Ensure CI Consistency
+        if: ${{ github.event.pull_request.head.sha != '' }}
+        run: |
+          # Compare the checked-out CI configuration files with the reference files
+          if ! git diff --no-index .github/workflows/ci.yml merge-commit-pipeline-files/.github/workflows/ci.yml; then
+            echo "Error: ci.yml changes in master (or PR base). *Usually* you can continue just fine. If you hit trouble with missing files, please merge these changes. This is to alert about potential surprises from Github Action's merge behavior."
+            exit 1
+          fi
+          if ! git diff --no-index .github/workflows/setup-runner.yml merge-commit-pipeline-files/.github/workflows/setup-runner.yml; then
+            echo "Error: setup-runner.yml changes in master (or PR base). *Usually* you can continue just fine. If you hit trouble with missing files, please merge these changes. This is to alert about potential surprises from Github Action's merge behavior."
+            exit 1
+          fi
+
   build:
     needs: setup
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
@@ -73,7 +106,7 @@ jobs:
         timeout-minutes: 40
         uses: ./.github/ensure-tester-with-images
         with:
-          runner_type: ${{ matrix.test == 'client-prover-integration' && '32core-tester-x86' || '8core-tester-x86' }}
+          runner_type: ${{ contains(matrix.test, 'prover') && '64core-tester-x86' || '8core-tester-x86' }}
           builder_type: builder-x86
           # these are copied to the tester and expected by the earthly command below
           # if they fail to copy, it will try to build them on the tester and fail
@@ -102,7 +135,7 @@ jobs:
         uses: ./.github/ensure-tester-with-images
         timeout-minutes: 40
         with:
-          runner_type: 16core-tester-x86
+          runner_type: ${{ contains(matrix.test, 'prover') && '64core-tester-x86' || '16core-tester-x86' }}
           builder_type: builder-x86
           # these are copied to the tester and expected by the earthly command below
           # if they fail to copy, it will try to build them on the tester and fail
@@ -208,9 +241,21 @@ jobs:
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: noir-x86
-      - name: "Test Noir JS packages"
+      - name: "Test Nargo"
         run: earthly-ci --no-output ./noir+test
 
+  noir-examples:
+    needs: setup
+    runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
+    steps:
+      - uses: actions/checkout@v4
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: ./.github/ci-setup-action
+        with:
+          concurrency_key: noir-examples-x86
+      - name: "Test Noir examples"
+        run: earthly-ci --no-output ./noir+examples
+
   noir-packages-test:
     needs: setup
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
@@ -357,7 +402,7 @@ jobs:
           concurrency_key: docs-preview-x86
       - name: "Docs Preview"
         timeout-minutes: 40
-        run: earthly-ci --no-output ./docs/+deploy-preview --PR=${{ github.event.number }} --AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} --NETLIFY_AUTH_TOKEN=${{ secrets.NETLIFY_AUTH_TOKEN }} --NETLIFY_SITE_ID=${{ secrets.NETLIFY_SITE_ID }}
+        run: earthly-ci --no-output ./docs/+deploy-preview --ENV=staging --PR=${{ github.event.number }} --AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} --NETLIFY_AUTH_TOKEN=${{ secrets.NETLIFY_AUTH_TOKEN }} --NETLIFY_SITE_ID=${{ secrets.NETLIFY_SITE_ID }}
 
   bb-bench:
     runs-on: ubuntu-latest
@@ -420,20 +465,19 @@ jobs:
   merge-check:
     runs-on: ubuntu-latest
     needs:
-      [
-        e2e,
-        bb-native-tests,
-        bb-bench,
-        yarn-project-formatting,
-        yarn-project-test,
-        prover-client-test,
-        bb-js-test,
-        barretenberg-acir-tests-bb-js,
-        barretenberg-acir-tests-bb,
-        barretenberg-acir-tests-sol,
-        noir-test,
-        noir-packages-test,
-      ]
+      - setup
+      - e2e
+      - bb-native-tests
+      - bb-bench
+      - yarn-project-formatting
+      - yarn-project-test
+      - prover-client-test
+      - bb-js-test
+      - barretenberg-acir-tests-bb-js
+      - barretenberg-acir-tests-bb
+      - barretenberg-acir-tests-sol
+      - noir-test
+      - noir-packages-test
     if: always()
     steps:
       - run: |

diff --git a/.github/workflows/setup-runner.yml b/.github/workflows/setup-runner.yml
@@ -53,30 +53,6 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: Checkout Merge Pipeline Files
-        uses: actions/checkout@v4
-        # Only check PRs for consistency (not master)
-        if: ${{ github.event.pull_request.head.sha != '' }}
-        with:
-          path: merge-commit-pipeline-files
-          sparse-checkout: |
-            .github/workflows/ci.yml
-            .github/workflows/setup-runner.yml
-
-      - name: Ensure CI Consistency
-        # Only check PRs for consistency (not master)
-        if: ${{ github.event.pull_request.head.sha != '' }}
-        run: |
-          # Compare the checked-out CI configuration files with the reference files
-          if ! git diff --no-index .github/workflows/ci.yml merge-commit-pipeline-files/.github/workflows/ci.yml; then
-            echo "Error: ci.yml changes in master (or PR base). Please merge these changes. This is to prevent surprises from Github Action's merge behavior."
-            exit 1
-          fi
-          if ! git diff --no-index .github/workflows/setup-runner.yml merge-commit-pipeline-files/.github/workflows/setup-runner.yml; then
-            echo "Error: setup-runner.yml changes in master (or PR base). Please merge these changes. This is to prevent surprises from Github Action's merge behavior."
-            exit 1
-          fi
-
       - name: Start EC2 runner
         uses: ./.github/ensure-builder
         with:

diff --git a/.gitmodules b/.gitmodules
@@ -12,4 +12,4 @@
 	url = https://github.com/Arachnid/solidity-stringutils
 [submodule "barretenberg/sol/lib/openzeppelin-contracts"]
 	path = barretenberg/sol/lib/openzeppelin-contracts
-	url = https://github.com/OpenZeppelin/openzeppelin-contracts
+	url = https://github.com/OpenZeppelin/openzeppelin-contracts
diff --git a/avm-transpiler/Cargo.lock b/avm-transpiler/Cargo.lock
diff --git a/avm-transpiler/Cargo.toml b/avm-transpiler/Cargo.toml
@@ -19,4 +19,5 @@ regex = "1.10"
 env_logger = "0.11"
 log = "0.4"
 serde_json = "1.0"
-serde = { version = "1.0.136", features = ["derive"]}
+serde = { version = "1.0.136", features = ["derive"] }
+flate2 = "1.0"
diff --git a/avm-transpiler/src/transpile_contract.rs b/avm-transpiler/src/transpile_contract.rs
@@ -1,9 +1,10 @@
+use std::io::Read;
+
 use base64::Engine;
 use log::info;
 use serde::{Deserialize, Serialize};
 
 use acvm::acir::circuit::Program;
-use noirc_errors::debug_info::DebugInfo;
 use noirc_errors::debug_info::ProgramDebugInfo;
 
 use crate::transpile::{brillig_to_avm, map_brillig_pcs_to_avm_pcs, patch_debug_info_pcs};
@@ -104,6 +105,21 @@ impl From<CompiledAcirContractArtifact> for TranspiledContractArtifact {
                 // Transpile to AVM
                 let avm_bytecode = brillig_to_avm(brillig_bytecode, &brillig_pcs_to_avm_pcs);
 
+                // Gzip AVM bytecode. This has to be removed once we need to do bytecode verification.
+                let mut compressed_avm_bytecode = Vec::new();
+                let mut encoder =
+                    flate2::read::GzEncoder::new(&avm_bytecode[..], flate2::Compression::best());
+                let _ = encoder.read_to_end(&mut compressed_avm_bytecode);
+
+                log::info!(
+                    "{}::{}: compressed {} to {} bytes ({}% reduction)",
+                    contract.name,
+                    function.name,
+                    avm_bytecode.len(),
+                    compressed_avm_bytecode.len(),
+                    100 - (compressed_avm_bytecode.len() * 100 / avm_bytecode.len())
+                );
+
                 // Patch the debug infos with updated PCs
                 let debug_infos = patch_debug_info_pcs(
                     &function.debug_symbols.debug_infos,
@@ -117,7 +133,7 @@ impl From<CompiledAcirContractArtifact> for TranspiledContractArtifact {
                         is_unconstrained: function.is_unconstrained,
                         custom_attributes: function.custom_attributes,
                         abi: function.abi,
-                        bytecode: base64::prelude::BASE64_STANDARD.encode(avm_bytecode),
+                        bytecode: base64::prelude::BASE64_STANDARD.encode(compressed_avm_bytecode),
                         debug_symbols: ProgramDebugInfo { debug_infos },
                     },
                 ));

diff --git a/barretenberg/.gitrepo b/barretenberg/.gitrepo
@@ -6,7 +6,7 @@
 [subrepo]
 	remote = https://github.com/AztecProtocol/barretenberg
 	branch = master
-	commit = 8cb6cf04f5812579546d31654910f49e507f01e2
-	parent = 92384ce9d69889ed63a913f8a53d2a4b9cc672bf
+	commit = af5ae45393b9dba4c77626c5f9591d53897ae6f2
+	parent = 1fa59637a0829208d382d1dded36df33f4d61582
 	method = merge
 	cmdver = 0.4.6
diff --git a/barretenberg/acir_tests/gen_inner_proof_inputs.sh b/barretenberg/acir_tests/gen_inner_proof_inputs.sh
@@ -28,18 +28,18 @@ VFLAG=${VERBOSE:+-v}
 RFLAG=${RECURSIVE:+-r}
 
 echo "Write VK to file for assert_statement..."
-$BIN write_vk $VFLAG -c $CRS_PATH -o
+$BIN write_vk $VFLAG -c $CRS_PATH -o ./target/vk
 
 echo "Write VK as fields for recursion..."
-$BIN vk_as_fields $VFLAG -c $CRS_PATH
+$BIN vk_as_fields $VFLAG -c $CRS_PATH -k ./target/vk -o ./target/vk_fields.json
 
 echo "Generate proof to file..."
 [ -d "$PROOF_DIR" ] || mkdir $PWD/proofs
 [ -e "$PROOF_PATH" ] || touch $PROOF_PATH
-$BIN prove $VFLAG -c $CRS_PATH -b ./target/program.json -o "./proofs/$PROOF_NAME" $RFLAG
+$BIN prove $VFLAG -c $CRS_PATH -b ./target/program.json -o "./proofs/$PROOF_NAME"
 
 echo "Write proof as fields for recursion..."
-$BIN proof_as_fields $VFLAG -c $CRS_PATH -p "./proofs/$PROOF_NAME"
+$BIN proof_as_fields $VFLAG -c $CRS_PATH -p "./proofs/$PROOF_NAME" -k ./target/vk -o "./proofs/${PROOF_NAME}_fields.json"
 
 cat ./proofs/${PROOF_NAME}_fields.json
 echo