Merge branch 'master' into db/toradixbe

.github/ci-setup-action/action.yml

@@ -35,6 +35,10 @@ runs:
     # TODO reconsider how jq gets into image
     - name: Setup jq
       uses: dcarbone/install-jq-action@v2.1.0
+    - name: Setup yq
+      uses: dcarbone/install-yq-action@v1.1.1
+      with:
+        version: v4.44.3
 
     - name: Setup Earthly
       uses: earthly/actions-setup@v1

.github/ensure-tester/action.yml

@@ -95,6 +95,16 @@ runs:
           " || true
         fi
 
+    - name: Setup yq
+      shell: bash
+      run: |
+        set -x
+        scripts/run_on_tester "
+          wget https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_linux_amd64 -O ./yq
+          chmod +x ./yq
+          sudo mv ./yq /usr/bin/yq
+        "
+
     - name: Ensure Tester Cleanup
       uses: gacts/run-and-post-run@v1
       with:

.github/workflows/ci-arm.yml

@@ -3,7 +3,7 @@ on:
   push:
     branches:
       - master
-      - '*/*arm-build'
+      - "*/*arm-build"
   workflow_dispatch:
     inputs: {}
 concurrency:
@@ -101,8 +101,9 @@ jobs:
       # prepare images locally, tagged by commit hash
       - name: "Build E2E Image"
         timeout-minutes: 40
+        working-directory: ./yarn-project/end-to-end
         run: |
-          earthly-ci ./yarn-project/end-to-end+uniswap-trade-on-l1-from-l2
+          ./scripts/e2e_compose_test.sh uniswap_trade_on_l1_from_l2.test.ts
 
   rerun-check:
     runs-on: ubuntu-20.04

.github/workflows/ci.yml

@@ -4,7 +4,7 @@ on:
     branches:
       - master
       - provernet
-      - '*/ci-push*'
+      - "*/ci-push*"
   pull_request:
     types:
       - opened
@@ -172,6 +172,7 @@ jobs:
             LABELS=""
           fi
           echo Labels: $LABELS
+          echo "list=$(./scripts/ci/get_e2e_jobs.sh ${{github.ref_name}} "$LABELS")"
           echo "list=$(./scripts/ci/get_e2e_jobs.sh ${{github.ref_name}} "$LABELS")" >> $GITHUB_OUTPUT
       - name: Create list of bench end-to-end jobs
         id: bench_list
@@ -197,24 +198,26 @@ jobs:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
+      - name: Determine runner type
+        id: runner_type
+        run: |
+          runner=$(./scripts/ci/select_runner.sh ${{ matrix.test }})
+          echo "type=$runner" >> $GITHUB_OUTPUT
       - name: Setup and Test
         timeout-minutes: 40
         uses: ./.github/ensure-tester-with-images
         env:
           USERNAME: ${{ needs.configure.outputs.username }}
         with:
-          runner_type: ${{ contains(matrix.test, 'prover') && '64core-tester-x86' || '8core-tester-x86' }}
+          runner_type: ${{ steps.runner_type.outputs.type }}
           builder_type: builder-x86
-          # these are copied to the tester and expected by the earthly command below
-          # if they fail to copy, it will try to build them on the tester and fail
           builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
-          # command to produce the images in case they don't exist
           builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images
           run: |
             set -eux
             cd ./yarn-project/end-to-end/
             export FORCE_COLOR=1
-            ../../scripts/earthly-ci -P --no-output +${{ matrix.test }}
+            ./scripts/e2e_test.sh ${{ matrix.test }}
 
   # all the benchmarking end-to-end integration tests for aztec (not required to merge)
   bench-e2e:
@@ -229,13 +232,18 @@ jobs:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
+      - name: Determine runner type
+        id: runner_type
+        run: |
+          runner=$(./scripts/ci/select_runner.sh ${{ matrix.test }})
+          echo "type=$runner" >> $GITHUB_OUTPUT
       - name: Setup and Test
         uses: ./.github/ensure-tester-with-images
         env:
           USERNAME: ${{ needs.configure.outputs.username }}
         timeout-minutes: 45
         with:
-          runner_type: ${{ contains(matrix.test, 'prover') && '64core-tester-x86' || '16core-tester-x86' }}
+          runner_type: ${{ steps.runner_type.outputs.type }}
           builder_type: builder-x86
           # these are copied to the tester and expected by the earthly command below
           # if they fail to copy, it will try to build them on the tester and fail
@@ -248,7 +256,9 @@ jobs:
             cd ./yarn-project/end-to-end/
             export FORCE_COLOR=1
             export EARTHLY_BUILD_ARGS="${{ env.EARTHLY_BUILD_ARGS }}"
-            ../../scripts/earthly-ci --exec-stats -P --no-output +${{ matrix.test }}
+            ./scripts/e2e_test.sh ${{ matrix.test }}
+
+            earthly-ci +UPLOAD_LOGS --PULL_REQUEST=${{ github.event.pull_request.number }} --BRANCH=${{ github.ref_name }} --COMMIT_HASH=${{ env.GIT_COMMIT }}
 
   acir-bench:
     runs-on: ubuntu-20.04
@@ -584,7 +594,7 @@ jobs:
 
   yarn-project-formatting:
     needs: [build, configure]
-    if: needs.configure.outputs.yarn-project == 'true' || github.ref_name == 'master' 
+    if: needs.configure.outputs.yarn-project == 'true' || github.ref_name == 'master'
     runs-on: ${{ needs.configure.outputs.username }}-x86
     steps:
       - uses: actions/checkout@v4
@@ -598,7 +608,7 @@ jobs:
 
   yarn-project-test:
     needs: [build, configure]
-    if: needs.configure.outputs.yarn-project == 'true' || github.ref_name == 'master' 
+    if: needs.configure.outputs.yarn-project == 'true' || github.ref_name == 'master'
     runs-on: ${{ needs.configure.outputs.username }}-x86
     steps:
       - uses: actions/checkout@v4
@@ -613,7 +623,7 @@ jobs:
   prover-client-test:
     needs: [build, configure]
     runs-on: ${{ needs.configure.outputs.username }}-x86
-    if: needs.configure.outputs.yarn-project == 'true' || github.ref_name == 'master' 
+    if: needs.configure.outputs.yarn-project == 'true' || github.ref_name == 'master'
     steps:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
@@ -627,7 +637,7 @@ jobs:
   # proving disabled
   network-test:
     needs: [build, configure]
-    if: needs.configure.outputs.yarn-project == 'true'
+    if: needs.configure.outputs.yarn-project == 'true' || github.ref_name == 'master'
     runs-on: ${{ needs.configure.outputs.username }}-x86
     strategy:
       max-parallel: 1
@@ -649,7 +659,7 @@ jobs:
   # note: proving disabled
   kind-network-test:
     needs: [build, configure]
-    if: needs.configure.output.yarn-project == 'true'
+    if: needs.configure.outputs.yarn-project == 'true' || github.ref_name == 'master'
     runs-on: ${{ needs.configure.outputs.username }}-x86
     strategy:
       fail-fast: false

.github/workflows/nightly-kind-test.yml

@@ -74,8 +74,7 @@ jobs:
             set -eux
             ./spartan/scripts/setup_local_k8s.sh
             export FORCE_COLOR=1
-            export EARTHLY_BUILD_ARGS="${{ env.EARTHLY_BUILD_ARGS }}"
-            ./scripts/earthly-ci --exec-stats -P --no-output ./yarn-project/end-to-end/+kind-network-transfer --values-file=${{ matrix.values_file }}
+            NAMESPACE=transfer FRESH_INSTALL=true VALUES_FILE=${{ matrix.values_file }} ./scripts/network_test.sh ./src/spartan/transfer.test.ts || true
 
   success-check:
     runs-on: ubuntu-20.04

.github/workflows/sepolia-test.yml

@@ -54,11 +54,11 @@ jobs:
           set -eux
           cd ./yarn-project/end-to-end/
           export FORCE_COLOR=1
-          ../../scripts/earthly-ci -P --no-output +e2e-public-testnet \
-            --SEQ_PUBLISHER_PRIVATE_KEY=${{ secrets.SEPOLIA_SEQ_PRIVATE_KEY }} \
-            --PROVER_PUBLISHER_PRIVATE_KEY=${{ secrets.SEPOLIA_PROVER_PRIVATE_KEY }} \
-            --ETHEREUM_HOST="https://sepolia.infura.io/v3/${{ secrets.SEPOLIA_API_KEY }}" \
-            --L1_CHAIN_ID=${{ env.L1_CHAIN_ID }}
+          SEQ_PUBLISHER_PRIVATE_KEY=${{ secrets.SEPOLIA_SEQ_PRIVATE_KEY }} \
+          PROVER_PUBLISHER_PRIVATE_KEY=${{ secrets.SEPOLIA_PROVER_PRIVATE_KEY }} \
+          ETHEREUM_HOST="https://sepolia.infura.io/v3/${{ secrets.SEPOLIA_API_KEY }}" \
+          L1_CHAIN_ID=${{ env.L1_CHAIN_ID }} \
+          ./scripts/e2e_test_public_testnet.sh ./src/public-testnet
 
   success-check:
     runs-on: ubuntu-20.04

Earthfile

@@ -20,9 +20,6 @@ build:
     # yarn-project has the entry point to Aztec
     BUILD ./yarn-project/+build
 
-test-end-to-end:
-    BUILD ./yarn-project/end-to-end+e2e-tests
-
 release-meta:
     COPY .release-please-manifest.json /usr/src/.release-please-manifest.json
     SAVE ARTIFACT /usr/src /usr/src
@@ -46,7 +43,7 @@ UPLOAD_LOGS:
     ENV BRANCH=$BRANCH
     ENV COMMIT_HASH=$COMMIT_HASH
     RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY /usr/src/scripts/logs/upload_logs_to_s3.sh /usr/var/log
-    
+
 base-log-uploader:
     # Install awscli on a fresh ubuntu, and copy the repo "scripts" folder, which we'll use to upload logs
     # Note that we cannot do this LOCALLY because Earthly does not support using secrets locally

barretenberg/cpp/src/barretenberg/bb/main.cpp

@@ -598,8 +598,8 @@ void prove_tube(const std::string& output_path)
     // these public inputs by turning proof into witnesses and call
     // set_public on each witness
     auto num_public_inputs = static_cast<uint32_t>(static_cast<uint256_t>(proof.folding_proof[1]));
-    num_public_inputs -= bb::AGGREGATION_OBJECT_SIZE; // don't add the agg object
-    num_public_inputs -= 1 * 8; // TODO(https://github.com/AztecProtocol/barretenberg/issues/1125) Make this dynamic
+    num_public_inputs -= bb::AGGREGATION_OBJECT_SIZE;             // don't add the agg object
+    num_public_inputs -= bb::PROPAGATED_DATABUS_COMMITMENTS_SIZE; // exclude propagated databus commitments
     for (size_t i = 0; i < num_public_inputs; i++) {
         auto offset = acir_format::HONK_RECURSION_PUBLIC_INPUT_OFFSET;
         builder->add_public_variable(proof.folding_proof[i + offset]);

barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp

@@ -54,6 +54,9 @@ void ClientIVC::perform_recursive_verification_and_databus_consistency_checks(
     const std::shared_ptr<RecursiveVerificationKey>& vkey,
     const QUEUE_TYPE type)
 {
+    // Store the decider vk for the incoming circuit; its data is used in the databus consistency checks below
+    std::shared_ptr<RecursiveDeciderVerificationKey> decider_vk;
+
     switch (type) {
     case QUEUE_TYPE::PG: {
         // Construct stdlib verifier accumulator from the native counterpart computed on a previous round
@@ -66,10 +69,8 @@ void ClientIVC::perform_recursive_verification_and_databus_consistency_checks(
         // Extract native verifier accumulator from the stdlib accum for use on the next round
         verifier_accumulator = std::make_shared<DeciderVerificationKey>(verifier_accum->get_value());
 
-        // Perform databus commitment consistency checks and propagate return data commitments via public inputs
-        bus_depot.execute(verifier.keys_to_fold[1]->witness_commitments,
-                          verifier.keys_to_fold[1]->public_inputs,
-                          verifier.keys_to_fold[1]->verification_key->databus_propagation_data);
+        decider_vk = verifier.keys_to_fold[1]; // decider vk for the incoming circuit
+
         break;
     }
     case QUEUE_TYPE::OINK: {
@@ -86,14 +87,20 @@ void ClientIVC::perform_recursive_verification_and_databus_consistency_checks(
         // Initialize the gate challenges to zero for use in first round of folding
         verifier_accumulator->gate_challenges = std::vector<FF>(CONST_PG_LOG_N, 0);
 
-        // Perform databus commitment consistency checks and propagate return data commitments via public inputs
-        bus_depot.execute(verifier_accum->witness_commitments,
-                          verifier_accum->public_inputs,
-                          verifier_accum->verification_key->databus_propagation_data);
+        decider_vk = verifier_accum; // decider vk for the incoming circuit
 
         break;
     }
     }
+
+    // Set the return data commitment to be propagated on the public inputs of the present kernel and peform consistency
+    // checks between the calldata commitments and the return data commitments contained within the public inputs
+    bus_depot.set_return_data_to_be_propagated_and_perform_consistency_checks(
+        decider_vk->witness_commitments.return_data,
+        decider_vk->witness_commitments.calldata,
+        decider_vk->witness_commitments.secondary_calldata,
+        decider_vk->public_inputs,
+        decider_vk->verification_key->databus_propagation_data);
 }
 
 /**
@@ -133,6 +140,9 @@ void ClientIVC::complete_kernel_circuit_logic(ClientCircuit& circuit)
     }
     stdlib_verification_queue.clear();
 
+    // Propagate return data commitments via the public inputs for use in databus consistency checks
+    bus_depot.propagate_return_data_commitments(circuit);
+
     // Perform recursive merge verification for every merge proof in the queue
     process_recursive_merge_verification_queue(circuit);
 }

barretenberg/cpp/src/barretenberg/client_ivc/client_ivc_integration.test.cpp

@@ -93,10 +93,14 @@ TEST_F(ClientIVCIntegrationTests, BenchmarkCasePrecomputedVKs)
 
     size_t NUM_CIRCUITS = 6;
 
-    MockCircuitProducer circuit_producer;
-
-    auto precomputed_vks = circuit_producer.precompute_verification_keys(NUM_CIRCUITS, ivc.trace_structure);
+    // Precompute the verification keys for each circuit in the IVC
+    std::vector<std::shared_ptr<VerificationKey>> precomputed_vks;
+    {
+        MockCircuitProducer circuit_producer;
+        precomputed_vks = circuit_producer.precompute_verification_keys(NUM_CIRCUITS, ivc.trace_structure);
+    }
 
+    MockCircuitProducer circuit_producer;
     // Construct and accumulate a series of mocked private function execution circuits
     for (size_t idx = 0; idx < NUM_CIRCUITS; ++idx) {
         Builder circuit = circuit_producer.create_next_circuit(ivc);

barretenberg/cpp/src/barretenberg/stdlib/primitives/bigfield/goblin_field.hpp

@@ -74,6 +74,16 @@ template <class Builder> class goblin_field {
         return goblin_field(lo, hi);
     }
 
+    /**
+     * Create a witness from a constant. This way the value of the witness is fixed and public.
+     **/
+    void convert_constant_to_fixed_witness(Builder* builder)
+    {
+        for (auto& limb : limbs) {
+            limb.convert_constant_to_fixed_witness(builder);
+        }
+    }
+
     static goblin_field conditional_assign(const bool_ct& predicate, const goblin_field& lhs, goblin_field& rhs)
     {
         goblin_field result;

barretenberg/cpp/src/barretenberg/stdlib/primitives/biggroup/biggroup.hpp

@@ -75,6 +75,15 @@ template <class Builder, class Fq, class Fr, class NativeGroup> class element {
         }
     }
 
+    /**
+     * @brief Creates fixed witnesses from a constant element.
+     **/
+    void convert_constant_to_fixed_witness(Builder* builder)
+    {
+        this->x.convert_constant_to_fixed_witness(builder);
+        this->y.convert_constant_to_fixed_witness(builder);
+    }
+
     static element one(Builder* ctx)
     {
         uint256_t x = uint256_t(NativeGroup::one.x);

barretenberg/cpp/src/barretenberg/stdlib/primitives/biggroup/biggroup_goblin.hpp

@@ -69,6 +69,15 @@ template <class Builder, class Fq, class Fr, class NativeGroup> class goblin_ele
         return out;
     }
 
+    /**
+     * @brief Creates fixed witnesses from a constant element.
+     **/
+    void convert_constant_to_fixed_witness(Builder* builder)
+    {
+        this->x.convert_constant_to_fixed_witness(builder);
+        this->y.convert_constant_to_fixed_witness(builder);
+    }
+
     void validate_on_curve() const
     {
         // happens in goblin eccvm