feat: prepend based merge (#12093)

Reorganize the existing merge protocol to establish the _pre_-pending of
subtables of ecc ops from each circuit, rather than appending. This is
facilitated by classes `UltraEccOpsTable` and `EccvmOpsTable`
(implemented in a previous PR) that handle the storage and virtual
pre-pending of subtables.

The merge protocol proceeds by opening univariate polynomials T_j,
T_{j,prev}, and t_j (columns of full table, previous full table, and
current subtable respectively) and checking the identity T_j(x) = t_j(x)
+ x^k * T_{j,prev}(x) at a single challenge point. (Polynomials t_j are
explicitly degree checked in main protocol via a relation that checks
that they are zero beyond idx k-1).

Note: Missing pieces in the merge are (1) connecting [t] from the main
protocol to [t] in the merge and (2) connecting [T] from step i-1 to
[T_prev] at step i. These will be handled in follow ons.

cpp/src/barretenberg/circuit_checker/mega_circuit_builder.test.cpp

@@ -149,7 +149,7 @@ TEST(MegaCircuitBuilder, GoblinEccOpQueueUltraOps)
     builder.queue_ecc_eq();
 
     // Check that the ultra ops recorded in the EccOpQueue match the ops recorded in the wires
-    auto ultra_ops = builder.op_queue->get_aggregate_transcript();
+    auto ultra_ops = builder.op_queue->construct_current_ultra_ops_subtable_columns();
     for (size_t i = 1; i < 4; ++i) {
         for (size_t j = 0; j < builder.blocks.ecc_op.size(); ++j) {
             auto op_wire_val = builder.variables[builder.blocks.ecc_op.wires[i][j]];
@@ -203,4 +203,4 @@ TEST(MegaCircuitBuilder, CompleteSelectorPartitioningCheck)
     }
 }
 
-} // namespace bb
+} // namespace bb

cpp/src/barretenberg/client_ivc/mock_kernel_pinning.test.cpp

@@ -33,7 +33,8 @@ TEST_F(MockKernelTest, PinFoldingKernelSizes)
         Builder circuit = circuit_producer.create_next_circuit(ivc);
 
         ivc.accumulate(circuit);
+        EXPECT_FALSE(circuit.blocks.has_overflow); // trace oveflow mechanism should not be triggered
     }
 
     EXPECT_EQ(ivc.fold_output.accumulator->proving_key.log_circuit_size, 19);
-}
+}

cpp/src/barretenberg/constants.hpp

@@ -24,4 +24,6 @@ static constexpr uint32_t MASKING_OFFSET = 4;
 // For ZK Flavors: the number of the commitments required by Libra and SmallSubgroupIPA.
 static constexpr uint32_t NUM_LIBRA_COMMITMENTS = 3;
 static constexpr uint32_t NUM_LIBRA_EVALUATIONS = 4;
-} // namespace bb
+
+static constexpr uint32_t MERGE_PROOF_SIZE = 65; // used to ensure mock proofs are generated correctly
+} // namespace bb

cpp/src/barretenberg/dsl/acir_format/ivc_recursion_constraint.cpp

@@ -241,11 +241,15 @@ ClientIVC::MergeProof create_dummy_merge_proof()
     using FF = ClientIVC::FF;
 
     std::vector<FF> proof;
+    proof.reserve(MERGE_PROOF_SIZE);
 
     FF mock_val(5);
     auto mock_commitment = curve::BN254::AffineElement::one();
     std::vector<FF> mock_commitment_frs = field_conversion::convert_to_bn254_frs(mock_commitment);
 
+    // Populate mock subtable size
+    proof.emplace_back(mock_val);
+
     // There are 12 entities in the merge protocol (4 columns x 3 components; aggregate transcript, previous aggregate
     // transcript, current transcript contribution)
     const size_t NUM_TRANSCRIPT_ENTITIES = 12;
@@ -265,6 +269,8 @@ ClientIVC::MergeProof create_dummy_merge_proof()
         proof.emplace_back(val);
     }
 
+    ASSERT(proof.size() == MERGE_PROOF_SIZE);
+
     return proof;
 }
 

cpp/src/barretenberg/dsl/acir_format/ivc_recursion_constraint.test.cpp

@@ -136,6 +136,15 @@ class IvcRecursionConstraintTest : public ::testing::Test {
     }
 };
 
+/**
+ * @brief Check that the size of a mock merge proof matches expectation
+ */
+TEST_F(IvcRecursionConstraintTest, MockMergeProofSize)
+{
+    ClientIVC::MergeProof merge_proof = create_dummy_merge_proof();
+    EXPECT_EQ(merge_proof.size(), MERGE_PROOF_SIZE);
+}
+
 /**
  * @brief Test IVC accumulation of a one app and one kernel; The kernel includes a recursive oink verification for the
  * app, specified via an ACIR RecursionConstraint.

cpp/src/barretenberg/eccvm/eccvm_circuit_builder.hpp

@@ -113,10 +113,10 @@ class ECCVMCircuitBuilder {
         std::vector<std::pair<size_t, size_t>> msm_mul_index;
         std::vector<size_t> msm_sizes;
 
-        const auto& raw_ops = op_queue->get_raw_ops();
+        const auto& eccvm_ops = op_queue->get_eccvm_ops();
         size_t op_idx = 0;
         // populate opqueue and mul indices
-        for (const auto& op : raw_ops) {
+        for (const auto& op : eccvm_ops) {
             if (op.mul) {
                 if ((op.z1 != 0 || op.z2 != 0) && !op.base_point.is_point_at_infinity()) {
                     msm_opqueue_index.push_back(op_idx);
@@ -131,7 +131,7 @@ class ECCVMCircuitBuilder {
             op_idx++;
         }
         // if last op is a mul we have not correctly computed the total number of msms
-        if (raw_ops.back().mul && active_mul_count > 0) {
+        if (eccvm_ops.back().mul && active_mul_count > 0) {
             msm_sizes.push_back(active_mul_count);
             msm_count++;
         }
@@ -143,7 +143,7 @@ class ECCVMCircuitBuilder {
 
         parallel_for_range(msm_opqueue_index.size(), [&](size_t start, size_t end) {
             for (size_t i = start; i < end; i++) {
-                const auto& op = raw_ops[msm_opqueue_index[i]];
+                const auto& op = eccvm_ops[msm_opqueue_index[i]];
                 auto [msm_index, mul_index] = msm_mul_index[i];
                 if (op.z1 != 0 && !op.base_point.is_point_at_infinity()) {
                     ASSERT(result.size() > msm_index);

cpp/src/barretenberg/eccvm/eccvm_circuit_builder.test.cpp

@@ -454,7 +454,7 @@ TEST(ECCVMCircuitBuilderTests, AddProducesDouble)
  * @details Currently, Goblin does not support clean initialization, which means that we have to create mock ECCOpQueue
  * to avoid commiting to zero polynomials. This test localizes the issue to the problem with populating ECCVM Transcript
  * rows in the method \ref  bb::ECCVMTranscriptBuilder::compute_rows "compute rows". Namely, we are loosing the point at
- * infinity contribution to the 'transcipt_Px' polynomials while parsing the raw ops of ECCOpQueue.
+ * infinity contribution to the 'transcipt_Px' polynomials while parsing the eccvm ops of ECCOpQueue.
  *
  * More specifically, in this test we add a simple MSM with the point at infinity multiplied by \f$0\f$. While the ECCVM
  * computes the result of this op correctly, i.e. outputs the point at infinity, the computation of 'transcript_Px' is
@@ -474,15 +474,15 @@ TEST(ECCVMCircuitBuilderTests, InfinityFailure)
     bb::srs::init_grumpkin_crs_factory(bb::srs::get_grumpkin_crs_path());
 
     // Add the same operations to the ECC op queue; the native computation is performed under the hood.
-    auto op_queue = std::make_shared<bb::ECCOpQueue>();
+    std::shared_ptr<ECCOpQueue> op_queue = std::make_shared<ECCOpQueue>();
 
     for (size_t i = 0; i < 1; i++) {
         op_queue->mul_accumulate(P1, Fr(0));
     }
 
     auto eccvm_builder = ECCVMCircuitBuilder(op_queue);
 
-    auto transcript_rows = ECCVMTranscriptBuilder::compute_rows(op_queue->get_raw_ops(), 1);
+    auto transcript_rows = ECCVMTranscriptBuilder::compute_rows(op_queue->get_eccvm_ops(), 1);
 
     // check that the corresponding op is mul
     bool row_op_code_correct = transcript_rows[1].opcode == 4;

cpp/src/barretenberg/eccvm/eccvm_flavor.hpp

@@ -533,7 +533,7 @@ class ECCVMFlavor {
         {
             // compute rows for the three different sections of the ECCVM execution trace
             const auto transcript_rows =
-                ECCVMTranscriptBuilder::compute_rows(builder.op_queue->get_raw_ops(), builder.get_number_of_muls());
+                ECCVMTranscriptBuilder::compute_rows(builder.op_queue->get_eccvm_ops(), builder.get_number_of_muls());
             const std::vector<MSM> msms = builder.get_msms();
             const auto point_table_rows =
                 ECCVMPointTablePrecomputationBuilder::compute_rows(CircuitBuilder::get_flattened_scalar_muls(msms));
@@ -1461,4 +1461,4 @@ class ECCVMFlavor {
 
 // NOLINTEND(cppcoreguidelines-avoid-const-or-ref-data-members)
 
-} // namespace bb
+} // namespace bb

cpp/src/barretenberg/eccvm/eccvm_transcript.test.cpp

@@ -405,4 +405,4 @@ TEST_F(ECCVMTranscriptTests, StructureTest)
     prover.transcript->deserialize_full_transcript();
     EXPECT_EQ(static_cast<typename Flavor::Commitment>(prover.transcript->transcript_Px_comm),
               one_group_val * rand_val);
-}
+}

cpp/src/barretenberg/goblin/goblin.hpp

@@ -75,7 +75,7 @@ class GoblinProver {
       // commitments (https://github.com/AztecProtocol/barretenberg/issues/871) which would otherwise appear in the
       // first round of the merge protocol. To be removed once the issue has been resolved.
         commitment_key = bn254_commitment_key ? bn254_commitment_key : nullptr;
-        GoblinMockCircuits::perform_op_queue_interactions_for_mock_first_circuit(op_queue, commitment_key);
+        GoblinMockCircuits::perform_op_queue_interactions_for_mock_first_circuit(op_queue);
     }
     /**
      * @brief Construct a MegaHonk proof and a merge proof for the present circuit.

cpp/src/barretenberg/goblin/mock_circuits.hpp

@@ -138,32 +138,14 @@ class GoblinMockCircuits {
      *
      * @param op_queue
      */
-    static void perform_op_queue_interactions_for_mock_first_circuit(
-        std::shared_ptr<bb::ECCOpQueue>& op_queue, std::shared_ptr<CommitmentKey> commitment_key = nullptr)
+    static void perform_op_queue_interactions_for_mock_first_circuit(std::shared_ptr<bb::ECCOpQueue>& op_queue)
     {
         PROFILE_THIS();
 
         bb::MegaCircuitBuilder builder{ op_queue };
 
         // Add some goblinized ecc ops
         MockCircuits::construct_goblin_ecc_op_circuit(builder);
-
-        op_queue->set_size_data();
-
-        // Manually compute the op queue transcript commitments (which would normally be done by the merge prover)
-#ifndef __wasm__
-        // TODO(Adam): This is crashing wasm-in-browser. It doesn't make sense to call this in browser...
-        bb::srs::init_crs_factory(bb::srs::get_ignition_crs_path());
-#endif
-        auto bn254_commitment_key =
-            commitment_key ? commitment_key : std::make_shared<CommitmentKey>(op_queue->get_current_size());
-        std::array<Point, Flavor::NUM_WIRES> op_queue_commitments;
-        size_t idx = 0;
-        for (auto& entry : op_queue->get_aggregate_transcript()) {
-            op_queue_commitments[idx++] = bn254_commitment_key->commit({ 0, entry });
-        }
-        // Store the commitment data for use by the prover of the next circuit
-        op_queue->set_commitment_data(op_queue_commitments);
     }
 
     /**

cpp/src/barretenberg/plonk_honk_shared/execution_trace/mega_execution_trace.hpp

@@ -294,7 +294,7 @@ static constexpr TraceStructure CLIENT_IVC_BENCH_STRUCTURE{ .ecc_op = 1 << 10,
                                                             .elliptic = 9000,
                                                             .aux = 136000,
                                                             .poseidon2_external = 2500,
-                                                            .poseidon2_internal = 14000,
+                                                            .poseidon2_internal = 14500,
                                                             .overflow = 0 };
 
 /**

cpp/src/barretenberg/stdlib/eccvm_verifier/eccvm_recursive_verifier.test.cpp

@@ -199,4 +199,4 @@ TYPED_TEST(ECCVMRecursiveTests, IndependentVKHash)
 {
     TestFixture::test_independent_vk_hash();
 };
-} // namespace bb
+} // namespace bb

cpp/src/barretenberg/stdlib/goblin_verifier/merge_recursive_verifier.cpp

@@ -8,59 +8,70 @@ MergeRecursiveVerifier_<CircuitBuilder>::MergeRecursiveVerifier_(CircuitBuilder*
 {}
 
 /**
- * @brief Construct recursive verifier for Goblin Merge protocol, up to but not including the pairing
+ * @brief Computes inputs to a pairing check that, if verified, establishes proper construction of the aggregate Goblin
+ * ECC op queue polynomials T_j, j = 1,2,3,4.
+ * @details Let T_j be the jth column of the aggregate ecc op table after prepending the subtable columns t_j containing
+ * the contribution from a single circuit. T_{j,prev} corresponds to the columns of the aggregate table at the
+ * previous stage. For each column we have the relationship T_j = t_j + right_shift(T_{j,prev}, k), where k is the
+ * length of the subtable columns t_j. This protocol demonstrates, assuming the length of t is at most k, that the
+ * aggregate ecc op table has been constructed correctly via the simple Schwartz-Zippel check:
  *
- * @tparam Flavor
+ *      T_j(\kappa) = t_j(\kappa) + \kappa^k * (T_{j,prev}(\kappa)).
+ *
+ * @tparam CircuitBuilder
  * @param proof
  * @return std::array<typename Flavor::GroupElement, 2> Inputs to final pairing
  */
 template <typename CircuitBuilder>
 std::array<typename bn254<CircuitBuilder>::Element, 2> MergeRecursiveVerifier_<CircuitBuilder>::verify_proof(
     const HonkProof& proof)
 {
-    // transform it into stdlib proof
+    // Transform proof into a stdlib object
     StdlibProof<CircuitBuilder> stdlib_proof = bb::convert_native_proof_to_stdlib(builder, proof);
     transcript = std::make_shared<Transcript>(stdlib_proof);
 
-    // Receive commitments [t_i^{shift}], [T_{i-1}], and [T_i]
-    std::array<Commitment, NUM_WIRES> C_T_prev;
-    std::array<Commitment, NUM_WIRES> C_t_shift;
-    std::array<Commitment, NUM_WIRES> C_T_current;
+    FF subtable_size = transcript->template receive_from_prover<FF>("subtable_size");
+
+    // Receive table column polynomial commitments [t_j], [T_{j,prev}], and [T_j], j = 1,2,3,4
+    std::array<Commitment, NUM_WIRES> t_commitments;
+    std::array<Commitment, NUM_WIRES> T_prev_commitments;
+    std::array<Commitment, NUM_WIRES> T_commitments;
     for (size_t idx = 0; idx < NUM_WIRES; ++idx) {
-        C_T_prev[idx] = transcript->template receive_from_prover<Commitment>("T_PREV_" + std::to_string(idx + 1));
-        C_t_shift[idx] = transcript->template receive_from_prover<Commitment>("t_SHIFT_" + std::to_string(idx + 1));
-        C_T_current[idx] = transcript->template receive_from_prover<Commitment>("T_CURRENT_" + std::to_string(idx + 1));
+        std::string suffix = std::to_string(idx);
+        t_commitments[idx] = transcript->template receive_from_prover<Commitment>("t_CURRENT_" + suffix);
+        T_prev_commitments[idx] = transcript->template receive_from_prover<Commitment>("T_PREV_" + suffix);
+        T_commitments[idx] = transcript->template receive_from_prover<Commitment>("T_CURRENT_" + suffix);
     }
 
     FF kappa = transcript->template get_challenge<FF>("kappa");
 
-    // Receive transcript poly evaluations and add corresponding univariate opening claims {(\kappa, p(\kappa), [p(X)]}
+    // Receive evaluations t_j(\kappa), T_{j,prev}(\kappa), T_j(\kappa), j = 1,2,3,4
+    std::array<FF, NUM_WIRES> t_evals;
     std::array<FF, NUM_WIRES> T_prev_evals;
-    std::array<FF, NUM_WIRES> t_shift_evals;
-    std::array<FF, NUM_WIRES> T_current_evals;
+    std::array<FF, NUM_WIRES> T_evals;
     std::vector<OpeningClaim> opening_claims;
     for (size_t idx = 0; idx < NUM_WIRES; ++idx) {
-        T_prev_evals[idx] = transcript->template receive_from_prover<FF>("T_prev_eval_" + std::to_string(idx + 1));
-        opening_claims.emplace_back(OpeningClaim{ { kappa, T_prev_evals[idx] }, C_T_prev[idx] });
+        t_evals[idx] = transcript->template receive_from_prover<FF>("t_eval_" + std::to_string(idx + 1));
+        opening_claims.emplace_back(OpeningClaim{ { kappa, t_evals[idx] }, t_commitments[idx] });
     }
     for (size_t idx = 0; idx < NUM_WIRES; ++idx) {
-        t_shift_evals[idx] = transcript->template receive_from_prover<FF>("t_shift_eval_" + std::to_string(idx + 1));
-        opening_claims.emplace_back(OpeningClaim{ { kappa, t_shift_evals[idx] }, C_t_shift[idx] });
+        T_prev_evals[idx] = transcript->template receive_from_prover<FF>("T_prev_eval_" + std::to_string(idx + 1));
+        opening_claims.emplace_back(OpeningClaim{ { kappa, T_prev_evals[idx] }, T_prev_commitments[idx] });
     }
     for (size_t idx = 0; idx < NUM_WIRES; ++idx) {
-        T_current_evals[idx] =
-            transcript->template receive_from_prover<FF>("T_current_eval_" + std::to_string(idx + 1));
-        opening_claims.emplace_back(OpeningClaim{ { kappa, T_current_evals[idx] }, C_T_current[idx] });
+        T_evals[idx] = transcript->template receive_from_prover<FF>("T_eval_" + std::to_string(idx + 1));
+        opening_claims.emplace_back(OpeningClaim{ { kappa, T_evals[idx] }, T_commitments[idx] });
     }
 
-    // Check the identity T_i(\kappa) = T_{i-1}(\kappa) + t_i^{shift}(\kappa)
+    // Check the identity T_j(\kappa) = t_j(\kappa) + \kappa^m * T_{j,prev}(\kappa)
     for (size_t idx = 0; idx < NUM_WIRES; ++idx) {
-        T_current_evals[idx].assert_equal(T_prev_evals[idx] + t_shift_evals[idx]);
+        FF T_prev_shifted_eval_reconstructed = T_prev_evals[idx] * kappa.pow(subtable_size);
+        T_evals[idx].assert_equal(t_evals[idx] + T_prev_shifted_eval_reconstructed);
     }
 
     FF alpha = transcript->template get_challenge<FF>("alpha");
 
-    // Constuct batched commitment and batched evaluation from constituents using batching challenge \alpha
+    // Constuct inputs to batched commitment and batched evaluation from constituents using batching challenge \alpha
     std::vector<FF> scalars;
     std::vector<Commitment> commitments;
     scalars.emplace_back(FF(builder, 1));

cpp/src/barretenberg/stdlib/goblin_verifier/merge_verifier.test.cpp

@@ -99,4 +99,4 @@ TYPED_TEST(RecursiveMergeVerifierTest, SingleRecursiveVerification)
     TestFixture::test_recursive_merge_verification();
 };
 
-} // namespace bb::stdlib::recursion::goblin
+} // namespace bb::stdlib::recursion::goblin

cpp/src/barretenberg/stdlib/translator_vm_verifier/translator_recursive_verifier.test.cpp

@@ -199,4 +199,4 @@ TYPED_TEST(TranslatorRecursiveTests, IndependentVKHash)
         GTEST_SKIP() << "Not built for this parameter";
     }
 };
-} // namespace bb
+} // namespace bb

cpp/src/barretenberg/stdlib_circuit_builders/mega_circuit_builder.hpp

@@ -50,10 +50,13 @@ template <typename FF> class MegaCircuitBuilder_ : public UltraCircuitBuilder_<M
         , op_queue(std::move(op_queue_in))
     {
         PROFILE_THIS();
+        // Instantiate the subtable to be populated with goblin ecc ops from this circuit
+        op_queue->initialize_new_subtable();
 
         // Set indices to constants corresponding to Goblin ECC op codes
         set_goblin_ecc_op_code_constant_variables();
     };
+
     MegaCircuitBuilder_(std::shared_ptr<ECCOpQueue> op_queue_in)
         : MegaCircuitBuilder_(0, op_queue_in)
     {}
@@ -79,6 +82,9 @@ template <typename FF> class MegaCircuitBuilder_ : public UltraCircuitBuilder_<M
         : UltraCircuitBuilder_<MegaExecutionTraceBlocks>(/*size_hint=*/0, witness_values, public_inputs, varnum)
         , op_queue(std::move(op_queue_in))
     {
+        // Instantiate the subtable to be populated with goblin ecc ops from this circuit
+        op_queue->initialize_new_subtable();
+
         // Set indices to constants corresponding to Goblin ECC op codes
         set_goblin_ecc_op_code_constant_variables();
     };