fix: memory fragmentation fixes to cut UltraHonk memory usage by 26%

barretenberg/cpp/src/barretenberg/bb/exec_pipe.hpp

@@ -8,7 +8,7 @@ inline std::vector<uint8_t> exec_pipe(std::string const& command)
 {
     FILE* pipe = popen(command.c_str(), "r");
     if (!pipe) {
-        throw std::runtime_error("popen() failed!");
+        throw std::runtime_error("popen() failed! Can't run: " + command);
     }
 
     std::vector<uint8_t> result;

barretenberg/cpp/src/barretenberg/bb/get_bn254_crs.cpp

@@ -41,7 +41,7 @@ std::vector<g1::affine_element> get_bn254_g1_data(const std::filesystem::path& p
 
     if (g1_file_size >= num_points * 64 && g1_file_size % 64 == 0) {
         vinfo("using cached bn254 crs of size ", std::to_string(g1_file_size / 64), " at ", g1_path);
-        auto data = read_file(g1_path, g1_file_size);
+        auto data = read_file(g1_path, num_points * 64);
         auto points = std::vector<g1::affine_element>(num_points);
         for (size_t i = 0; i < num_points; ++i) {
             points[i] = from_buffer<g1::affine_element>(data, i * 64);

barretenberg/cpp/src/barretenberg/dsl/acir_proofs/c_bind.cpp

@@ -311,6 +311,15 @@ WASM_EXPORT void acir_verify_aztec_client(uint8_t const* proof_buf, uint8_t cons
     *result = ClientIVC::verify(proof, vk);
 }
 
+template <typename Flavor>
+UltraProver_<Flavor> construct_prover(acir_format::AcirProgram& program, const acir_format::ProgramMetadata& metadata)
+{
+    auto builder = acir_format::create_circuit<UltraCircuitBuilder>(program, metadata);
+
+    UltraProver_<Flavor> prover{ builder };
+    return prover;
+}
+
 WASM_EXPORT void acir_prove_ultra_honk(uint8_t const* acir_vec,
                                        bool const* recursive,
                                        uint8_t const* witness_vec,
@@ -323,9 +332,7 @@ WASM_EXPORT void acir_prove_ultra_honk(uint8_t const* acir_vec,
         acir_format::witness_buf_to_witness_data(from_buffer<std::vector<uint8_t>>(witness_vec))
     };
 
-    auto builder = acir_format::create_circuit<UltraCircuitBuilder>(program, metadata);
-
-    UltraProver prover{ builder };
+    UltraProver prover = construct_prover<UltraFlavor>(program, metadata);
     auto proof = prover.construct_proof();
     *out = to_heap_buffer(to_buffer</*include_size=*/true>(proof));
 }
@@ -335,14 +342,14 @@ WASM_EXPORT void acir_prove_ultra_keccak_honk(uint8_t const* acir_vec,
                                               uint8_t const* witness_vec,
                                               uint8_t** out)
 {
-    auto constraint_system =
-        acir_format::circuit_buf_to_acir_format(from_buffer<std::vector<uint8_t>>(acir_vec), /*honk_recursion=*/1);
-    auto witness = acir_format::witness_buf_to_witness_data(from_buffer<std::vector<uint8_t>>(witness_vec));
+    const acir_format::ProgramMetadata metadata{ .recursive = *recursive, .honk_recursion = 1 };
 
-    auto builder = acir_format::create_circuit<UltraCircuitBuilder>(
-        constraint_system, *recursive, 0, witness, /*honk_recursion=*/1);
+    acir_format::AcirProgram program{
+        acir_format::circuit_buf_to_acir_format(from_buffer<std::vector<uint8_t>>(acir_vec), metadata.honk_recursion),
+        acir_format::witness_buf_to_witness_data(from_buffer<std::vector<uint8_t>>(witness_vec))
+    };
 
-    UltraKeccakProver prover{ builder };
+    UltraKeccakProver prover = construct_prover<UltraKeccakFlavor>(program, metadata);
     auto proof = prover.construct_proof();
     *out = to_heap_buffer(to_buffer</*include_size=*/true>(proof));
 }
@@ -377,6 +384,7 @@ WASM_EXPORT void acir_verify_ultra_keccak_honk(uint8_t const* proof_buf, uint8_t
     *result = verifier.verify_proof(proof);
 }
 
+// WORKTODO: fix memory for write_vk UH flows
 WASM_EXPORT void acir_write_vk_ultra_honk(uint8_t const* acir_vec, bool const* recursive, uint8_t** out)
 {
     using DeciderProvingKey = DeciderProvingKey_<UltraFlavor>;

barretenberg/cpp/src/barretenberg/eccvm/eccvm_transcript.test.cpp

@@ -278,13 +278,16 @@ TEST_F(ECCVMTranscriptTests, ProverManifestConsistency)
 
     // Automatically generate a transcript manifest by constructing a proof
     ECCVMProver prover(builder);
+    prover.transcript->enable_manifest();
+    prover.ipa_transcript->enable_manifest();
     ECCVMProof proof = prover.construct_proof();
 
     // Check that the prover generated manifest agrees with the manifest hard coded in this suite
     auto manifest_expected = this->construct_eccvm_honk_manifest();
     auto prover_manifest = prover.transcript->get_manifest();
 
     // Note: a manifest can be printed using manifest.print()
+    ASSERT(manifest_expected.size() > 0);
     for (size_t round = 0; round < manifest_expected.size(); ++round) {
         ASSERT_EQ(prover_manifest[round], manifest_expected[round]) << "Prover manifest discrepency in round " << round;
     }
@@ -293,6 +296,7 @@ TEST_F(ECCVMTranscriptTests, ProverManifestConsistency)
     auto prover_ipa_manifest = prover.ipa_transcript->get_manifest();
 
     // Note: a manifest can be printed using manifest.print()
+    ASSERT(ipa_manifest_expected.size() > 0);
     for (size_t round = 0; round < ipa_manifest_expected.size(); ++round) {
         ASSERT_EQ(prover_ipa_manifest[round], ipa_manifest_expected[round])
             << "IPA prover manifest discrepency in round " << round;
@@ -311,6 +315,8 @@ TEST_F(ECCVMTranscriptTests, VerifierManifestConsistency)
 
     // Automatically generate a transcript manifest in the prover by constructing a proof
     ECCVMProver prover(builder);
+    prover.transcript->enable_manifest();
+    prover.ipa_transcript->enable_manifest();
     ECCVMProof proof = prover.construct_proof();
 
     // Automatically generate a transcript manifest in the verifier by verifying a proof
@@ -325,10 +331,20 @@ TEST_F(ECCVMTranscriptTests, VerifierManifestConsistency)
     // The last challenge generated by the ECCVM Prover is the translation univariate batching challenge and, on the
     // verifier side, is only generated in the translator verifier hence the ECCVM prover's manifest will have one extra
     // challenge
+    ASSERT(prover_manifest.size() > 0);
     for (size_t round = 0; round < prover_manifest.size() - 1; ++round) {
         ASSERT_EQ(prover_manifest[round], verifier_manifest[round])
             << "Prover/Verifier manifest discrepency in round " << round;
     }
+
+    // Check consistency of IPA transcripts
+    auto prover_ipa_manifest = prover.ipa_transcript->get_manifest();
+    auto verifier_ipa_manifest = verifier.ipa_transcript->get_manifest();
+    ASSERT(prover_ipa_manifest.size() > 0);
+    for (size_t round = 0; round < prover_ipa_manifest.size(); ++round) {
+        ASSERT_EQ(prover_ipa_manifest[round], verifier_ipa_manifest[round])
+            << "Prover/Verifier IPA manifest discrepency in round " << round;
+    }
 }
 
 /**

barretenberg/cpp/src/barretenberg/eccvm/eccvm_verifier.cpp

@@ -20,6 +20,9 @@ bool ECCVMVerifier::verify_proof(const ECCVMProof& proof)
     RelationParameters<FF> relation_parameters;
     transcript = std::make_shared<Transcript>(proof.pre_ipa_proof);
     ipa_transcript = std::make_shared<Transcript>(proof.ipa_proof);
+    transcript->enable_manifest();
+    ipa_transcript->enable_manifest();
+
     VerifierCommitments commitments{ key };
     CommitmentLabels commitment_labels;
 

barretenberg/cpp/src/barretenberg/protogalaxy/protogalaxy_verifier.cpp

@@ -19,6 +19,7 @@ void ProtogalaxyVerifier_<DeciderVerificationKeys>::run_oink_verifier_on_each_in
     const std::vector<FF>& proof)
 {
     transcript = std::make_shared<Transcript>(proof);
+    transcript->enable_manifest();
     size_t index = 0;
     auto key = keys_to_fold[0];
     auto domain_separator = std::to_string(index);

barretenberg/cpp/src/barretenberg/stdlib/eccvm_verifier/eccvm_recursive_verifier.cpp

@@ -34,6 +34,8 @@ ECCVMRecursiveVerifier_<Flavor>::verify_proof(const ECCVMProof& proof)
     StdlibProof<Builder> stdlib_ipa_proof = bb::convert_native_proof_to_stdlib(builder, proof.ipa_proof);
     transcript = std::make_shared<Transcript>(stdlib_proof);
     ipa_transcript = std::make_shared<Transcript>(stdlib_ipa_proof);
+    transcript->enable_manifest();
+    ipa_transcript->enable_manifest();
 
     VerifierCommitments commitments{ key };
     CommitmentLabels commitment_labels;

barretenberg/cpp/src/barretenberg/stdlib/eccvm_verifier/eccvm_recursive_verifier.test.cpp

@@ -102,6 +102,8 @@ template <typename RecursiveFlavor> class ECCVMRecursiveTests : public ::testing
         EXPECT_TRUE(native_result);
         auto recursive_manifest = verifier.transcript->get_manifest();
         auto native_manifest = native_verifier.transcript->get_manifest();
+
+        EXPECT(recursive_manifest.size() > 0);
         for (size_t i = 0; i < recursive_manifest.size(); ++i) {
             EXPECT_EQ(recursive_manifest[i], native_manifest[i])
                 << "Recursive Verifier/Verifier manifest discrepency in round " << i;

barretenberg/cpp/src/barretenberg/stdlib/protogalaxy_verifier/protogalaxy_recursive_verifier.cpp

@@ -19,6 +19,7 @@ void ProtogalaxyRecursiveVerifier_<DeciderVerificationKeys>::run_oink_verifier_o
     const std::vector<FF>& proof)
 {
     transcript = std::make_shared<Transcript>(proof);
+    transcript->enable_manifest();
     size_t index = 0;
     auto key = keys_to_fold[0];
     auto domain_separator = std::to_string(index);

barretenberg/cpp/src/barretenberg/stdlib/protogalaxy_verifier/protogalaxy_recursive_verifier.test.cpp

@@ -238,6 +238,7 @@ template <typename RecursiveFlavor> class ProtogalaxyRecursiveTests : public tes
         auto recursive_folding_manifest = verifier.transcript->get_manifest();
         auto native_folding_manifest = native_folding_verifier.transcript->get_manifest();
 
+        ASSERT(recursive_folding_manifest.size() > 0);
         for (size_t i = 0; i < recursive_folding_manifest.size(); ++i) {
             EXPECT_EQ(recursive_folding_manifest[i], native_folding_manifest[i])
                 << "Recursive Verifier/Verifier manifest discrepency in round " << i;
@@ -316,6 +317,7 @@ template <typename RecursiveFlavor> class ProtogalaxyRecursiveTests : public tes
         auto recursive_folding_manifest = verifier.transcript->get_manifest();
         auto native_folding_manifest = native_folding_verifier.transcript->get_manifest();
 
+        ASSERT(recursive_folding_manifest.size() > 0);
         for (size_t i = 0; i < recursive_folding_manifest.size(); ++i) {
             EXPECT_EQ(recursive_folding_manifest[i], native_folding_manifest[i])
                 << "Recursive Verifier/Verifier manifest discrepency in round " << i;

barretenberg/cpp/src/barretenberg/trace_to_polynomials/trace_to_polynomials.cpp

@@ -55,9 +55,11 @@ void TraceToPolynomials<Flavor>::add_memory_records_to_proving_key(TraceData& tr
     ASSERT(proving_key.memory_read_records.empty() && proving_key.memory_write_records.empty());
 
     // Update indices of RAM/ROM reads/writes based on where block containing these gates sits in the trace
+    proving_key.memory_read_records.reserve(builder.memory_read_records.size());
     for (auto& index : builder.memory_read_records) {
         proving_key.memory_read_records.emplace_back(index + trace_data.ram_rom_offset);
     }
+    proving_key.memory_write_records.reserve(builder.memory_write_records.size());
     for (auto& index : builder.memory_write_records) {
         proving_key.memory_write_records.emplace_back(index + trace_data.ram_rom_offset);
     }

barretenberg/cpp/src/barretenberg/transcript/transcript.hpp

@@ -48,7 +48,8 @@ class TranscriptManifest {
         }
     }
 
-    template <typename... Strings> void add_challenge(size_t round, Strings&... labels)
+    template <typename... Strings>
+    void add_challenge([[maybe_unused]] size_t round, [[maybe_unused]] Strings&... labels)
     {
         manifest[round].challenge_label = { labels... };
     }
@@ -65,7 +66,9 @@ class TranscriptManifest {
         call_add_challenge();
     }
 
-    void add_entry(size_t round, const std::string& element_label, size_t element_size)
+    void add_entry([[maybe_unused]] size_t round,
+                   [[maybe_unused]] const std::string& element_label,
+                   [[maybe_unused]] size_t element_size)
     {
         manifest[round].entries.emplace_back(element_label, element_size);
     }
@@ -150,6 +153,8 @@ template <typename TranscriptParams> class BaseTranscript {
     Fr previous_challenge{};        // default-initialized to zeros
     std::vector<Fr> current_round_data;
 
+    bool use_manifest = false; // indicates whether the manifest is turned on, currently only on for manifest tests.
+
     // "Manifest" object that records a summary of the transcript interactions
     TranscriptManifest manifest;
 
@@ -211,8 +216,10 @@ template <typename TranscriptParams> class BaseTranscript {
      */
     void consume_prover_element_frs(const std::string& label, std::span<const Fr> element_frs)
     {
-        // Add an entry to the current round of the manifest
-        manifest.add_entry(round_number, label, element_frs.size());
+        if (use_manifest) {
+            // Add an entry to the current round of the manifest
+            manifest.add_entry(round_number, label, element_frs.size());
+        }
 
         current_round_data.insert(current_round_data.end(), element_frs.begin(), element_frs.end());
 
@@ -276,6 +283,12 @@ template <typename TranscriptParams> class BaseTranscript {
         std::copy(proof.begin(), proof.end(), std::back_inserter(proof_data));
     }
 
+    /**
+     * @brief Enables the manifest
+     *
+     */
+    void enable_manifest() { use_manifest = true; }
+
     /**
      * @brief After all the prover messages have been sent, finalize the round by hashing all the data and then
      * create the number of requested challenges.
@@ -290,8 +303,10 @@ template <typename TranscriptParams> class BaseTranscript {
     {
         constexpr size_t num_challenges = sizeof...(Strings);
 
-        // Add challenge labels for current round to the manifest
-        manifest.add_challenge(round_number, labels...);
+        if (use_manifest) {
+            // Add challenge labels for current round to the manifest
+            manifest.add_challenge(round_number, labels...);
+        }
 
         // Compute the new challenge buffer from which we derive the challenges.
 
@@ -438,7 +453,13 @@ template <typename TranscriptParams> class BaseTranscript {
 
     [[nodiscard]] TranscriptManifest get_manifest() const { return manifest; };
 
-    void print() { manifest.print(); }
+    void print()
+    {
+        if (!use_manifest) {
+            info("Warning: manifest is not enabled!");
+        }
+        manifest.print();
+    }
 };
 
 template <typename Builder>

barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp

@@ -93,7 +93,7 @@ template <IsUltraFlavor Flavor> void DeciderProvingKey_<Flavor>::allocate_select
 template <IsUltraFlavor Flavor>
 void DeciderProvingKey_<Flavor>::allocate_table_lookup_polynomials(const Circuit& circuit)
 {
-    PROFILE_THIS_NAME("allocate_table_lookup_polynomials_and_inverses");
+    PROFILE_THIS_NAME("allocate_table_lookup_and_lookup_read_polynomials");
 
     size_t table_offset = circuit.blocks.lookup.trace_offset;
     // TODO(https://github.com/AztecProtocol/barretenberg/issues/1193): can potentially improve memory footprint
@@ -143,6 +143,7 @@ template <IsUltraFlavor Flavor>
 void DeciderProvingKey_<Flavor>::allocate_databus_polynomials(const Circuit& circuit)
     requires HasDataBus<Flavor>
 {
+    PROFILE_THIS_NAME("allocate_databus_and_lookup_inverse_polynomials");
     proving_key.polynomials.calldata = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size);
     proving_key.polynomials.calldata_read_counts = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size);
     proving_key.polynomials.calldata_read_tags = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size);

barretenberg/cpp/src/barretenberg/ultra_honk/mega_transcript.test.cpp

@@ -193,12 +193,14 @@ TYPED_TEST(MegaTranscriptTests, ProverManifestConsistency)
     // Automatically generate a transcript manifest by constructing a proof
     auto proving_key = std::make_shared<DeciderProvingKey>(builder);
     Prover prover(proving_key);
+    prover.transcript->enable_manifest();
     auto proof = prover.construct_proof();
 
     // Check that the prover generated manifest agrees with the manifest hard coded in this suite
     auto manifest_expected = TestFixture::construct_mega_honk_manifest();
     auto prover_manifest = prover.transcript->get_manifest();
     // Note: a manifest can be printed using manifest.print()
+    ASSERT(manifest_expected.size() > 0);
     for (size_t round = 0; round < manifest_expected.size(); ++round) {
         if (prover_manifest[round] != manifest_expected[round]) {
             info("Prover manifest discrepency in round ", round);
@@ -229,6 +231,7 @@ TYPED_TEST(MegaTranscriptTests, VerifierManifestConsistency)
     // Automatically generate a transcript manifest in the prover by constructing a proof
     auto proving_key = std::make_shared<DeciderProvingKey>(builder);
     Prover prover(proving_key);
+    prover.transcript->enable_manifest();
     auto proof = prover.construct_proof();
 
     // Automatically generate a transcript manifest in the verifier by verifying a proof
@@ -242,6 +245,7 @@ TYPED_TEST(MegaTranscriptTests, VerifierManifestConsistency)
     auto verifier_manifest = verifier.transcript->get_manifest();
 
     // Note: a manifest can be printed using manifest.print()
+    ASSERT(prover_manifest.size() > 0);
     for (size_t round = 0; round < prover_manifest.size(); ++round) {
         if (prover_manifest[round] != verifier_manifest[round]) {
             info("Prover/Verifier manifest discrepency in round ", round);

barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp

@@ -59,10 +59,10 @@ template <IsUltraFlavor Flavor> void OinkProver<Flavor>::prove()
     // Generate relation separators alphas for sumcheck/combiner computation
     proving_key->alphas = generate_alphas_round();
 
-#ifndef __wasm__
+    // #ifndef __wasm__
     // Free the commitment key
     proving_key->proving_key.commitment_key = nullptr;
-#endif
+    // #endif
 }
 
 /**

barretenberg/cpp/src/barretenberg/ultra_honk/ultra_transcript.test.cpp

@@ -200,6 +200,7 @@ TYPED_TEST(UltraTranscriptTests, ProverManifestConsistency)
     // Automatically generate a transcript manifest by constructing a proof
     auto proving_key = std::make_shared<typename TestFixture::DeciderProvingKey>(builder);
     typename TestFixture::Prover prover(proving_key);
+    prover.transcript->enable_manifest();
     auto proof = prover.construct_proof();
 
     // Check that the prover generated manifest agrees with the manifest hard coded in this suite
@@ -208,6 +209,7 @@ TYPED_TEST(UltraTranscriptTests, ProverManifestConsistency)
     // Note: a manifest can be printed using manifest.print()
     manifest_expected.print();
     prover_manifest.print();
+    ASSERT(manifest_expected.size() > 0);
     for (size_t round = 0; round < manifest_expected.size(); ++round) {
         ASSERT_EQ(prover_manifest[round], manifest_expected[round]) << "Prover manifest discrepency in round " << round;
     }
@@ -228,6 +230,7 @@ TYPED_TEST(UltraTranscriptTests, VerifierManifestConsistency)
     // Automatically generate a transcript manifest in the prover by constructing a proof
     auto proving_key = std::make_shared<typename TestFixture::DeciderProvingKey>(builder);
     typename TestFixture::Prover prover(proving_key);
+    prover.transcript->enable_manifest();
     auto proof = prover.construct_proof();
 
     // Automatically generate a transcript manifest in the verifier by verifying a proof
@@ -258,6 +261,7 @@ TYPED_TEST(UltraTranscriptTests, VerifierManifestConsistency)
     auto verifier_manifest = verifier.transcript->get_manifest();
 
     // Note: a manifest can be printed using manifest.print()
+    ASSERT(prover_manifest.size() > 0);
     for (size_t round = 0; round < prover_manifest.size(); ++round) {
         ASSERT_EQ(prover_manifest[round], verifier_manifest[round])
             << "Prover/Verifier manifest discrepency in round " << round;

barretenberg/cpp/src/barretenberg/ultra_honk/ultra_verifier.cpp

@@ -15,6 +15,7 @@ template <typename Flavor> bool UltraVerifier_<Flavor>::verify_proof(const HonkP
     using FF = typename Flavor::FF;
 
     transcript = std::make_shared<Transcript>(proof);
+    transcript->enable_manifest(); // Enable manifest for the verifier.
     OinkVerifier<Flavor> oink_verifier{ verification_key, transcript };
     oink_verifier.verify();
 
@@ -58,7 +59,8 @@ template <typename Flavor> bool UltraVerifier_<Flavor>::verify_proof(const HonkP
             };
 
             // verify the ipa_proof with this claim
-            auto ipa_transcript = std::make_shared<Transcript>(ipa_proof);
+            ipa_transcript = std::make_shared<Transcript>(ipa_proof);
+            ipa_transcript->enable_manifest(); // Enable manifest for the verifier.
             bool ipa_result = IPA<curve::Grumpkin>::reduce_verify(ipa_verification_key, ipa_claim, ipa_transcript);
             if (!ipa_result) {
                 return false;