Move to yarn 3 plug 'n play and workspaces, pure ESM modules. (#1599)

* Transferrables and webpack to build a worker.js until we have loader chaining... * Prod/dev conf * Update bb imports. * bb tests passing. * Integrate wasm back in bb.js. Passing tests. Reduce wasm memory footprint. * Can run halloumi and falafel. * Falafel tests pass. * Sdk tests pass. * e2e.test passes. * webpack * bb.js yarn 3.2.2 * fix mem align * fix bitwise warning on newer clang * Tweaks towards wasi 16 (more to figure out) * yarn 3.2.2 * remove encoding headers from block source client * bb.js updates * Blockchain test transpile to cjs for hardhat. * Fix blockchain. * SDK alternates proving keys. * Falafel/halloumi start_e2e scripts args. * Shared linting, formatting for bb.js and blockchain. * explorer import changes. build needs fixing. * end-to-end, falafel, halloumi linting * Hummus just a terminal. Linting. Version upgrades. * Linting. Bootstrapping. Wallet. * prettier config. * Basic working wallet. * Fix apollo server stuff in falafel... * account-migrator. explorer. * Cleanup JSON stuff. * wasabi. * Revert jest->mocha * Merge fixes. * Initial zk-money. * - Reconnect bootstrap scripts - set prettier as js default * - es-module style imports - upgrade typescript - copy wasm * zk-money builds. * Linting * Dockerfile and build stuff. * Move yarn project into own folder. * explorer build. min cci yml. * Introduce a build manifest to compute rebuild patterns. * Addition of a script for auto launching tmux and running a test. * Streamline bootstrapping. * Add bash to falafel image. * build image
AztecProtocol · Oct 10, 2022 · febe8f0 · febe8f0
1 parent d93f118
commit febe8f0
Show file tree

Hide file tree

Showing 18 changed files with 82 additions and 35 deletions.
diff --git a/bootstrap.sh b/bootstrap.sh
@@ -45,13 +45,14 @@ fi
 # Build native.
 mkdir -p build && cd build
 cmake -DCMAKE_BUILD_TYPE=RelWithAssert -DTOOLCHAIN=$TOOLCHAIN ..
-cmake --build . --parallel
+make -j$(nproc) $@
 cd ..
 
 # Install the webassembly toolchain.
-rm -rf ./src/wasi-sdk-12.0
+WASI_VERSION=12
+rm -rf ./src/wasi-sdk-*
 cd ./src
-curl -s -L https://github.com/CraneStation/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-$OS.tar.gz | tar zxfv -
+curl -s -L https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-$WASI_VERSION/wasi-sdk-$WASI_VERSION.0-$OS.tar.gz | tar zxfv -
 cd ..
 
 # Build WASM.

diff --git a/src/aztec/CMakeLists.txt b/src/aztec/CMakeLists.txt
@@ -40,6 +40,8 @@ if(BENCHMARKS)
 endif()
 
 if(WASM)
+    add_definitions(-D_WASI_EMULATED_PROCESS_CLOCKS=1)
+
     # Well, this is awkward. We can't build a wasm module by just linking to the libraries as that produces, nothing.
     # There are a couple of other ways to avoiding listing all the object files here and leveraging the dependency
     # tree, but they come with the problem that they will import the 'env' object files. We explicitly want to avoid
@@ -72,7 +74,7 @@ if(WASM)
     target_link_options(
         barretenberg.wasm
         PRIVATE
-        -nostartfiles -Wl,--no-entry -Wl,--export-dynamic -Wl,--import-memory -Wl,--allow-undefined -Wl,--stack-first -Wl,-z,stack-size=8388608
+        -nostartfiles -Wl,--no-entry -Wl,--export-dynamic -Wl,--import-memory -Wl,--allow-undefined -Wl,--stack-first -Wl,-z,stack-size=1048576
     )
 
 endif()
diff --git a/src/aztec/common/mem.hpp b/src/aztec/common/mem.hpp
@@ -25,7 +25,7 @@ inline void aligned_free(void* mem)
 #if defined(__linux__) || defined(__wasm__)
 inline void* protected_aligned_alloc(size_t alignment, size_t size)
 {
-    size += alignment - (size % alignment);
+    size += (size % alignment);
     void* t = 0;
     t = aligned_alloc(alignment, size);
     if (t == 0) {

diff --git a/src/aztec/crypto/pedersen/generator_data.cpp b/src/aztec/crypto/pedersen/generator_data.cpp
@@ -6,13 +6,13 @@ namespace {
 
 // The number of unique base points with default main index with precomputed ladders
 #ifdef __wasm__
-constexpr size_t num_default_generators = 64;
+constexpr size_t num_default_generators = 32;
 #else
 constexpr size_t num_default_generators = 2048;
 #endif
 
 constexpr size_t hash_indices_generator_offset = 2048;
-constexpr size_t num_hash_indices = 32;
+constexpr size_t num_hash_indices = 16;
 constexpr size_t num_generators_per_hash_index = 8;
 constexpr size_t num_indexed_generators = num_hash_indices * num_generators_per_hash_index;
 constexpr size_t size_of_generator_data_array = hash_indices_generator_offset + num_indexed_generators;

diff --git a/src/aztec/ecc/curves/bn254/scalar_multiplication/c_bind.cpp b/src/aztec/ecc/curves/bn254/scalar_multiplication/c_bind.cpp
@@ -1,26 +1,29 @@
-#include "c_bind.hpp"
 #include "./scalar_multiplication.hpp"
 #include "pippenger.hpp"
 #include <common/mem.hpp>
 #include <srs/io.hpp>
 
 using namespace barretenberg;
 
+#define WASM_EXPORT __attribute__((visibility("default")))
+
 extern "C" {
 
 WASM_EXPORT void* bbmalloc(size_t size)
 {
-    return aligned_alloc(64, size);
+    auto ptr = aligned_alloc(64, size);
+    return ptr;
 }
 
 WASM_EXPORT void bbfree(void* ptr)
 {
     aligned_free(ptr);
 }
 
-WASM_EXPORT void* new_pippenger(uint8_t* points, size_t num_points)
+WASM_EXPORT void* new_pippenger(g1::affine_element* points, size_t num_points)
 {
-    return new scalar_multiplication::Pippenger(points, num_points);
+    auto ptr = new scalar_multiplication::Pippenger(points, num_points);
+    return ptr;
 }
 
 WASM_EXPORT void delete_pippenger(void* pippenger)

diff --git a/src/aztec/ecc/curves/bn254/scalar_multiplication/c_bind.hpp b/src/aztec/ecc/curves/bn254/scalar_multiplication/c_bind.hpp
diff --git a/src/aztec/ecc/curves/bn254/scalar_multiplication/pippenger.cpp b/src/aztec/ecc/curves/bn254/scalar_multiplication/pippenger.cpp
@@ -4,6 +4,16 @@
 namespace barretenberg {
 namespace scalar_multiplication {
 
+Pippenger::Pippenger(g1::affine_element* points, size_t num_points)
+    : monomials_(points)
+    , num_points_(num_points)
+{
+    monomials_[0] = barretenberg::g1::affine_one;
+    io::byteswap(&monomials_[1], (num_points - 1) * 64);
+
+    scalar_multiplication::generate_pippenger_point_table(monomials_, monomials_, num_points);
+}
+
 Pippenger::Pippenger(uint8_t const* points, size_t num_points)
     : num_points_(num_points)
 {

diff --git a/src/aztec/ecc/curves/bn254/scalar_multiplication/pippenger.hpp b/src/aztec/ecc/curves/bn254/scalar_multiplication/pippenger.hpp
@@ -34,6 +34,13 @@ template <typename T> inline T* point_table_alloc(size_t num_points)
 
 class Pippenger {
   public:
+    /**
+     * Expects points to be buffer of size as per point_table_size().
+     * It expects the crs to start at points[1], and it fills in affine_one at points[0].
+     * The crs undergoes a byteswap, and then the point table is generated.
+     */
+    Pippenger(g1::affine_element* points, size_t num_points);
+
     Pippenger(uint8_t const* points, size_t num_points);
 
     Pippenger(std::string const& path, size_t num_points);

diff --git a/src/aztec/ecc/groups/element_impl.hpp b/src/aztec/ecc/groups/element_impl.hpp
@@ -154,7 +154,7 @@ constexpr void element<Fq, Fr, T>::self_mixed_add_or_sub(const affine_element<Fq
             return;
         }
     } else {
-        const bool edge_case_trigger = x.is_msb_set() | other.x.is_msb_set();
+        const bool edge_case_trigger = x.is_msb_set() || other.x.is_msb_set();
         if (edge_case_trigger) {
             if (x.is_msb_set()) {
                 conditional_negate_affine(other, *(affine_element<Fq, Fr, T>*)this, predicate);
@@ -246,7 +246,7 @@ constexpr element<Fq, Fr, T> element<Fq, Fr, T>::operator+=(const affine_element
             return *this;
         }
     } else {
-        const bool edge_case_trigger = x.is_msb_set() | other.x.is_msb_set();
+        const bool edge_case_trigger = x.is_msb_set() || other.x.is_msb_set();
         if (edge_case_trigger) {
             if (x.is_msb_set()) {
                 *this = { other.x, other.y, Fq::one() };

diff --git a/src/aztec/rollup/proofs/account/account.cpp b/src/aztec/rollup/proofs/account/account.cpp
@@ -173,6 +173,10 @@ void account_circuit(Composer& composer, account_tx const& tx)
 
 void init_proving_key(std::shared_ptr<waffle::ReferenceStringFactory> const& crs_factory, bool mock)
 {
+    if (proving_key) {
+        return;
+    }
+
     // Junk data required just to create proving key.
     account_tx tx;
     tx.account_public_key = grumpkin::g1::affine_one;
@@ -195,9 +199,15 @@ void init_proving_key(std::shared_ptr<waffle::ReferenceStringFactory> const& crs
 
 void init_proving_key(std::shared_ptr<waffle::ProverReferenceString> const& crs, waffle::proving_key_data&& pk_data)
 {
+    release_key();
     proving_key = std::make_shared<waffle::proving_key>(std::move(pk_data), crs);
 }
 
+void release_key()
+{
+    proving_key.reset();
+}
+
 void init_verification_key(std::shared_ptr<waffle::ReferenceStringFactory> const& crs_factory)
 {
     if (!proving_key) {

diff --git a/src/aztec/rollup/proofs/account/account.hpp b/src/aztec/rollup/proofs/account/account.hpp
@@ -14,6 +14,8 @@ void init_proving_key(std::shared_ptr<waffle::ReferenceStringFactory> const& crs
 
 void init_proving_key(std::shared_ptr<waffle::ProverReferenceString> const& crs, waffle::proving_key_data&& pk_data);
 
+void release_key();
+
 void init_verification_key(std::shared_ptr<waffle::ReferenceStringFactory> const& crs_factory);
 
 void init_verification_key(std::shared_ptr<waffle::VerifierMemReferenceString> const& crs,

diff --git a/src/aztec/rollup/proofs/account/account_tx.test.cpp b/src/aztec/rollup/proofs/account/account_tx.test.cpp
@@ -3,7 +3,6 @@
 #include "account.hpp"
 #include <common/streams.hpp>
 #include <crypto/schnorr/schnorr.hpp>
-#include <ecc/curves/bn254/scalar_multiplication/c_bind.hpp>
 #include <fstream>
 #include <gtest/gtest.h>
 #include <plonk/reference_string/pippenger_reference_string.hpp>

diff --git a/src/aztec/rollup/proofs/account/c_bind.cpp b/src/aztec/rollup/proofs/account/c_bind.cpp
@@ -26,6 +26,11 @@ WASM_EXPORT void account__init_proving_key(bool mock)
     init_proving_key(crs_factory, mock);
 }
 
+WASM_EXPORT void account__release_key()
+{
+    release_key();
+}
+
 WASM_EXPORT void account__init_proving_key_from_buffer(uint8_t const* pk_buf)
 {
     std::shared_ptr<waffle::ProverReferenceString> crs;

diff --git a/src/aztec/rollup/proofs/join_split/c_bind.cpp b/src/aztec/rollup/proofs/join_split/c_bind.cpp
@@ -35,6 +35,11 @@ WASM_EXPORT void join_split__init_proving_key_from_buffer(uint8_t const* pk_buf)
     init_proving_key(crs, std::move(pk_data));
 }
 
+WASM_EXPORT void join_split__release_key()
+{
+    release_key();
+}
+
 WASM_EXPORT uint32_t join_split__get_new_proving_key_data(uint8_t** output)
 {
     // Computing the size of the serialized key is non trivial. We know it's ~331mb.

diff --git a/src/aztec/rollup/proofs/join_split/join_split.cpp b/src/aztec/rollup/proofs/join_split/join_split.cpp
@@ -16,6 +16,10 @@ static std::shared_ptr<waffle::verification_key> verification_key;
 
 void init_proving_key(std::shared_ptr<waffle::ReferenceStringFactory> const& crs_factory, bool mock)
 {
+    if (proving_key) {
+        return;
+    }
+
     // Junk data required just to create proving key.
     join_split_tx tx = noop_tx();
 
@@ -34,9 +38,15 @@ void init_proving_key(std::shared_ptr<waffle::ReferenceStringFactory> const& crs
 
 void init_proving_key(std::shared_ptr<waffle::ProverReferenceString> const& crs, waffle::proving_key_data&& pk_data)
 {
+    release_key();
     proving_key = std::make_shared<waffle::proving_key>(std::move(pk_data), crs);
 }
 
+void release_key()
+{
+    proving_key.reset();
+}
+
 void init_verification_key(std::unique_ptr<waffle::ReferenceStringFactory>&& crs_factory)
 {
     if (!proving_key) {

diff --git a/src/aztec/rollup/proofs/join_split/join_split.hpp b/src/aztec/rollup/proofs/join_split/join_split.hpp
@@ -14,6 +14,8 @@ void init_proving_key(std::shared_ptr<waffle::ReferenceStringFactory> const& crs
 
 void init_proving_key(std::shared_ptr<waffle::ProverReferenceString> const& crs, waffle::proving_key_data&& pk_data);
 
+void release_key();
+
 void init_verification_key(std::unique_ptr<waffle::ReferenceStringFactory>&& crs_factory);
 
 void init_verification_key(std::shared_ptr<waffle::VerifierMemReferenceString> const& crs,

diff --git a/src/aztec/rollup/proofs/join_split/join_split.test.cpp b/src/aztec/rollup/proofs/join_split/join_split.test.cpp
@@ -4,6 +4,7 @@
 #include "../notes/native/index.hpp"
 #include <common/streams.hpp>
 #include <common/test.hpp>
+#include <plonk/proof_system/proving_key/serialize.hpp>
 #include <stdlib/merkle_tree/index.hpp>
 
 namespace rollup {
@@ -2472,6 +2473,17 @@ TEST_F(join_split_tests, test_send_two_virtual_notes_full_proof)
     EXPECT_TRUE(verify_proof(proof));
 }
 
+// *************************************************************************************************************
+// Miscellaneous
+// *************************************************************************************************************
+
+TEST_F(join_split_tests, serialzed_proving_key_size)
+{
+    uint8_t* ptr;
+    auto len = join_split__get_new_proving_key_data(&ptr);
+    EXPECT_LE(len, 170 * 1024 * 1024);
+}
+
 } // namespace join_split
 } // namespace proofs
 } // namespace rollup
diff --git a/src/aztec/rollup/proofs/join_split/join_split_tx.test.cpp b/src/aztec/rollup/proofs/join_split/join_split_tx.test.cpp
@@ -5,7 +5,6 @@
 
 #include <common/streams.hpp>
 #include <crypto/schnorr/schnorr.hpp>
-#include <ecc/curves/bn254/scalar_multiplication/c_bind.hpp>
 #include <numeric/random/engine.hpp>
 #include <plonk/reference_string/pippenger_reference_string.hpp>
 #include <srs/io.hpp>