feat(perf): speed up construction of bbjs Frs & cache zero hashes in …

…ephemeral trees (#11851)

Improves time for tx-hash computation from ~26-35ms down to ~20-23ms.
This is because we were doing slower conversion of 1800+ Uint8Arrays to
Frs. Allowing them to be actual Buffers lets us use
`read/writeBigUint64` functions which are faster than the per-byte
shifting we were doing.

Cache zero-hashes in static member of ephemeral tree class. Brings
Ephemeral forest construction from 20ms down to 10ms.

barretenberg/ts/src/bigint-array/index.ts

@@ -10,12 +10,20 @@ export function toBigIntBE(bytes: Uint8Array) {
   return bigint;
 }
 
-export function toBufferBE(value: bigint, byteLength = 32) {
-  const bytes = new Uint8Array(byteLength);
-  const view = new DataView(bytes.buffer);
-  for (let i = 0; i < byteLength; i++) {
-    view.setUint8(byteLength - i - 1, Number(value & BigInt(0xff)));
-    value >>= BigInt(8);
-  }
-  return bytes;
+export function bufToBigIntBE(buf: Buffer) {
+  return (
+    (buf.readBigInt64BE(0) << 192n) +
+    (buf.readBigInt64BE(8) << 128n) +
+    (buf.readBigInt64BE(16) << 64n) +
+    buf.readBigInt64BE(24)
+  );
+}
+
+export function toBufferBE(value: bigint, byteLength = 32): Buffer {
+  const buf = Buffer.alloc(byteLength);
+  buf.writeBigUInt64BE(value >> 192n, 0);
+  buf.writeBigUInt64BE((value >> 128n) & 0xffffffffffffffffn, 8);
+  buf.writeBigUInt64BE((value >> 64n) & 0xffffffffffffffffn, 16);
+  buf.writeBigUInt64BE(value & 0xffffffffffffffffn, 24);
+  return buf;
 }

barretenberg/ts/src/types/fields.ts

@@ -1,5 +1,5 @@
 import { randomBytes } from '../random/index.js';
-import { toBigIntBE, toBufferBE } from '../bigint-array/index.js';
+import { bufToBigIntBE, toBigIntBE, toBufferBE } from '../bigint-array/index.js';
 import { BufferReader, uint8ArrayToHexString } from '../serialize/index.js';
 
 // TODO(#4189): Replace with implementation in yarn-project/foundation/src/fields/fields.ts
@@ -15,9 +15,10 @@ export class Fr {
   static SIZE_IN_BYTES = 32;
   value: Uint8Array;
 
-  constructor(value: Uint8Array | bigint) {
+  constructor(value: Uint8Array | bigint | Buffer) {
     // We convert buffer value to bigint to be able to check it fits within modulus
-    const valueBigInt = typeof value === 'bigint' ? value : toBigIntBE(value);
+    const valueBigInt =
+      typeof value === 'bigint' ? value : value instanceof Buffer ? bufToBigIntBE(value) : toBigIntBE(value);
 
     if (valueBigInt > Fr.MAX_VALUE) {
       throw new Error(`Value 0x${valueBigInt.toString(16)} is greater or equal to field modulus.`);

yarn-project/simulator/src/avm/avm_simulator.ts

@@ -205,7 +205,7 @@ export class AvmSimulator {
 
       const endTotalTime = performance.now();
       const totalTime = endTotalTime - startTotalTime;
-      this.log.debug(`Total execution time: ${totalTime}ms`);
+      this.log.debug(`Core AVM simulation took ${totalTime}ms`);
 
       // Return results for processing by calling context
       return results;

yarn-project/simulator/src/avm/avm_tree.ts

@@ -7,6 +7,21 @@ import { type IndexedTreeLeafPreimage, type TreeLeafPreimage } from '@aztec/foun
 import { strict as assert } from 'assert';
 import cloneDeep from 'lodash.clonedeep';
 
+const MAX_TREE_DEPTH = 128;
+
+/**
+ * Helper function to precompute zero hashes
+ */
+async function preComputeZeroHashes(): Promise<Fr[]> {
+  let currentHash = Fr.zero();
+  const zeroHashes: Fr[] = [];
+  for (let i = 0; i < MAX_TREE_DEPTH; i++) {
+    zeroHashes.push(currentHash);
+    currentHash = await poseidon2Hash([currentHash, currentHash]);
+  }
+  return zeroHashes;
+}
+
 /****************************************************/
 /****** Structs Used by the AvmEphemeralForest ******/
 /****************************************************/
@@ -554,6 +569,8 @@ export class EphemeralAvmTree {
   private tree: Tree;
   public frontier: Fr[];
 
+  private static precomputedZeroHashes: Fr[] | undefined;
+
   private constructor(private root: Leaf, public leafCount: bigint, public depth: number, private zeroHashes: Fr[]) {
     this.tree = root;
     this.frontier = [];
@@ -565,13 +582,12 @@ export class EphemeralAvmTree {
     treeDb: MerkleTreeReadOperations,
     merkleId: MerkleTreeId,
   ): Promise<EphemeralAvmTree> {
-    let zeroHash = Fr.zero();
-    // Can probably cache this elsewhere
-    const zeroHashes = [];
-    for (let i = 0; i < depth; i++) {
-      zeroHashes.push(zeroHash);
-      zeroHash = await poseidon2Hash([zeroHash, zeroHash]);
+    let zeroHashes = EphemeralAvmTree.precomputedZeroHashes;
+    if (zeroHashes === undefined) {
+      zeroHashes = await preComputeZeroHashes();
+      EphemeralAvmTree.precomputedZeroHashes = zeroHashes;
     }
+    const zeroHash = zeroHashes[depth];
     const tree = new EphemeralAvmTree(Leaf(zeroHash), forkedLeafCount, depth, zeroHashes);
     await tree.initializeFrontier(treeDb, merkleId);
     return tree;

yarn-project/simulator/src/avm/journal/journal.ts

@@ -144,16 +144,16 @@ export class AvmPersistableStateManager {
    * @param value - the value being written to the slot
    */
   public async writeStorage(contractAddress: AztecAddress, slot: Fr, value: Fr, protocolWrite = false): Promise<void> {
-    this.log.debug(`Storage write (address=${contractAddress}, slot=${slot}): value=${value}`);
+    this.log.trace(`Storage write (address=${contractAddress}, slot=${slot}): value=${value}`);
     const leafSlot = await computePublicDataTreeLeafSlot(contractAddress, slot);
-    this.log.debug(`leafSlot=${leafSlot}`);
+    this.log.trace(`leafSlot=${leafSlot}`);
     // Cache storage writes for later reference/reads
     this.publicStorage.write(contractAddress, slot, value);
 
     if (this.doMerkleOperations) {
       const result = await this.merkleTrees.writePublicStorage(leafSlot, value);
       assert(result !== undefined, 'Public data tree insertion error. You might want to disable doMerkleOperations.');
-      this.log.debug(`Inserted public data tree leaf at leafSlot ${leafSlot}, value: ${value}`);
+      this.log.trace(`Inserted public data tree leaf at leafSlot ${leafSlot}, value: ${value}`);
 
       const lowLeafInfo = result.lowWitness;
       const lowLeafPreimage = result.lowWitness.preimage as PublicDataTreeLeafPreimage;
@@ -195,9 +195,9 @@ export class AvmPersistableStateManager {
    */
   public async readStorage(contractAddress: AztecAddress, slot: Fr): Promise<Fr> {
     const { value, cached } = await this.publicStorage.read(contractAddress, slot);
-    this.log.debug(`Storage read  (address=${contractAddress}, slot=${slot}): value=${value}, cached=${cached}`);
+    this.log.trace(`Storage read  (address=${contractAddress}, slot=${slot}): value=${value}, cached=${cached}`);
     const leafSlot = await computePublicDataTreeLeafSlot(contractAddress, slot);
-    this.log.debug(`leafSlot=${leafSlot}`);
+    this.log.trace(`leafSlot=${leafSlot}`);
 
     if (this.doMerkleOperations) {
       // Get leaf if present, low leaf if absent
@@ -212,8 +212,8 @@ export class AvmPersistableStateManager {
       const leafPath = await this.merkleTrees.getSiblingPath(MerkleTreeId.PUBLIC_DATA_TREE, leafIndex);
       const leafPreimage = preimage as PublicDataTreeLeafPreimage;
 
-      this.log.debug(`leafPreimage.slot: ${leafPreimage.slot}, leafPreimage.value: ${leafPreimage.value}`);
-      this.log.debug(
+      this.log.trace(`leafPreimage.slot: ${leafPreimage.slot}, leafPreimage.value: ${leafPreimage.value}`);
+      this.log.trace(
         `leafPreimage.nextSlot: ${leafPreimage.nextSlot}, leafPreimage.nextIndex: ${Number(leafPreimage.nextIndex)}`,
       );
 
@@ -223,7 +223,7 @@ export class AvmPersistableStateManager {
           `Value mismatch when performing public data read (got value: ${value}, value in ephemeral tree: ${leafPreimage.value})`,
         );
       } else {
-        this.log.debug(`Slot has never been written before!`);
+        this.log.trace(`Slot has never been written before!`);
         // Sanity check that the leaf slot is skipped by low leaf when it doesn't exist
         assert(
           leafPreimage.slot.toBigInt() < leafSlot.toBigInt() &&
@@ -250,7 +250,7 @@ export class AvmPersistableStateManager {
    */
   public async peekStorage(contractAddress: AztecAddress, slot: Fr): Promise<Fr> {
     const { value, cached } = await this.publicStorage.read(contractAddress, slot);
-    this.log.debug(`Storage peek  (address=${contractAddress}, slot=${slot}): value=${value},  cached=${cached}`);
+    this.log.trace(`Storage peek  (address=${contractAddress}, slot=${slot}): value=${value},  cached=${cached}`);
     return Promise.resolve(value);
   }
 
@@ -266,7 +266,7 @@ export class AvmPersistableStateManager {
   public async checkNoteHashExists(contractAddress: AztecAddress, noteHash: Fr, leafIndex: Fr): Promise<boolean> {
     const gotLeafValue = (await this.worldStateDB.getCommitmentValue(leafIndex.toBigInt())) ?? Fr.ZERO;
     const exists = gotLeafValue.equals(noteHash);
-    this.log.debug(
+    this.log.trace(
       `noteHashes(${contractAddress})@${noteHash} ?? leafIndex: ${leafIndex} | gotLeafValue: ${gotLeafValue}, exists: ${exists}.`,
     );
     if (this.doMerkleOperations) {
@@ -306,7 +306,7 @@ export class AvmPersistableStateManager {
    * @param noteHash - the siloed unique hash to write
    */
   public async writeUniqueNoteHash(noteHash: Fr): Promise<void> {
-    this.log.debug(`noteHashes += @${noteHash}.`);
+    this.log.trace(`noteHashes += @${noteHash}.`);
 
     if (this.doMerkleOperations) {
       // Should write a helper for this
@@ -325,7 +325,7 @@ export class AvmPersistableStateManager {
    * @returns exists - whether the nullifier exists in the nullifier set
    */
   public async checkNullifierExists(contractAddress: AztecAddress, nullifier: Fr): Promise<boolean> {
-    this.log.debug(`Checking existence of nullifier (address=${contractAddress}, nullifier=${nullifier})`);
+    this.log.trace(`Checking existence of nullifier (address=${contractAddress}, nullifier=${nullifier})`);
     const siloedNullifier = await siloNullifier(contractAddress, nullifier);
     const [exists, leafOrLowLeafPreimage, leafOrLowLeafIndex, leafOrLowLeafPath] = await this.getNullifierMembership(
       siloedNullifier,
@@ -367,7 +367,7 @@ export class AvmPersistableStateManager {
     ]
   > {
     const [exists, isPending, _] = await this.nullifiers.checkExists(siloedNullifier);
-    this.log.debug(`Checked siloed nullifier ${siloedNullifier} (exists=${exists}), pending=${isPending}`);
+    this.log.trace(`Checked siloed nullifier ${siloedNullifier} (exists=${exists}), pending=${isPending}`);
 
     if (this.doMerkleOperations) {
       // Get leaf if present, low leaf if absent
@@ -386,7 +386,7 @@ export class AvmPersistableStateManager {
       );
 
       if (exists) {
-        this.log.debug(`Siloed nullifier ${siloedNullifier} exists at leafIndex=${leafIndex}`);
+        this.log.trace(`Siloed nullifier ${siloedNullifier} exists at leafIndex=${leafIndex}`);
       } else {
         // Sanity check that the leaf value is skipped by low leaf when it doesn't exist
         assert(
@@ -407,7 +407,7 @@ export class AvmPersistableStateManager {
    * @param nullifier - the unsiloed nullifier to write
    */
   public async writeNullifier(contractAddress: AztecAddress, nullifier: Fr) {
-    this.log.debug(`Inserting new nullifier (address=${nullifier}, nullifier=${contractAddress})`);
+    this.log.trace(`Inserting new nullifier (address=${nullifier}, nullifier=${contractAddress})`);
     const siloedNullifier = await siloNullifier(contractAddress, nullifier);
     await this.writeSiloedNullifier(siloedNullifier);
   }
@@ -417,7 +417,7 @@ export class AvmPersistableStateManager {
    * @param siloedNullifier - the siloed nullifier to write
    */
   public async writeSiloedNullifier(siloedNullifier: Fr) {
-    this.log.debug(`Inserting siloed nullifier=${siloedNullifier}`);
+    this.log.trace(`Inserting siloed nullifier=${siloedNullifier}`);
 
     if (this.doMerkleOperations) {
       // Maybe overkill, but we should check if the nullifier is already present in the tree before attempting to insert
@@ -447,13 +447,13 @@ export class AvmPersistableStateManager {
         // Cache pending nullifiers for later access
         await this.nullifiers.append(siloedNullifier);
         // We append the new nullifier
-        this.log.debug(
+        this.log.trace(
           `Nullifier tree root before insertion ${await this.merkleTrees.treeMap
             .get(MerkleTreeId.NULLIFIER_TREE)!
             .getRoot()}`,
         );
         const appendResult = await this.merkleTrees.appendNullifier(siloedNullifier);
-        this.log.debug(
+        this.log.trace(
           `Nullifier tree root after insertion ${await this.merkleTrees.treeMap
             .get(MerkleTreeId.NULLIFIER_TREE)!
             .getRoot()}`,
@@ -496,7 +496,7 @@ export class AvmPersistableStateManager {
   ): Promise<boolean> {
     const valueAtIndex = (await this.worldStateDB.getL1ToL2LeafValue(msgLeafIndex.toBigInt())) ?? Fr.ZERO;
     const exists = valueAtIndex.equals(msgHash);
-    this.log.debug(
+    this.log.trace(
       `l1ToL2Messages(@${msgLeafIndex}) ?? exists: ${exists}, expected: ${msgHash}, found: ${valueAtIndex}.`,
     );
 
@@ -522,7 +522,7 @@ export class AvmPersistableStateManager {
    * @param content - Message content.
    */
   public writeL2ToL1Message(contractAddress: AztecAddress, recipient: Fr, content: Fr) {
-    this.log.debug(`L2ToL1Messages(${contractAddress}) += (recipient: ${recipient}, content: ${content}).`);
+    this.log.trace(`L2ToL1Messages(${contractAddress}) += (recipient: ${recipient}, content: ${content}).`);
     this.trace.traceNewL2ToL1Message(contractAddress, recipient, content);
   }
 
@@ -532,7 +532,7 @@ export class AvmPersistableStateManager {
    * @param log - log contents
    */
   public writePublicLog(contractAddress: AztecAddress, log: Fr[]) {
-    this.log.debug(`PublicLog(${contractAddress}) += event with ${log.length} fields.`);
+    this.log.trace(`PublicLog(${contractAddress}) += event with ${log.length} fields.`);
     this.trace.tracePublicLog(contractAddress, log);
   }
 
@@ -542,7 +542,7 @@ export class AvmPersistableStateManager {
    * @returns the contract instance or undefined if it does not exist.
    */
   public async getContractInstance(contractAddress: AztecAddress): Promise<SerializableContractInstance | undefined> {
-    this.log.debug(`Getting contract instance for address ${contractAddress}`);
+    this.log.trace(`Getting contract instance for address ${contractAddress}`);
     const instanceWithAddress = await this.worldStateDB.getContractInstance(contractAddress);
     const exists = instanceWithAddress !== undefined;
 
@@ -568,7 +568,7 @@ export class AvmPersistableStateManager {
 
     if (exists) {
       const instance = new SerializableContractInstance(instanceWithAddress);
-      this.log.debug(
+      this.log.trace(
         `Got contract instance (address=${contractAddress}): exists=${exists}, instance=${jsonStringify(instance)}`,
       );
       if (this.doMerkleOperations) {

yarn-project/simulator/src/public/public_tx_context.ts

@@ -162,15 +162,15 @@ export class PublicTxContext {
    * NOTE: this does not "halt" the entire transaction execution.
    */
   revert(phase: TxExecutionPhase, revertReason: SimulationError | undefined = undefined, culprit = '') {
-    this.log.debug(`${TxExecutionPhase[phase]} phase reverted! ${culprit} failed with reason: ${revertReason}`);
+    this.log.warn(`${TxExecutionPhase[phase]} phase reverted! ${culprit} failed with reason: ${revertReason}`);
 
     if (revertReason && !this.revertReason) {
       // don't override revertReason
       // (if app logic and teardown both revert, we want app logic's reason)
       this.revertReason = revertReason;
     }
     if (phase === TxExecutionPhase.SETUP) {
-      this.log.debug(`Setup phase reverted! The transaction will be thrown out.`);
+      this.log.warn(`Setup phase reverted! The transaction will be thrown out.`);
       if (revertReason) {
         throw revertReason;
       } else {