feat: add wasm memory benchmark to bench.json (redo)

barretenberg/.gitignore

@@ -6,4 +6,5 @@ ts/dest
 .tsbuildinfo
 .idea
 cmake-build-debug
-*_opt.pil
+*_opt.pil
+bench-out

barretenberg/acir_tests/.gitignore

@@ -2,4 +2,5 @@ acir_tests
 crs
 **/.yarn/*
 !**/.yarn/releases
-*.jsonl
+*.jsonl
+bench-out

barretenberg/acir_tests/bench_acir_tests.sh

@@ -3,8 +3,7 @@ set -e
 
 cd "$(dirname "$0")"
 
-NO_CACHE=0 ./bootstrap.sh full
-
+set -x
 TEST_NAMES=("$@")
 THREADS=(1 4 16 32 64)
 BENCHMARKS=$LOG_FILE

barretenberg/acir_tests/bootstrap.sh

@@ -134,10 +134,31 @@ function test_cmds_internal {
   echo FLOW=prove_then_verify_client_ivc $run_test databus_two_calldata
 }
 
+function ultra_honk_wasm_memory {
+  VERBOSE=1 BIN=../ts/dest/node/main.js SYS=ultra_honk_deprecated FLOW=prove_then_verify ./run_test.sh verify_honk_proof &> ./bench-out/ultra_honk_rec_wasm_memory.txt
+}
+
+function run_benchmark {
+  local start_core=$(( ($1 - 1) * HARDWARE_CONCURRENCY ))
+  local end_core=$(( start_core + (HARDWARE_CONCURRENCY - 1) ))
+  echo taskset -c $start_core-$end_core bash -c "$2"
+  taskset -c $start_core-$end_core bash -c "$2"
+}
+
 # TODO(https://github.com/AztecProtocol/barretenberg/issues/1254): More complete testing, including failure tests
 function bench {
   # TODO: Move to scripts dir along with run_test.sh.
-  LOG_FILE=bench-acir.jsonl ./bench_acir_tests.sh
+  # TODO(https://github.com/AztecProtocol/barretenberg/issues/1265) fix acir benchmarking
+  # LOG_FILE=bench-acir.jsonl ./bench_acir_tests.sh
+
+  export HARDWARE_CONCURRENCY=16
+
+  rm -rf bench-out && mkdir -p bench-out
+  export -f ultra_honk_wasm_memory run_benchmark
+  local num_cpus=$(get_num_cpus)
+  local jobs=$((num_cpus / HARDWARE_CONCURRENCY))
+  parallel -v --line-buffer --tag --jobs "$jobs" run_benchmark {#} {} ::: \
+    ultra_honk_wasm_memory
 }
 
 case "$cmd" in

barretenberg/bootstrap.sh

@@ -8,3 +8,20 @@ source $(git rev-parse --show-toplevel)/ci3/source
 ./cpp/bootstrap.sh $@
 ./ts/bootstrap.sh $@
 ./acir_tests/bootstrap.sh $@
+
+cmd=${1:-}
+if [ "$cmd" == "bench" ]; then
+    rm -rf bench-out && mkdir -p bench-out
+    ./scripts/combine_benchmarks.py \
+    native ./cpp/bench-out/client_ivc_17_in_20_release.json \
+    native ./cpp/bench-out/client_ivc_release.json \
+    native ./cpp/bench-out/ultra_honk_release.json \
+    wasm ./cpp/bench-out/client_ivc_wasm.json \
+    wasm ./cpp/bench-out/ultra_honk_wasm.json \
+    "" ./cpp/bench-out/client_ivc_op_count.json \
+    "" ./cpp/bench-out/client_ivc_op_count_time.json \
+    wasm ./acir_tests/bench-out/ultra_honk_rec_wasm_memory.txt \
+    > ./bench-out/bb-bench.json
+
+  cache_upload barretenberg-bench-results-$COMMIT_HASH.tar.gz ./bench-out/bb-bench.json
+fi 

barretenberg/cpp/Earthfile

@@ -248,6 +248,8 @@ bench:
   COPY +bench-client-ivc/* .
   END
   COPY ./scripts/ci/combine_benchmarks.py ./scripts/ci/combine_benchmarks.py
+  RUN cd ../acir_tests && VERBOSE=1 BIN=../ts/dest/node/main.js SYS=ultra_honk FLOW=prove_then_verify ./run_test.sh verify_honk_proof > ultra_honk_rec_verifier_memory.txt
+  SAVE ARTIFACT ultra_honk_rec_verifier_memory.txt
   RUN ./scripts/ci/combine_benchmarks.py \
     native client_ivc_17_in_20_release.json \
     native client_ivc_release.json \
@@ -256,6 +258,7 @@ bench:
     wasm ultra_honk_wasm.json \
     "" client_ivc_op_count.json \
     "" client_ivc_op_count_time.json \
+    wasm ultra_honk_rec_verifier_memory.txt \
     > bench.json
   SAVE ARTIFACT bench.json
 

barretenberg/cpp/bootstrap.sh

@@ -272,17 +272,6 @@ function bench {
     client_ivc_op_count_time \
     client_ivc_wasm
 
-  ./scripts/ci/combine_benchmarks.py \
-    native ./bench-out/client_ivc_17_in_20_release.json \
-    native ./bench-out/client_ivc_release.json \
-    native ./bench-out/ultra_honk_release.json \
-    wasm ./bench-out/client_ivc_wasm.json \
-    wasm ./bench-out/ultra_honk_wasm.json \
-    "" ./bench-out/client_ivc_op_count.json \
-    "" ./bench-out/client_ivc_op_count_time.json \
-    > ./bench-out/bb-bench.json
-
-    cache_upload barretenberg-bench-results-$COMMIT_HASH.tar.gz ./bench-out/bb-bench.json
 }
 
 # Upload assets to release.

barretenberg/scripts/combine_benchmarks.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+import json
+import sys
+import re
+
+TIME_COUNTERS_USED = ["commit(t)", "Goblin::merge(t)"]
+
+def modify_benchmark_data(file_paths, prefixes):
+    combined_results = {"benchmarks": []}
+    memory_pattern = re.compile(r"\(mem: ([\d.]+)MiB\)")
+
+    for file_path, prefix in zip(file_paths, prefixes):
+        with open(file_path, 'r') as file:
+            # if file is a txt, load as text
+            if file_path.endswith(".txt"):
+                last_memory = None
+                for line in reversed(file.readlines()):
+                    match = memory_pattern.search(line)
+                    if match:
+                        last_memory = match.group(1)
+                        break
+                if last_memory:
+                    new_entry = {
+                        "name": f"{prefix}UltraHonkVerifierWasmMemory",
+                        "real_time": last_memory,
+                        "time_unit": "MiB"
+                    }
+                    combined_results['benchmarks'].append(new_entry)
+                else:
+                    print(f"Warning: No memory found in {file_path}")
+            else:
+                data = json.load(file)
+                # Modify benchmark names and extract specific data
+                for benchmark in data['benchmarks']:
+                    # Prefix benchmark names
+                    benchmark['name'] = f"{prefix}{benchmark['name']}"
+                    benchmark['run_name'] = f"{prefix}{benchmark['run_name']}"
+
+                    if prefix != "":
+                        combined_results['benchmarks'].append(benchmark)
+                    # Isolate batch_mul_with_endomorphism
+                    for counter in TIME_COUNTERS_USED:
+                        if counter in benchmark:
+                            new_entry = {
+                                "name": f"{counter}",
+                                "run_name": benchmark['run_name'],
+                                "run_type": benchmark['run_type'],
+                                "repetitions": benchmark['repetitions'],
+                                "repetition_index": benchmark['repetition_index'],
+                                "threads": benchmark['threads'],
+                                "iterations": benchmark['iterations'],
+                                "real_time": benchmark[counter],
+                                "cpu_time": benchmark[counter],
+                                "time_unit": "ns"
+                            }
+                            combined_results['benchmarks'].append(new_entry)
+
+    return combined_results
+
+# Using command line arguments to get prefixes and file paths
+if len(sys.argv) < 3 or len(sys.argv) % 2 != 1:
+    print("Usage: python script.py <Prefix1> <file1> <Prefix2> <file2> ...")
+    sys.exit(1)
+
+prefixes = sys.argv[1::2]
+file_paths = sys.argv[2::2]
+
+final_data = modify_benchmark_data(file_paths, prefixes)
+
+# Save the combined results to a file
+print(json.dumps(final_data, indent=4))
+

barretenberg/ts/bootstrap.sh

@@ -64,6 +64,9 @@ case "$cmd" in
   "hash")
     echo "$hash"
     ;;
+  "bench")
+    echo "ts/bootstrap.sh bench is empty"
+    ;;
   test|test_cmds|release|release_commit)
     $cmd
     ;;

bootstrap.sh

@@ -179,7 +179,7 @@ function bench {
   if [ "$CI_FULL" -eq 0 ] || [ $(arch) == arm64 ]; then
     return
   fi
-  denoise "barretenberg/cpp/bootstrap.sh bench"
+  denoise "barretenberg/bootstrap.sh bench"
   denoise "yarn-project/end-to-end/bootstrap.sh bench"
 }