diff --git a/.github/workflows/vm_full_tests.yml b/.github/workflows/vm_full_tests.yml index cfd832b4469..a237aff696d 100644 --- a/.github/workflows/vm_full_tests.yml +++ b/.github/workflows/vm_full_tests.yml @@ -24,6 +24,7 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} + GITHUB_RUN_URL: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}" GIT_COMMIT: ${{ github.sha }} steps: - uses: actions/checkout@v4 @@ -34,4 +35,4 @@ jobs: with: run: | sudo shutdown -P 75 # hack until core part of the scripts - earthly-ci --no-output ./barretenberg/cpp/+vm-full-test --hardware_concurrency=64 # limit our parallelism to half our cores + scripts/earthly-ci --no-output ./barretenberg/cpp/+vm-full-test --hardware_concurrency=64 # limit our parallelism to half our cores diff --git a/avm-transpiler/bootstrap.sh b/avm-transpiler/bootstrap.sh index 7e1f37c8ea6..d4429e25a4d 100755 --- a/avm-transpiler/bootstrap.sh +++ b/avm-transpiler/bootstrap.sh @@ -17,8 +17,8 @@ function build { } function test { - cargo fmt --check - cargo clippy + denoise cargo fmt --check + denoise cargo clippy } case "$cmd" in diff --git a/barretenberg/acir_tests/browser-test-app/yarn.lock b/barretenberg/acir_tests/browser-test-app/yarn.lock index d209e4296d3..37157884379 100644 --- a/barretenberg/acir_tests/browser-test-app/yarn.lock +++ b/barretenberg/acir_tests/browser-test-app/yarn.lock @@ -6,8 +6,8 @@ __metadata: cacheKey: 10c0 "@aztec/bb.js@file:../../ts::locator=browser-test-app%40workspace%3A.": - version: 0.67.0 - resolution: "@aztec/bb.js@file:../../ts#../../ts::hash=e236e3&locator=browser-test-app%40workspace%3A." + version: 0.67.1 + resolution: "@aztec/bb.js@file:../../ts#../../ts::hash=29e47a&locator=browser-test-app%40workspace%3A." dependencies: comlink: "npm:^4.4.1" commander: "npm:^12.1.0" @@ -17,7 +17,7 @@ __metadata: tslib: "npm:^2.4.0" bin: bb.js: ./dest/node/main.js - checksum: 10c0/4ca2234f9d1a4b59a60733979f97c5ca09f508e4d3091bf33755ee6a80f03f733e393ea848b812bc73bc1f0673a3d92f96462a1cbd5aa01eeca4e9d5a3be367f + checksum: 10c0/c01128ff74f29b6bbc5c46362792525ef5612c5fc8787341551bcf457ba9816a971e24a74292ab230c47b0b9efe8d7e0d1cabd44247e1b6e718727d0b6372400 languageName: node linkType: hard diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index 7ea2de2e510..aad3427e64a 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -115,7 +115,7 @@ case "$cmd" in test ;; "ci") - build_native + build test ;; "hash") diff --git a/build-images/bootstrap.sh b/build-images/bootstrap.sh index b177159ed28..6b5d377ad96 100755 --- a/build-images/bootstrap.sh +++ b/build-images/bootstrap.sh @@ -3,7 +3,7 @@ source $(git rev-parse --show-toplevel)/ci3/source_bootstrap cmd=${1:-} -hash=$(REBUILD_PATTERNS="^build-images/Earthfile" cache_content_hash) +hash=$(cache_content_hash "^build-images/Earthfile") function build { github_group "build-images build" diff --git a/ci3/bootstrap_local b/ci3/bootstrap_local index 2b1c78d4e60..28a141afe13 100755 --- a/ci3/bootstrap_local +++ b/ci3/bootstrap_local @@ -9,30 +9,24 @@ root=$(git rev-parse --show-toplevel) source $root/ci3/source -current_commit=$(git rev-parse HEAD) -cmd=${1:-"CI=1 ./bootstrap.sh fast || exec bash"} +cmd=${1:-"./bootstrap.sh ci || exec bash"} if ! git diff --quiet; then - echo "There are local changes to tracked files." - exit 1 + echo -e "${yellow}WARNING: There are local changes to tracked files.${reset}" fi -interactive_args="" -# Only add -ti and a name if we are not in CI. -[ "${CI:-0}" = "0" ] && interactive_args="--name aztec_build -ti" -docker run $interactive_args --rm \ +docker run --name aztec_build -ti --rm \ --privileged \ -v bootstrap_ci_local_docker:/var/lib/docker \ -v $root:/aztec-packages-host:ro \ -v $HOME/.aws:/root/.aws \ aztecprotocol/ci:2.0 bash -c " + set -e /usr/local/share/docker-init.sh &> /dev/null git config --global --add safe.directory /aztec-packages-host/.git - mkdir -p /root/aztec-packages && cd /root/aztec-packages + cd /root # Ensure we get a clean clone of the repo. - git init &>/dev/null - git remote add origin https://github.com/aztecprotocol/aztec-packages - git fetch --depth 1 origin $current_commit 2>/dev/null || (echo 'The commit was not pushed, run aborted.' && exit 1) - git checkout FETCH_HEAD &>/dev/null + git clone --depth 1 file:///aztec-packages-host aztec-packages + cd aztec-packages $cmd " diff --git a/ci3/cache_content_hash b/ci3/cache_content_hash index ad0d8e7e2cd..833a1907441 100755 --- a/ci3/cache_content_hash +++ b/ci3/cache_content_hash @@ -2,24 +2,29 @@ set -euo pipefail [ "${BUILD_SYSTEM_DEBUG:-}" = 1 ] && set -x -# Ensure REBUILD_PATTERNS or args are set -if [[ -z "${REBUILD_PATTERNS:-}" && "$#" = 0 ]]; then - echo "Error: No arguments provided and REBUILD_PATTERNS environment variable is not set." +# Ensure args are set +if [[ "$#" = 0 ]]; then + echo "Error: No arguments provided." exit 1 fi # If too many spurious cache misses: can be customized to pin artifacts to a specific version AZTEC_CACHE_COMMIT=${AZTEC_CACHE_COMMIT:-HEAD} PLATFORM_TAG="${PLATFORM_TAG:-${OSTYPE:-unknown}-$(uname -m)}" -# Literal patterns can be provided with REBUILD_PATTERNS -REBUILD_PATTERNS=${REBUILD_PATTERNS:-} -if [ "$#" != 0 ]; then - REBUILD_PATTERNS+=$'\n'$(cat "$@") -fi + +rebuild_patterns=() +for arg in "$@"; do + if [[ -f "$arg" ]]; then + rebuild_patterns+=$(cat "$arg") + rebuild_patterns+=$'\n' + else + rebuild_patterns+="$arg"$'\n' + fi +done # Concatenate patterns with '|' and double escape backslashes for AWK # filter empty lines -AWK_PATTERN=$(echo "$REBUILD_PATTERNS" | grep -v '^$' | sed 's/\\/\\\\/g' | tr '\n' '|' | sed 's/|$//') +AWK_PATTERN=$(echo "$rebuild_patterns" | grep -v '^$' | sed 's/\\/\\\\/g' | tr '\n' '|' | sed 's/|$//') # use git repo root because that is where our patterns are focused cd $(git rev-parse --show-toplevel) @@ -30,7 +35,7 @@ CONTENT_HASH=$(git ls-tree -r $AZTEC_CACHE_COMMIT | awk -v pattern="($AWK_PATTER # Check if file list was empty by comparing against the result of 'echo '' | git hash-object --stdin | cut -c1-16' ECHO_BLANK_HASH="8b137891791fe969" if [ "$CONTENT_HASH" = "$ECHO_BLANK_HASH" ]; then - echo "No files matched the rebuild patterns $REBUILD_PATTERNS." + echo "No files matched the rebuild patterns $rebuild_patterns." echo "Awk pattern expanded: $AWK_PATTERN." exit 1 fi diff --git a/ci3/dump_fail b/ci3/dump_fail index c40ddf0cbf2..5e933822055 100755 --- a/ci3/dump_fail +++ b/ci3/dump_fail @@ -30,7 +30,7 @@ if [ "$status" -ne 0 ]; then cat "$stderr" } >&2 else - echo "$output" + echo -n "$output" fi exit $status diff --git a/ci3/source b/ci3/source index b54c70f663e..063437456d9 100644 --- a/ci3/source +++ b/ci3/source @@ -18,6 +18,11 @@ function hash_str { set -eu echo $1 | git hash-object --stdin | cut -c1-16 | tr -d '\n' } -export -f hash_str + +function echo_stderr { + echo $@ >&2 +} + +export -f hash_str echo_stderr source $ci3/source_color \ No newline at end of file diff --git a/ci3/source_bootstrap b/ci3/source_bootstrap index 6bc88a65f32..2a3cc2ba030 100644 --- a/ci3/source_bootstrap +++ b/ci3/source_bootstrap @@ -10,7 +10,7 @@ case "${1:-}" in export DENOISE=${DENOISE:-1} ;; ""|"fast") - export USE_CACHE=1 + export USE_CACHE=${USE_CACHE:-1} ;; "test") export TEST=1 diff --git a/noir-projects/noir-contracts/bootstrap.sh b/noir-projects/noir-contracts/bootstrap.sh index 7330bbc26e1..aa43652e2d7 100755 --- a/noir-projects/noir-contracts/bootstrap.sh +++ b/noir-projects/noir-contracts/bootstrap.sh @@ -7,7 +7,7 @@ # - The exported functions called by parallel must enable their own flags at the start e.g. set -euo pipefail # - The exported functions are using stdin/stdout, so be very careful about what's printed where. # - The exported functions need to have external variables they require, to have been exported first. -# - If you want to echo something, send it to stderr e.g. echo "My debug" >&2 +# - If you want to echo something, send it to stderr e.g. echo_stderr "My debug" # - If you call another script, be sure it also doesn't output something you don't want. # - Note calls to cache scripts swallow everything with &> /dev/null. # - Local assignments with subshells don't propagate errors e.g. local capture=$(false). Declare locals separately. @@ -58,7 +58,7 @@ function process_function() { set +e make_vk=$(echo "$func" | jq -e '(.custom_attributes | index("public") == null) and (.is_unconstrained == false)') if [ $? -ne 0 ] && [ "$make_vk" != "false" ]; then - echo "Failed to check function $name is neither public nor unconstrained." >&2 + echo_stderr "Failed to check function $name is neither public nor unconstrained." exit 1 fi set -e @@ -70,7 +70,7 @@ function process_function() { hash=$((echo "$BB_HASH"; echo "$bytecode_b64") | sha256sum | tr -d ' -') if ! cache_download vk-$hash.tar.gz &> /dev/null; then # It's not in the cache. Generate the vk file and upload it to the cache. - echo "Generating vk for function: $name..." >&2 + echo_stderr "Generating vk for function: $name..." echo "$bytecode_b64" | base64 -d | gunzip | $BB write_vk_for_ivc -h -b - -o $tmp_dir/$hash 2>/dev/null cache_upload vk-$hash.tar.gz $tmp_dir/$hash &> /dev/null fi @@ -97,8 +97,12 @@ function compile { contract_name=$(cat contracts/$1/src/main.nr | awk '/^contract / { print $2 }') local filename="$contract-$contract_name.json" local json_path="./target/$filename" - export REBUILD_PATTERNS="^noir-projects/noir-contracts/contracts/$contract/" - contract_hash="$(cache_content_hash ../../noir/.rebuild_patterns ../../avm-transpiler/.rebuild_patterns)" + contract_hash="$(cache_content_hash \ + ../../noir/.rebuild_patterns \ + ../../avm-transpiler/.rebuild_patterns \ + "^noir-projects/noir-contracts/contracts/$contract/" \ + "^noir-projects/aztec-nr/" \ + )" if ! cache_download contract-$contract_hash.tar.gz &> /dev/null; then $NARGO compile --package $contract --silence-warnings --inliner-aggressiveness 0 $TRANSPILER $json_path $json_path @@ -125,7 +129,7 @@ function build { compile $1 else set +e - echo "Compiling contracts (bb-hash: $BB_HASH)..." + echo_stderr "Compiling contracts (bb-hash: $BB_HASH)..." grep -oP '(?<=contracts/)[^"]+' Nargo.toml | \ parallel -j16 --joblog joblog.txt -v --line-buffer --tag --halt now,fail=1 compile {} code=$? @@ -143,15 +147,12 @@ case "$cmd" in ;; "clean-keys") for artifact in target/*.json; do - echo "Scrubbing vk from $artifact..." + echo_stderr "Scrubbing vk from $artifact..." jq '.functions |= map(del(.verification_key))' "$artifact" > "${artifact}.tmp" mv "${artifact}.tmp" "$artifact" done ;; - ""|"fast"|"ci") - USE_CACHE=1 build - ;; - "full") + ""|"fast"|"full"|"ci") build ;; "compile") @@ -163,6 +164,6 @@ case "$cmd" in exit 0 ;; *) - echo "Unknown command: $cmd" + echo_stderr "Unknown command: $cmd" exit 1 esac \ No newline at end of file diff --git a/noir-projects/noir-protocol-circuits/bootstrap.sh b/noir-projects/noir-protocol-circuits/bootstrap.sh index 4038f9c9b05..1abd3b83255 100755 --- a/noir-projects/noir-protocol-circuits/bootstrap.sh +++ b/noir-projects/noir-protocol-circuits/bootstrap.sh @@ -33,7 +33,7 @@ function on_exit() { } trap on_exit EXIT -[ -f package.json ] && yarn && node ./scripts/generate_variants.js +[ -f package.json ] && denoise "yarn && node ./scripts/generate_variants.js" mkdir -p $tmp_dir mkdir -p $key_dir @@ -48,13 +48,19 @@ function compile { local filename="$name.json" local json_path="./target/$filename" local program_hash hash bytecode_hash vk vk_fields - program_hash=$($NARGO check --package $name --silence-warnings --show-program-hash | cut -d' ' -f2) - hash=$(echo "$NARGO_HASH-$program_hash" | sha256sum | tr -d ' -') - if ! cache_download circuit-$hash.tar.gz &> /dev/null; then + local program_hash_cmd="$NARGO check --package $name --silence-warnings --show-program-hash | cut -d' ' -f2" + # echo_stderr $program_hash_cmd + program_hash=$(dump_fail "$program_hash_cmd") + echo_stderr "Hash preimage: $NARGO_HASH-$program_hash" + hash=$(hash_str "$NARGO_HASH-$program_hash") + if ! cache_download circuit-$hash.tar.gz 1>&2; then SECONDS=0 + rm -f $json_path # TODO: --skip-brillig-constraints-check added temporarily for blobs build time. - $NARGO compile --package $name --silence-warnings --skip-brillig-constraints-check - echo "Compilation complete for: $name (${SECONDS}s)" + local compile_cmd="$NARGO compile --package $name --silence-warnings --skip-brillig-constraints-check" + echo_stderr "$compile_cmd" + dump_fail "$compile_cmd" + echo_stderr "Compilation complete for: $name (${SECONDS}s)" cache_upload circuit-$hash.tar.gz $json_path &> /dev/null fi @@ -74,19 +80,19 @@ function compile { # Change this to add verification_key to original json, like contracts does. # Will require changing TS code downstream. bytecode_hash=$(jq -r '.bytecode' $json_path | sha256sum | tr -d ' -') - hash=$(echo "$BB_HASH-$bytecode_hash-$proto" | sha256sum | tr -d ' -') - if ! cache_download vk-$hash.tar.gz &> /dev/null; then + hash=$(hash_str "$BB_HASH-$bytecode_hash-$proto") + if ! cache_download vk-$hash.tar.gz 1>&2; then local key_path="$key_dir/$name.vk.data.json" - echo "Generating vk for function: $name..." >&2 + echo_stderr "Generating vk for function: $name..." SECONDS=0 local vk_cmd="jq -r '.bytecode' $json_path | base64 -d | gunzip | $BB $write_vk_cmd -h -b - -o - --recursive | xxd -p -c 0" - echo $vk_cmd >&2 + echo_stderr $vk_cmd vk=$(dump_fail "$vk_cmd") local vkf_cmd="echo '$vk' | xxd -r -p | $BB $vk_as_fields_cmd -k - -o -" - # echo $vkf_cmd >&2 + # echo_stderrr $vkf_cmd vk_fields=$(dump_fail "$vkf_cmd") jq -n --arg vk "$vk" --argjson vkf "$vk_fields" '{keyAsBytes: $vk, keyAsFields: $vkf}' > $key_path - echo "Key output at: $key_path (${SECONDS}s)" + echo_stderr "Key output at: $key_path (${SECONDS}s)" cache_upload vk-$hash.tar.gz $key_path &> /dev/null fi } @@ -111,7 +117,7 @@ function test { set -eu # Whether we run the tests or not is coarse grained. name=$(basename "$PWD") - CIRCUITS_HASH=$(REBUILD_PATTERNS="^noir-projects/$name" cache_content_hash ../../noir/.rebuild_patterns) + CIRCUITS_HASH=$(cache_content_hash ../../noir/.rebuild_patterns "^noir-projects/$name") if ! test_should_run $name-tests-$CIRCUITS_HASH; then return fi @@ -144,6 +150,6 @@ case "$CMD" in parallel --line-buffered bash -c {} ::: build test ;; *) - echo "Unknown command: $CMD" + echo_stderr "Unknown command: $CMD" exit 1 esac diff --git a/scripts/ci/get_e2e_jobs.sh b/scripts/ci/get_e2e_jobs.sh index 3b1f8fc7f9a..21b2fe85105 100755 --- a/scripts/ci/get_e2e_jobs.sh +++ b/scripts/ci/get_e2e_jobs.sh @@ -41,6 +41,7 @@ allow_list=( "e2e_prover_fake_proofs" "e2e_prover_coordination" "e2e_lending_contract" + "e2e_p2p_gossip" "kind_network_smoke" "guides_dapp_testing" "guides_sample_dapp" diff --git a/scripts/earthly-ci b/scripts/earthly-ci index b120eab6351..40ad19bb513 100755 --- a/scripts/earthly-ci +++ b/scripts/earthly-ci @@ -67,8 +67,10 @@ while [ $ATTEMPT_COUNT -lt $MAX_ATTEMPTS ]; do || grep 'docker: failed to write digest data' >/dev/null \ || grep 'docker: unexpected EOF' $OUTPUT_FILE >/dev/null; then # try to wait it out - echo sleep 20 + echo "earthly-ci: recoverable error, waiting 20 seconds" + sleep 20 + else + exit $earthly_exit_code fi - exit $earthly_exit_code fi -done \ No newline at end of file +done diff --git a/spartan/aztec-network/templates/boot-node.yaml b/spartan/aztec-network/templates/boot-node.yaml index 62dbcab873c..f6f52b4b9b2 100644 --- a/spartan/aztec-network/templates/boot-node.yaml +++ b/spartan/aztec-network/templates/boot-node.yaml @@ -175,8 +175,6 @@ spec: value: "{{ .Values.bootNode.logLevel }}" - name: LOG_JSON value: "1" - - name: DEBUG - value: "{{ .Values.bootNode.debug }}" - name: P2P_ENABLED value: "{{ .Values.bootNode.p2p.enabled }}" - name: COINBASE diff --git a/spartan/aztec-network/templates/prover-node.yaml b/spartan/aztec-network/templates/prover-node.yaml index 3cdfb76721c..908c30902bd 100644 --- a/spartan/aztec-network/templates/prover-node.yaml +++ b/spartan/aztec-network/templates/prover-node.yaml @@ -125,8 +125,6 @@ spec: value: "{{ .Values.proverNode.logLevel }}" - name: LOG_JSON value: "1" - - name: DEBUG - value: "{{ .Values.proverNode.debug }}" - name: PROVER_REAL_PROOFS value: "{{ .Values.aztec.realProofs }}" - name: PROVER_AGENT_COUNT diff --git a/spartan/aztec-network/templates/validator.yaml b/spartan/aztec-network/templates/validator.yaml index 7d7c131b176..b866f154ef7 100644 --- a/spartan/aztec-network/templates/validator.yaml +++ b/spartan/aztec-network/templates/validator.yaml @@ -162,8 +162,6 @@ spec: value: "{{ .Values.validator.logLevel }}" - name: LOG_JSON value: "1" - - name: DEBUG - value: "{{ .Values.validator.debug }}" - name: P2P_ENABLED value: "{{ .Values.validator.p2p.enabled }}" - name: VALIDATOR_DISABLED diff --git a/spartan/aztec-network/values.yaml b/spartan/aztec-network/values.yaml index dd218203c29..8c58606c7cd 100644 --- a/spartan/aztec-network/values.yaml +++ b/spartan/aztec-network/values.yaml @@ -45,8 +45,7 @@ bootNode: p2pTcpPort: 40400 p2pUdpPort: 40400 nodePort: 8080 - logLevel: "debug" - debug: "" + logLevel: "debug; info: aztec:simulator, json-rpc" coinbaseAddress: "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" sequencer: maxSecondsBetweenBlocks: 0 @@ -90,8 +89,7 @@ validator: p2pTcpPort: 40400 p2pUdpPort: 40400 nodePort: 8080 - logLevel: "debug" - debug: "" + logLevel: "debug; info: aztec:simulator, json-rpc" sequencer: maxSecondsBetweenBlocks: 0 minTxsPerBlock: 1 @@ -123,8 +121,7 @@ proverNode: p2pTcpPort: 40400 p2pUdpPort: 40400 nodePort: 8080 - logLevel: "debug" - debug: "" + logLevel: "debug; info: aztec:simulator, json-rpc" proverAgent: count: 0 pollIntervalMs: 1000 @@ -142,7 +139,7 @@ proverNode: storage: "8Gi" pxe: - logLevel: "debug" + logLevel: "debug; info: aztec:simulator, json-rpc" replicas: 1 service: nodePort: 8081 @@ -160,7 +157,7 @@ pxe: bot: enabled: true nodeUrl: "" - logLevel: "debug" + logLevel: "debug; info: aztec:simulator, json-rpc" replicas: 1 botPrivateKey: "0xcafe" txIntervalSeconds: 24 @@ -221,7 +218,7 @@ proverAgent: proofTypes: ["foo", "bar", "baz"] gke: spotEnabled: false - logLevel: "debug" + logLevel: "debug; info: aztec:simulator, json-rpc" bb: hardwareConcurrency: "" nodeSelector: {} @@ -236,9 +233,12 @@ proverBroker: pollIntervalMs: 1000 jobMaxRetries: 3 dataDirectory: "" - logLevel: "debug" + logLevel: "debug; info: aztec:simulator, json-rpc" nodeSelector: {} - resources: {} + resources: + requests: + memory: "2Gi" + cpu: "200m" jobs: deployL1Verifier: diff --git a/spartan/aztec-network/values/exp-1.yaml b/spartan/aztec-network/values/exp-1.yaml index 05247cd7722..0f086a93c65 100644 --- a/spartan/aztec-network/values/exp-1.yaml +++ b/spartan/aztec-network/values/exp-1.yaml @@ -117,7 +117,8 @@ validator: - 0x3c3E2E178C69D4baD964568415a0f0c84fd6320A resources: requests: - memory: "512Mi" + memory: "8Gi" + cpu: "3.5" validator: disabled: false sequencer: @@ -129,12 +130,28 @@ bootNode: storageSize: "100Gi" validator: disabled: true + resources: + requests: + memory: "8Gi" + cpu: "3.5" proverAgent: - replicas: 4 + replicas: 8 + +proverBroker: + resources: + requests: + memory: "8Gi" + cpu: "3.5" + +proverNode: + resources: + requests: + memory: "8Gi" + cpu: "3.5" bot: - replicas: 6 + replicas: 10 followChain: "NONE" enabled: true txIntervalSeconds: 1 @@ -142,3 +159,9 @@ bot: jobs: deployL1Verifier: enable: false + +ethereum: + resources: + requests: + memory: "8Gi" + cpu: "3.5" diff --git a/yarn-project/circuit-types/src/l2_block_downloader/l2_block_stream.ts b/yarn-project/circuit-types/src/l2_block_downloader/l2_block_stream.ts index 10724c9b545..7597f27c284 100644 --- a/yarn-project/circuit-types/src/l2_block_downloader/l2_block_stream.ts +++ b/yarn-project/circuit-types/src/l2_block_downloader/l2_block_stream.ts @@ -8,6 +8,7 @@ import { type L2BlockId, type L2BlockSource, type L2Tips } from '../l2_block_sou /** Creates a stream of events for new blocks, chain tips updates, and reorgs, out of polling an archiver or a node. */ export class L2BlockStream { private readonly runningPromise: RunningPromise; + private isSyncing = false; constructor( private l2BlockSource: Pick, @@ -37,8 +38,10 @@ export class L2BlockStream { return this.runningPromise.isRunning(); } - public sync() { - return this.runningPromise.trigger(); + public async sync() { + this.isSyncing = true; + await this.runningPromise.trigger(); + this.isSyncing = false; } protected async work() { @@ -132,7 +135,7 @@ export class L2BlockStream { `Emitting ${event.type} (${event.type === 'blocks-added' ? event.blocks.length : event.blockNumber})`, ); await this.handler.handleBlockStreamEvent(event); - if (!this.isRunning()) { + if (!this.isRunning() && !this.isSyncing) { throw new AbortError(); } } diff --git a/yarn-project/cli/src/cmds/devnet/faucet.ts b/yarn-project/cli/src/cmds/devnet/faucet.ts index 73e9d5c0642..c99b19b973c 100644 --- a/yarn-project/cli/src/cmds/devnet/faucet.ts +++ b/yarn-project/cli/src/cmds/devnet/faucet.ts @@ -10,7 +10,7 @@ export async function dripFaucet( json: boolean, log: LogFn, ): Promise { - const url = new URL(`${faucetUrl}/drip/${account.toString()}`); + const url = new URL(`/drip/${account.toString()}`, faucetUrl); url.searchParams.set('asset', asset); const res = await fetch(url); if (res.status === 200) { diff --git a/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts b/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts index 2d55474e7c8..4bfe4c45e99 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts @@ -80,6 +80,28 @@ export class P2PNetworkTest { }); } + static async create({ + testName, + numberOfNodes, + basePort, + metricsPort, + }: { + testName: string; + numberOfNodes: number; + basePort?: number; + metricsPort?: number; + }) { + const port = basePort || (await getPort()); + + const telemetry = await getEndToEndTestTelemetryClient(metricsPort); + const bootstrapNode = await createBootstrapNodeFromPrivateKey(BOOTSTRAP_NODE_PRIVATE_KEY, port, telemetry); + const bootstrapNodeEnr = bootstrapNode.getENR().encodeTxt(); + + const initialValidatorConfig = await createValidatorConfig({} as AztecNodeConfig, bootstrapNodeEnr); + + return new P2PNetworkTest(testName, bootstrapNode, port, numberOfNodes, initialValidatorConfig); + } + /** * Start a loop to sync the mock system time with the L1 block time */ @@ -108,28 +130,6 @@ export class P2PNetworkTest { dateProvider.setTime(Number(timestamp.timestamp) * 1000); } - static async create({ - testName, - numberOfNodes, - basePort, - metricsPort, - }: { - testName: string; - numberOfNodes: number; - basePort?: number; - metricsPort?: number; - }) { - const port = basePort || (await getPort()); - - const telemetry = await getEndToEndTestTelemetryClient(metricsPort); - const bootstrapNode = await createBootstrapNodeFromPrivateKey(BOOTSTRAP_NODE_PRIVATE_KEY, port, telemetry); - const bootstrapNodeEnr = bootstrapNode.getENR().encodeTxt(); - - const initialValidatorConfig = await createValidatorConfig({} as AztecNodeConfig, bootstrapNodeEnr); - - return new P2PNetworkTest(testName, bootstrapNode, port, numberOfNodes, initialValidatorConfig); - } - async applyBaseSnapshots() { await this.snapshotManager.snapshot( 'add-validators', diff --git a/yarn-project/end-to-end/src/fixtures/fixtures.ts b/yarn-project/end-to-end/src/fixtures/fixtures.ts index 79d10d0d2da..7fec6f90d39 100644 --- a/yarn-project/end-to-end/src/fixtures/fixtures.ts +++ b/yarn-project/end-to-end/src/fixtures/fixtures.ts @@ -7,6 +7,8 @@ export const shouldCollectMetrics = () => { return undefined; }; +export const TEST_PEER_CHECK_INTERVAL_MS = 1000; + export const MNEMONIC = 'test test test test test test test test test test test junk'; export const privateKey = Buffer.from('ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80', 'hex'); export const privateKey2 = Buffer.from('59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d', 'hex'); diff --git a/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts b/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts index 1d31d41ab6c..5ddcc3b6ee2 100644 --- a/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts +++ b/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts @@ -9,6 +9,7 @@ import { type PXEService } from '@aztec/pxe'; import getPort from 'get-port'; +import { TEST_PEER_CHECK_INTERVAL_MS } from './fixtures.js'; import { getPrivateKeyFromIndex } from './utils.js'; import { getEndToEndTestTelemetryClient } from './with_telemetry_utils.js'; @@ -101,6 +102,7 @@ export async function createValidatorConfig( tcpAnnounceAddress: `127.0.0.1:${port}`, udpAnnounceAddress: `127.0.0.1:${port}`, p2pEnabled: true, + peerCheckIntervalMS: TEST_PEER_CHECK_INTERVAL_MS, blockCheckIntervalMS: 1000, transactionProtocol: '', dataDirectory, diff --git a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts index 2acb78f7d47..d031d18c5bd 100644 --- a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts +++ b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts @@ -34,7 +34,7 @@ import path, { join } from 'path'; import { type Hex, getContract } from 'viem'; import { mnemonicToAccount } from 'viem/accounts'; -import { MNEMONIC } from './fixtures.js'; +import { MNEMONIC, TEST_PEER_CHECK_INTERVAL_MS } from './fixtures.js'; import { getACVMConfig } from './get_acvm_config.js'; import { getBBConfig } from './get_bb_config.js'; import { setupL1Contracts } from './setup_l1_contracts.js'; @@ -281,6 +281,7 @@ async function setupFromFresh( // Fetch the AztecNode config. // TODO: For some reason this is currently the union of a bunch of subsystems. That needs fixing. const aztecNodeConfig: AztecNodeConfig & SetupOptions = { ...getConfigEnvVars(), ...opts }; + aztecNodeConfig.peerCheckIntervalMS = TEST_PEER_CHECK_INTERVAL_MS; // Create a temp directory for all ephemeral state and cleanup afterwards const directoryToCleanup = path.join(tmpdir(), randomBytes(8).toString('hex')); diff --git a/yarn-project/end-to-end/src/fixtures/utils.ts b/yarn-project/end-to-end/src/fixtures/utils.ts index b931dfe1260..d6698a0ca04 100644 --- a/yarn-project/end-to-end/src/fixtures/utils.ts +++ b/yarn-project/end-to-end/src/fixtures/utils.ts @@ -67,7 +67,7 @@ import { import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts'; import { foundry } from 'viem/chains'; -import { MNEMONIC } from './fixtures.js'; +import { MNEMONIC, TEST_PEER_CHECK_INTERVAL_MS } from './fixtures.js'; import { getACVMConfig } from './get_acvm_config.js'; import { getBBConfig } from './get_bb_config.js'; import { isMetricsLoggingRequested, setupMetricsLogger } from './logging.js'; @@ -315,6 +315,8 @@ export async function setup( chain: Chain = foundry, ): Promise { const config = { ...getConfigEnvVars(), ...opts }; + config.peerCheckIntervalMS = TEST_PEER_CHECK_INTERVAL_MS; + const logger = getLogger(); // Create a temp directory for any services that need it and cleanup later diff --git a/yarn-project/p2p/src/config.ts b/yarn-project/p2p/src/config.ts index 5f5ea19258a..0e30bc2efe3 100644 --- a/yarn-project/p2p/src/config.ts +++ b/yarn-project/p2p/src/config.ts @@ -170,7 +170,7 @@ export const p2pConfigMappings: ConfigMappingsType = { peerCheckIntervalMS: { env: 'P2P_PEER_CHECK_INTERVAL_MS', description: 'The frequency in which to check for new peers.', - ...numberConfigHelper(30_000), + ...numberConfigHelper(1_000), }, l2QueueSize: { env: 'P2P_L2_QUEUE_SIZE',