From 367c38c02b6cda494e9d3c64ea27a1cf3465f082 Mon Sep 17 00:00:00 2001 From: Maddiaa <47148561+Maddiaa0@users.noreply.github.com> Date: Mon, 21 Oct 2024 13:39:21 +0100 Subject: [PATCH] chore(seq)!: disable sequencer and disable validator as one env var, update p2p listen port names (#9266) ## Overview Breaking change as SEQ_DISABLE is removed When validator is disabled - the sequencer is also disabled, update TCP_LISTEN_ADDR to be P2P_LISTEN_ADDR that is used in config files fixes: https://github.com/AztecProtocol/aztec-packages/issues/9262 --- scripts/run_native_testnet.sh | 69 ++++++++++++------- scripts/tmux_split_args.sh | 4 +- spartan/aztec-network/templates/_helpers.tpl | 8 +++ spartan/aztec-network/templates/pxe.yaml | 10 +++ yarn-project/Earthfile | 6 +- .../archiver/src/archiver/archiver.ts | 10 ++- .../aztec-node/src/aztec-node/config.ts | 8 --- .../aztec-node/src/aztec-node/server.ts | 2 +- yarn-project/aztec/src/cli/cmds/start_node.ts | 2 +- .../scripts/native-network/.gitignore | 6 +- .../scripts/native-network/boot-node.sh | 12 ++-- .../native-network/deploy-l1-contracts.sh | 20 ++++-- .../native-network/deploy-l2-contracts.sh | 15 ++-- .../scripts/native-network/ethereum.sh | 41 ++++++++--- .../scripts/native-network/prover-node.sh | 21 ++++-- .../end-to-end/scripts/native-network/pxe.sh | 10 ++- .../scripts/native-network/state/.gitignore | 1 + .../scripts/native-network/state/README.md | 1 + .../scripts/native-network/test-transfer.sh | 6 +- .../end-to-end/scripts/native-network/test.sh | 36 ++++++++++ .../scripts/native-network/transaction-bot.sh | 5 +- .../scripts/native-network/validator.sh | 20 +++--- .../scripts/native-network/validators.sh | 34 +++++++++ .../end-to-end/scripts/native_network_test.sh | 2 +- .../benchmarks/bench_process_history.test.ts | 2 +- .../benchmarks/bench_publish_rollup.test.ts | 2 +- .../end-to-end/src/e2e_synching.test.ts | 2 +- yarn-project/foundation/src/config/env_var.ts | 5 +- yarn-project/p2p/src/config.ts | 4 +- 29 files changed, 265 insertions(+), 99 deletions(-) create mode 100644 yarn-project/end-to-end/scripts/native-network/state/.gitignore create mode 100644 yarn-project/end-to-end/scripts/native-network/state/README.md create mode 100755 yarn-project/end-to-end/scripts/native-network/test.sh create mode 100755 yarn-project/end-to-end/scripts/native-network/validators.sh diff --git a/scripts/run_native_testnet.sh b/scripts/run_native_testnet.sh index 6ae04506b08..be9902b6573 100755 --- a/scripts/run_native_testnet.sh +++ b/scripts/run_native_testnet.sh @@ -4,7 +4,7 @@ This script sets up and runs a native testnet for the Aztec network. Steps: -1. Parse command-line arguments for custom test script and number of validators. +1. Parse command-line arguments for custom test script, number of validators, and logging level. 2. Set up the base command for running the native network test, including: - Running the specified test script - Deploying L1 and L2 contracts @@ -21,9 +21,7 @@ Usage: ./run_native_testnet.sh [options] Options: - -h: Display help message - -t: Specify a custom test script (default: ./test-transfer.sh) - -v: Specify the number of validators (default: 3) + see display_help() below ' # Function to display help message @@ -33,51 +31,74 @@ display_help() { echo "Options:" echo " -h Display this help message" echo " -t Specify the test script file (default: ./test-transfer.sh)" - echo " -v Specify the number of validators (default: 3)" + echo " -val Specify the number of validators (default: 3)" + echo " -v Set logging level to verbose" + echo " -vv Set logging level to debug" + echo " -i Run interleaved (default: false)" echo echo "Example:" - echo " $0 -t ./custom-test.sh -v 5" + echo " $0 -t ./custom-test.sh -val 5 -v" } # Default values TEST_SCRIPT="./test-transfer.sh" NUM_VALIDATORS=3 +LOG_LEVEL="info" +INTERLEAVED=false # Parse command line arguments -while getopts "ht:v:" opt; do - case $opt in - h) +while [[ $# -gt 0 ]]; do + case $1 in + -h) display_help exit 0 ;; - t) TEST_SCRIPT="$OPTARG" + -t) + TEST_SCRIPT="$2" + shift 2 ;; - v) NUM_VALIDATORS="$OPTARG" + -val) + NUM_VALIDATORS="$2" + shift 2 ;; - \?) echo "Invalid option -$OPTARG" >&2 - display_help - exit 1 + -v) + if [[ $LOG_LEVEL == "info" ]]; then + LOG_LEVEL="verbose" + elif [[ $LOG_LEVEL == "verbose" ]]; then + LOG_LEVEL="debug" + fi + shift + ;; + -i) + INTERLEAVED=true + shift + ;; + -vv) + LOG_LEVEL="debug" + shift + ;; + *) + echo "Invalid option: $1" >&2 + display_help + exit 1 ;; esac done +## Set log level for all child commands +export LOG_LEVEL + # Base command -BASE_CMD="./yarn-project/end-to-end/scripts/native_network_test.sh \ +BASE_CMD="INTERLEAVED=$INTERLEAVED ./yarn-project/end-to-end/scripts/native_network_test.sh \ $TEST_SCRIPT \ ./deploy-l1-contracts.sh \ ./deploy-l2-contracts.sh \ ./boot-node.sh \ ./ethereum.sh \ - \"./prover-node.sh false\" \ + \"./prover-node.sh 8078 false\" \ ./pxe.sh \ - ./transaction-bot.sh" - -# Generate validator commands -for ((i=0; i/dev/null || true -# Start a new tmux session -tmux new-session -d -s "$session_name" +# Start a new tmux session with log level set +tmux new-session -d -s "$session_name" -e LOG_LEVEL=${LOG_LEVEL:-"debug"} shift 1 commands=("$@") diff --git a/spartan/aztec-network/templates/_helpers.tpl b/spartan/aztec-network/templates/_helpers.tpl index 2eef45fdbb0..2fcc4b7b974 100644 --- a/spartan/aztec-network/templates/_helpers.tpl +++ b/spartan/aztec-network/templates/_helpers.tpl @@ -74,6 +74,14 @@ http://{{ include "aztec-network.fullname" . }}-boot-node-0.{{ include "aztec-ne {{- end -}} {{- end -}} +{{- define "aztec-network.validatorUrl" -}} +{{- if .Values.validator.externalTcpHost -}} +http://{{ .Values.validator.externalTcpHost }}:{{ .Values.validator.service.nodePort }} +{{- else -}} +http://{{ include "aztec-network.fullname" . }}-validator-0.{{ include "aztec-network.fullname" . }}-validator.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.validator.service.nodePort }} +{{- end -}} +{{- end -}} + {{- define "aztec-network.metricsHost" -}} http://{{ include "aztec-network.fullname" . }}-metrics.{{ .Release.Namespace }} {{- end -}} diff --git a/spartan/aztec-network/templates/pxe.yaml b/spartan/aztec-network/templates/pxe.yaml index 50d0d1fb418..0ae3702943e 100644 --- a/spartan/aztec-network/templates/pxe.yaml +++ b/spartan/aztec-network/templates/pxe.yaml @@ -28,6 +28,16 @@ spec: echo "Waiting for boot node..." sleep 5 done + - name: wait-for-validator-service + image: {{ .Values.images.curl.image }} + command: + - /bin/sh + - -c + - | + until curl --head --silent {{ include "aztec-network.validatorUrl" . }}/status; do + echo "Waiting for validator service..." + sleep 5 + done containers: - name: pxe image: "{{ .Values.images.aztec.image }}" diff --git a/yarn-project/Earthfile b/yarn-project/Earthfile index 6b6abd466ed..0e6fa4fb698 100644 --- a/yarn-project/Earthfile +++ b/yarn-project/Earthfile @@ -289,14 +289,16 @@ network-test-fake-proofs: COPY ../scripts/+scripts/run_interleaved.sh scripts/run_interleaved.sh WORKDIR /usr/src/yarn-project # All script arguments are in the end-to-end/scripts/native-network folder + ENV LOG_LEVEL=verbose RUN INTERLEAVED=true end-to-end/scripts/native_network_test.sh \ ./test-transfer.sh \ ./deploy-l1-contracts.sh \ ./deploy-l2-contracts.sh \ ./boot-node.sh \ ./ethereum.sh \ - "./prover-node.sh false" \ - ./pxe.sh + "./prover-node.sh 8078 false" \ + ./pxe.sh \ + "./validator.sh 8081 40401" publish-npm: FROM +build diff --git a/yarn-project/archiver/src/archiver/archiver.ts b/yarn-project/archiver/src/archiver/archiver.ts index d40be435fc1..15cf33ae20f 100644 --- a/yarn-project/archiver/src/archiver/archiver.ts +++ b/yarn-project/archiver/src/archiver/archiver.ts @@ -537,8 +537,14 @@ export class Archiver implements ArchiveSource { if (number === 'latest') { number = await this.store.getSynchedL2BlockNumber(); } - const headers = await this.store.getBlockHeaders(number, 1); - return headers.length === 0 ? undefined : headers[0]; + try { + const headers = await this.store.getBlockHeaders(number, 1); + return headers.length === 0 ? undefined : headers[0]; + } catch (e) { + // If the latest is 0, then getBlockHeaders will throw an error + this.log.error(`getBlockHeader: error fetching block number: ${number}`); + return undefined; + } } public getTxEffect(txHash: TxHash): Promise { diff --git a/yarn-project/aztec-node/src/aztec-node/config.ts b/yarn-project/aztec-node/src/aztec-node/config.ts index 1280f96c0d7..2bd36b1052e 100644 --- a/yarn-project/aztec-node/src/aztec-node/config.ts +++ b/yarn-project/aztec-node/src/aztec-node/config.ts @@ -22,9 +22,6 @@ export type AztecNodeConfig = ArchiverConfig & WorldStateConfig & Pick & P2PConfig & { - /** Whether the sequencer is disabled for this node. */ - disableSequencer: boolean; - /** Whether the validator is disabled for this node */ disableValidator: boolean; }; @@ -36,11 +33,6 @@ export const aztecNodeConfigMappings: ConfigMappingsType = { ...proverClientConfigMappings, ...worldStateConfigMappings, ...p2pConfigMappings, - disableSequencer: { - env: 'SEQ_DISABLED', - description: 'Whether the sequencer is disabled for this node.', - ...booleanConfigHelper(), - }, disableValidator: { env: 'VALIDATOR_DISABLED', description: 'Whether the validator is disabled for this node.', diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 34b8442158d..89942fb2330 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -164,7 +164,7 @@ export class AztecNodeService implements AztecNode { const validatorClient = createValidatorClient(config, p2pClient, telemetry); // now create the sequencer - const sequencer = config.disableSequencer + const sequencer = config.disableValidator ? undefined : await SequencerClient.new( config, diff --git a/yarn-project/aztec/src/cli/cmds/start_node.ts b/yarn-project/aztec/src/cli/cmds/start_node.ts index 80d538c675d..37fcb45385a 100644 --- a/yarn-project/aztec/src/cli/cmds/start_node.ts +++ b/yarn-project/aztec/src/cli/cmds/start_node.ts @@ -64,7 +64,7 @@ export const startNode = async ( } if (!options.sequencer) { - nodeConfig.disableSequencer = true; + nodeConfig.disableValidator = true; } else { const sequencerConfig = extractNamespacedOptions(options, 'sequencer'); let account; diff --git a/yarn-project/end-to-end/scripts/native-network/.gitignore b/yarn-project/end-to-end/scripts/native-network/.gitignore index 8b5dc6656c3..43314c6eb25 100644 --- a/yarn-project/end-to-end/scripts/native-network/.gitignore +++ b/yarn-project/end-to-end/scripts/native-network/.gitignore @@ -1,3 +1,3 @@ -l1-contracts.env -l2-contracts.env -*.log \ No newline at end of file +state/*.env +state/*.json +*.log diff --git a/yarn-project/end-to-end/scripts/native-network/boot-node.sh b/yarn-project/end-to-end/scripts/native-network/boot-node.sh index ecc5cf83a20..379a6842ec6 100755 --- a/yarn-project/end-to-end/scripts/native-network/boot-node.sh +++ b/yarn-project/end-to-end/scripts/native-network/boot-node.sh @@ -10,9 +10,9 @@ exec > >(tee -a "$(dirname $0)/logs/${SCRIPT_NAME}.log") 2> >(tee -a "$(dirname # Starts the Boot Node # Set environment variables -export PORT="8080" -export LOG_LEVEL="debug" -export DEBUG="aztec:*,-aztec:avm_simulator:*" +export PORT=${PORT:-"8080"} +export DEBUG=${DEBUG:-"aztec:*,-aztec:avm_simulator:*"} +export LOG_LEVEL=${LOG_LEVEL:-"debug"} export ETHEREUM_HOST="http://127.0.0.1:8545" export P2P_ENABLED="true" export VALIDATOR_DISABLED="true" @@ -22,16 +22,18 @@ export P2P_TCP_ANNOUNCE_ADDR="127.0.0.1:40400" export P2P_UDP_ANNOUNCE_ADDR="127.0.0.1:40400" export P2P_TCP_LISTEN_ADDR="0.0.0.0:40400" export P2P_UDP_LISTEN_ADDR="0.0.0.0:40400" +export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="" +export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="" export VALIDATOR_PRIVATE_KEY="0x47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a" REPO=$(git rev-parse --show-toplevel) echo "Waiting for l1 contracts to be deployed..." -until [ -f "$REPO"/yarn-project/end-to-end/scripts/native-network/l1-contracts.env ] ; do +until [ -f "$REPO"/yarn-project/end-to-end/scripts/native-network/state/l1-contracts.env ] ; do sleep 1 done echo "Done waiting." -source "$REPO"/yarn-project/end-to-end/scripts/native-network/l1-contracts.env +source "$REPO"/yarn-project/end-to-end/scripts/native-network/state/l1-contracts.env function filter_noise() { grep -Ev "node_getProvenBlockNumber|getBlocks|Last block mined|Running random nodes query|Not creating block because not enough txs in the pool|Peers to connect" diff --git a/yarn-project/end-to-end/scripts/native-network/deploy-l1-contracts.sh b/yarn-project/end-to-end/scripts/native-network/deploy-l1-contracts.sh index 319dc7babbe..413b30d1b25 100755 --- a/yarn-project/end-to-end/scripts/native-network/deploy-l1-contracts.sh +++ b/yarn-project/end-to-end/scripts/native-network/deploy-l1-contracts.sh @@ -10,6 +10,14 @@ exec > >(tee -a "$(dirname $0)/logs/${SCRIPT_NAME}.log") 2> >(tee -a "$(dirname set -eu +# Check for validator addresses +if [ $# -gt 0 ]; then + INIT_VALIDATORS="true" + VALIDATOR_ADDRESSES="$1" +else + INIT_VALIDATORS="false" +fi + echo "Waiting for Anvil to be up at port 8545..." until curl -s -X POST -H 'Content-Type: application/json' \ --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \ @@ -20,7 +28,11 @@ echo "Done waiting." # Run the deploy-l1-contracts command and capture the output export ETHEREUM_HOST="http://127.0.0.1:8545" -output=$(node --no-warnings $(git rev-parse --show-toplevel)/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --salt 1337) +if [ "$INIT_VALIDATORS" = "true" ]; then + output=$(node --no-warnings $(git rev-parse --show-toplevel)/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --validators "$VALIDATOR_ADDRESSES" --salt 1337) +else + output=$(node --no-warnings $(git rev-parse --show-toplevel)/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --salt 1337) +fi echo "$output" @@ -32,8 +44,8 @@ OUTBOX_CONTRACT_ADDRESS=$(echo "$output" | grep -oP 'L2 -> L1 Outbox Address: \K FEE_JUICE_CONTRACT_ADDRESS=$(echo "$output" | grep -oP 'Fee Juice Address: \K0x[a-fA-F0-9]{40}') FEE_JUICE_PORTAL_CONTRACT_ADDRESS=$(echo "$output" | grep -oP 'Fee Juice Portal Address: \K0x[a-fA-F0-9]{40}') -# Save contract addresses to l1-contracts.env -cat << EOCONFIG > $(git rev-parse --show-toplevel)/yarn-project/end-to-end/scripts/native-network/l1-contracts.env +# Save contract addresses to state/l1-contracts.env +cat << EOCONFIG > $(git rev-parse --show-toplevel)/yarn-project/end-to-end/scripts/native-network/state/l1-contracts.env export ROLLUP_CONTRACT_ADDRESS=$ROLLUP_CONTRACT_ADDRESS export REGISTRY_CONTRACT_ADDRESS=$REGISTRY_CONTRACT_ADDRESS export INBOX_CONTRACT_ADDRESS=$INBOX_CONTRACT_ADDRESS @@ -42,4 +54,4 @@ export FEE_JUICE_CONTRACT_ADDRESS=$FEE_JUICE_CONTRACT_ADDRESS export FEE_JUICE_PORTAL_CONTRACT_ADDRESS=$FEE_JUICE_PORTAL_CONTRACT_ADDRESS EOCONFIG -echo "Contract addresses saved to l1-contracts.env" +echo "Contract addresses saved to state/l1-contracts.env" diff --git a/yarn-project/end-to-end/scripts/native-network/deploy-l2-contracts.sh b/yarn-project/end-to-end/scripts/native-network/deploy-l2-contracts.sh index e015b8a862b..4c32ca45699 100755 --- a/yarn-project/end-to-end/scripts/native-network/deploy-l2-contracts.sh +++ b/yarn-project/end-to-end/scripts/native-network/deploy-l2-contracts.sh @@ -28,10 +28,11 @@ export PXE_URL="http://127.0.0.1:8079" node --no-warnings $(git rev-parse --show-toplevel)/yarn-project/aztec/dest/bin/index.js setup-protocol-contracts $ARGS echo "Deployed L2 contracts" # Use file just as done signal -echo "" > l2-contracts.env -echo "Wrote to l2-contracts.env to signal completion" -sleep 5 -function close_tmux_pane() { - tmux kill-pane -t $(tmux display -p '#{pane_id}') -} -close_tmux_pane 2>/dev/null || true \ No newline at end of file +echo "" > state/l2-contracts.env +echo "Wrote to state/l2-contracts.env to signal completion" + +# sleep 5 +# function close_tmux_pane() { +# tmux kill-pane -t $(tmux display -p '#{pane_id}') +# } +# close_tmux_pane 2>/dev/null || true \ No newline at end of file diff --git a/yarn-project/end-to-end/scripts/native-network/ethereum.sh b/yarn-project/end-to-end/scripts/native-network/ethereum.sh index 0bedc54556e..417bf4ae349 100755 --- a/yarn-project/end-to-end/scripts/native-network/ethereum.sh +++ b/yarn-project/end-to-end/scripts/native-network/ethereum.sh @@ -1,8 +1,10 @@ #!/bin/bash set -eu +set -o pipefail # Get the name of the script without the path and extension SCRIPT_NAME=$(basename "$0" .sh) +PORT=8545 # Redirect stdout and stderr to .log while also printing to the console exec > >(tee -a "$(dirname $0)/logs/${SCRIPT_NAME}.log") 2> >(tee -a "$(dirname $0)/logs/${SCRIPT_NAME}.log" >&2) @@ -11,12 +13,11 @@ exec > >(tee -a "$(dirname $0)/logs/${SCRIPT_NAME}.log") 2> >(tee -a "$(dirname # Ensure anvil is installed command -v anvil >/dev/null 2>&1 || { echo >&2 "Anvil is not installed. Aborting."; exit 1; } - function report_when_anvil_up() { - echo "Starting anvil." + echo "Starting anvil with port $PORT" until curl -s -X POST -H 'Content-Type: application/json' \ --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \ - http://127.0.0.1:8545 2>/dev/null | grep -q 'result' ; do + http://127.0.0.1:$PORT 2>/dev/null | grep -q 'result' ; do sleep 1 done echo "Anvil has started." @@ -26,10 +27,30 @@ function filter_noise() { } report_when_anvil_up & -# Start Anvil with specified parameters -anvil \ - --host 0.0.0.0 \ - --block-time 12 \ - --chain-id 31337 \ - --gas-limit 1000000000 \ - -p 8545 2>&1 | filter_noise +STATE_DIR="$(git rev-parse --show-toplevel)/yarn-project/end-to-end/scripts/native-network/state" +STATE_FILE="$STATE_DIR/state.json" + +mkdir -p "$STATE_DIR" + +if [ -f "$STATE_FILE" ]; then + echo "State file found. Loading existing state." + anvil \ + --host 0.0.0.0 \ + --load-state "$STATE_DIR" \ + --dump-state "$STATE_DIR" \ + --block-time 12 \ + --chain-id 31337 \ + --gas-limit 1000000000 \ + --accounts 3 \ + -p $PORT 2>&1 | filter_noise +else + echo "No state file found. Starting with a fresh state and enabling state dumping." + anvil \ + --host 0.0.0.0 \ + --dump-state "$STATE_DIR" \ + --block-time 12 \ + --chain-id 31337 \ + --gas-limit 1000000000 \ + --accounts 3 \ + -p $PORT 2>&1 | filter_noise +fi \ No newline at end of file diff --git a/yarn-project/end-to-end/scripts/native-network/prover-node.sh b/yarn-project/end-to-end/scripts/native-network/prover-node.sh index a7e6218e9c7..734916441e3 100755 --- a/yarn-project/end-to-end/scripts/native-network/prover-node.sh +++ b/yarn-project/end-to-end/scripts/native-network/prover-node.sh @@ -1,10 +1,12 @@ #!/bin/bash set -eu -export PROVER_REAL_PROOFS="$1" # Get the name of the script without the path and extension SCRIPT_NAME=$(basename "$0" .sh) +PORT="$1" +export PROVER_REAL_PROOFS="$2" + # Redirect stdout and stderr to .log while also printing to the console exec > >(tee -a "$(dirname $0)/logs/${SCRIPT_NAME}.log") 2> >(tee -a "$(dirname $0)/logs/${SCRIPT_NAME}.log" >&2) @@ -12,12 +14,16 @@ exec > >(tee -a "$(dirname $0)/logs/${SCRIPT_NAME}.log") 2> >(tee -a "$(dirname REPO=$(git rev-parse --show-toplevel) echo "Waiting for l1 contracts to be deployed..." -until [ -f "$REPO"/yarn-project/end-to-end/scripts/native-network/l1-contracts.env ] ; do +until [ -f "$REPO"/yarn-project/end-to-end/scripts/native-network/state/l1-contracts.env ] ; do + sleep 1 +done +echo "Waiting for Aztec Node..." +until curl -s http://127.0.0.1:8080/status >/dev/null ; do sleep 1 done echo "Done waiting." -source "$REPO"/yarn-project/end-to-end/scripts/native-network/l1-contracts.env +source "$REPO"/yarn-project/end-to-end/scripts/native-network/state/l1-contracts.env # Get node info from the boot node output=$(node --no-warnings "$REPO"/yarn-project/aztec/dest/bin/index.js get-node-info -u http://127.0.0.1:8080) @@ -26,13 +32,14 @@ output=$(node --no-warnings "$REPO"/yarn-project/aztec/dest/bin/index.js get-nod export BOOTSTRAP_NODES=$(echo "$output" | grep -oP 'Node ENR: \K.*') # Set environment variables -export LOG_LEVEL="debug" -export DEBUG="aztec:*" +export LOG_LEVEL=${LOG_LEVEL:-"debug"} +export DEBUG=${DEBUG:-"aztec:*"} export ETHEREUM_HOST="http://127.0.0.1:8545" export PROVER_AGENT_ENABLED="true" -export PROVER_PUBLISHER_PRIVATE_KEY="0xdbda1821b80551c9d65939329250298aa3472ba22feea921c0cf5d620ea67b97" +export PROVER_PUBLISHER_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" export PROVER_COORDINATION_NODE_URL="http://127.0.0.1:8080" export AZTEC_NODE_URL="http://127.0.0.1:8080" +export PROVER_JOB_SOURCE_URL="http://127.0.0.1:$PORT" # Start the Prover Node with the prover and archiver -node --no-warnings "$REPO"/yarn-project/aztec/dest/bin/index.js start --port=8078 --prover-node --prover --archiver +node --no-warnings "$REPO"/yarn-project/aztec/dest/bin/index.js start --port="$PORT" --prover-node --prover --archiver diff --git a/yarn-project/end-to-end/scripts/native-network/pxe.sh b/yarn-project/end-to-end/scripts/native-network/pxe.sh index 10649d1efa9..e1ec0eab112 100755 --- a/yarn-project/end-to-end/scripts/native-network/pxe.sh +++ b/yarn-project/end-to-end/scripts/native-network/pxe.sh @@ -11,7 +11,7 @@ exec > >(tee -a "$(dirname $0)/logs/${SCRIPT_NAME}.log") 2> >(tee -a "$(dirname # Set environment variables export ETHEREUM_HOST="http://127.0.0.1:8545" export AZTEC_NODE_URL="http://127.0.0.1:8080" -export LOG_LEVEL="debug" +export LOG_LEVEL=${LOG_LEVEL:-"debug"} export DEBUG="aztec:*" echo "Waiting for Aztec Node..." @@ -20,6 +20,14 @@ until curl -s http://127.0.0.1:8080/status >/dev/null ; do done echo "Done waiting." +# We need to also wait for the validator, as the initial node cannot +# Produce blocks on it's own +echo "Waiting for Validator 0..." +until curl -s http://127.0.0.1:8081/status >/dev/null ; do + sleep 1 +done +echo "Done waiting." + function filter_noise() { grep -v node_getProvenBlockNumber } diff --git a/yarn-project/end-to-end/scripts/native-network/state/.gitignore b/yarn-project/end-to-end/scripts/native-network/state/.gitignore new file mode 100644 index 00000000000..b7887cb1903 --- /dev/null +++ b/yarn-project/end-to-end/scripts/native-network/state/.gitignore @@ -0,0 +1 @@ +state.json \ No newline at end of file diff --git a/yarn-project/end-to-end/scripts/native-network/state/README.md b/yarn-project/end-to-end/scripts/native-network/state/README.md new file mode 100644 index 00000000000..3ede1fe3252 --- /dev/null +++ b/yarn-project/end-to-end/scripts/native-network/state/README.md @@ -0,0 +1 @@ +This README ensures this folder exists in git. Ethereum state is written out here. \ No newline at end of file diff --git a/yarn-project/end-to-end/scripts/native-network/test-transfer.sh b/yarn-project/end-to-end/scripts/native-network/test-transfer.sh index d6044e4487a..93e218a045e 100755 --- a/yarn-project/end-to-end/scripts/native-network/test-transfer.sh +++ b/yarn-project/end-to-end/scripts/native-network/test-transfer.sh @@ -22,13 +22,13 @@ until curl -s -X POST -H 'content-type: application/json' \ sleep 1 done echo "Waiting for l2 contracts to be deployed..." -until [ -f "$REPO"/yarn-project/end-to-end/scripts/native-network/l2-contracts.env ] ; do +until [ -f "$REPO"/yarn-project/end-to-end/scripts/native-network/state/l2-contracts.env ] ; do sleep 1 done echo "Done waiting." export DEBUG="aztec:*" -export LOG_LEVEL=debug +export LOG_LEVEL=${LOG_LEVEL:-"debug"} export PXE_URL=http://localhost:8079 cd $(git rev-parse --show-toplevel)/yarn-project/end-to-end -yarn test src/spartan/transfer.test.ts \ No newline at end of file +yarn test src/spartan/transfer.test.ts diff --git a/yarn-project/end-to-end/scripts/native-network/test.sh b/yarn-project/end-to-end/scripts/native-network/test.sh new file mode 100755 index 00000000000..ac034cfe99e --- /dev/null +++ b/yarn-project/end-to-end/scripts/native-network/test.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -eu + +TEST=$1 +# Get the name of the script without the path and extension +SCRIPT_NAME=$(basename "$0" .sh) + +# Redirect stdout and stderr to .log while also printing to the console +exec > >(tee -a "$(dirname $0)/logs/${SCRIPT_NAME}.log") 2> >(tee -a "$(dirname $0)/logs/${SCRIPT_NAME}.log" >&2) + +REPO=$(git rev-parse --show-toplevel) +# Run our test assuming the port in pxe.sh +# Wait for the Aztec Node to be ready +echo "Waiting for Aztec Node..." +until curl -s http://127.0.0.1:8080/status >/dev/null ; do + sleep 1 +done +echo "Waiting for PXE service..." +until curl -s -X POST -H 'content-type: application/json' \ + -d '{"jsonrpc":"2.0","method":"pxe_getNodeInfo","params":[],"id":67}' \ + http://127.0.0.1:8079 | grep -q '"enr:-'; do + sleep 1 +done +echo "Waiting for l2 contracts to be deployed..." +until [ -f "$REPO"/yarn-project/end-to-end/scripts/native-network/state/l2-contracts.env ] ; do + sleep 1 +done +echo "Done waiting." + +export DEBUG="aztec:*" +export LOG_LEVEL=${LOG_LEVEL:-"debug"} +export PXE_URL=http://localhost:8079 +export ETHEREUM_HOST=http://localhost:8545 +cd $(git rev-parse --show-toplevel)/yarn-project/end-to-end +yarn test "$TEST" diff --git a/yarn-project/end-to-end/scripts/native-network/transaction-bot.sh b/yarn-project/end-to-end/scripts/native-network/transaction-bot.sh index 37abb2725f9..a1fc7cc641a 100755 --- a/yarn-project/end-to-end/scripts/native-network/transaction-bot.sh +++ b/yarn-project/end-to-end/scripts/native-network/transaction-bot.sh @@ -21,7 +21,7 @@ until curl -s -X POST -H 'content-type: application/json' \ sleep 1 done echo "Waiting for l2 contracts to be deployed..." -until [ -f "$REPO"/yarn-project/end-to-end/scripts/native-network/l2-contracts.env ] ; do +until [ -f "$REPO"/yarn-project/end-to-end/scripts/native-network/state/l2-contracts.env ] ; do sleep 1 done echo "Done waiting." @@ -30,13 +30,14 @@ echo "Done waiting." export ETHEREUM_HOST="http://127.0.0.1:8545" export AZTEC_NODE_URL="http://127.0.0.1:8080" export LOG_JSON="1" -export LOG_LEVEL="debug" +export LOG_LEVEL=${LOG_LEVEL:-"debug"} export DEBUG="aztec:*" export BOT_PRIVATE_KEY="0xcafe" export BOT_TX_INTERVAL_SECONDS="5" export BOT_PRIVATE_TRANSFERS_PER_TX="1" export BOT_PUBLIC_TRANSFERS_PER_TX="0" export BOT_NO_WAIT_FOR_TRANSFERS="true" +export BOT_FOLLOW_CHAIN="NONE" export BOT_NO_START="false" export PXE_PROVER_ENABLED="false" export PROVER_REAL_PROOFS="false" diff --git a/yarn-project/end-to-end/scripts/native-network/validator.sh b/yarn-project/end-to-end/scripts/native-network/validator.sh index 5c9c9fec4c4..26d5e983263 100755 --- a/yarn-project/end-to-end/scripts/native-network/validator.sh +++ b/yarn-project/end-to-end/scripts/native-network/validator.sh @@ -7,17 +7,19 @@ SCRIPT_NAME=$(basename "$0" .sh) # Redirect stdout and stderr to .log while also printing to the console exec > >(tee -a "$(dirname $0)/logs/${SCRIPT_NAME}.log") 2> >(tee -a "$(dirname $0)/logs/${SCRIPT_NAME}.log" >&2) +# PORTS PORT="$1" +P2P_PORT="$2" # Starts the Validator Node REPO=$(git rev-parse --show-toplevel) echo "Waiting for l1 contracts to be deployed..." -until [ -f "$REPO"/yarn-project/end-to-end/scripts/native-network/l1-contracts.env ] ; do +until [ -f "$REPO"/yarn-project/end-to-end/scripts/native-network/state/l1-contracts.env ] ; do sleep 1 done -source "$REPO"/yarn-project/end-to-end/scripts/native-network/l1-contracts.env +source "$REPO"/yarn-project/end-to-end/scripts/native-network/state/l1-contracts.env echo "Waiting for Aztec Node..." until curl -s http://127.0.0.1:8080/status >/dev/null ; do @@ -33,24 +35,25 @@ output=$(node --no-warnings "$REPO"/yarn-project/aztec/dest/bin/index.js get-nod # Extract boot node ENR export BOOTSTRAP_NODES=$(echo "$output" | grep -oP 'Node ENR: \K.*') +echo "BOOTSTRAP_NODES: $BOOTSTRAP_NODES" # Generate a private key for the validator json_account=$(node --no-warnings "$REPO"/yarn-project/aztec/dest/bin/index.js generate-l1-account) export ADDRESS=$(echo $json_account | jq -r '.address') +export LOG_LEVEL=${LOG_LEVEL:-"debug"} export VALIDATOR_PRIVATE_KEY=$(echo $json_account | jq -r '.privateKey') export L1_PRIVATE_KEY=$VALIDATOR_PRIVATE_KEY export SEQ_PUBLISHER_PRIVATE_KEY=$VALIDATOR_PRIVATE_KEY -export LOG_LEVEL="debug" -export DEBUG="aztec:*,-aztec:avm_simulator:*" +export DEBUG=${DEBUG:-"aztec:*,-aztec:avm_simulator:*"} export ETHEREUM_HOST="http://127.0.0.1:8545" export P2P_ENABLED="true" export VALIDATOR_DISABLED="false" export SEQ_MAX_SECONDS_BETWEEN_BLOCKS="0" export SEQ_MIN_TX_PER_BLOCK="1" -export P2P_TCP_ANNOUNCE_ADDR="127.0.0.1:40400" -export P2P_UDP_ANNOUNCE_ADDR="127.0.0.1:40400" -export P2P_TCP_LISTEN_ADDR="0.0.0.0:40400" -export P2P_UDP_LISTEN_ADDR="0.0.0.0:40400" +export P2P_TCP_ANNOUNCE_ADDR="127.0.0.1:$P2P_PORT" +export P2P_UDP_ANNOUNCE_ADDR="127.0.0.1:$P2P_PORT" +export P2P_TCP_LISTEN_ADDR="0.0.0.0:$P2P_PORT" +export P2P_UDP_LISTEN_ADDR="0.0.0.0:$P2P_PORT" # Add L1 validator node --no-warnings "$REPO"/yarn-project/aztec/dest/bin/index.js add-l1-validator --validator $ADDRESS --rollup $ROLLUP_CONTRACT_ADDRESS @@ -58,3 +61,4 @@ node --no-warnings "$REPO"/yarn-project/aztec/dest/bin/index.js add-l1-validator node --no-warnings "$REPO"/yarn-project/aztec/dest/bin/index.js fast-forward-epochs --rollup $ROLLUP_CONTRACT_ADDRESS --count 1 # Start the Validator Node with the sequencer and archiver node --no-warnings "$REPO"/yarn-project/aztec/dest/bin/index.js start --port="$PORT" --node --archiver --sequencer + diff --git a/yarn-project/end-to-end/scripts/native-network/validators.sh b/yarn-project/end-to-end/scripts/native-network/validators.sh new file mode 100755 index 00000000000..6a9ac7f4f40 --- /dev/null +++ b/yarn-project/end-to-end/scripts/native-network/validators.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Takes a number of validators to start, starting from port 8080. Calls out to validator.sh +set -eu + +# Get the name of the script without the path and extension +SCRIPT_NAME=$(basename "$0" .sh) + +# Redirect stdout and stderr to .log while also printing to the console +exec > >(tee -a "$(dirname $0)/logs/${SCRIPT_NAME}.log") 2> >(tee -a "$(dirname $0)/logs/${SCRIPT_NAME}.log" >&2) + +NUM_VALIDATORS="$1" + +# enter script dir +cd "$(dirname "${BASH_SOURCE[0]}")" + +CMD=() + +# Generate validator commands +for ((i=0; i { // Create a new node and measure how much time it takes it to sync const dataDirectory = makeDataDirectory(chainLength); context.logger.info(`Set up data directory at ${dataDirectory}`); - const nodeConfig: AztecNodeConfig = { ...context.config, disableSequencer: true, dataDirectory }; + const nodeConfig: AztecNodeConfig = { ...context.config, disableValidator: true, dataDirectory }; const [nodeSyncTime, node] = await elapsed(async () => { const node = await AztecNodeService.createAndSync(nodeConfig); // call getPublicStorageAt (which calls #getWorldState, which calls #syncWorldState) to force a sync with diff --git a/yarn-project/end-to-end/src/benchmarks/bench_publish_rollup.test.ts b/yarn-project/end-to-end/src/benchmarks/bench_publish_rollup.test.ts index 67b3a8f1c7f..0463c9c8ac9 100644 --- a/yarn-project/end-to-end/src/benchmarks/bench_publish_rollup.test.ts +++ b/yarn-project/end-to-end/src/benchmarks/bench_publish_rollup.test.ts @@ -36,7 +36,7 @@ describe('benchmarks/publish_rollup', () => { // and call getPublicStorageAt (which calls #getWorldState, which calls #syncWorldState) to force a sync with // world state to ensure the node has caught up context.logger.info(`Starting new aztec node`); - const node = await AztecNodeService.createAndSync({ ...context.config, disableSequencer: true }); + const node = await AztecNodeService.createAndSync({ ...context.config, disableValidator: true }); await node.getPublicStorageAt(AztecAddress.random(), Fr.random(), 'latest'); // Spin up a new pxe and sync it, we'll use it to test sync times of new accounts for the last block diff --git a/yarn-project/end-to-end/src/e2e_synching.test.ts b/yarn-project/end-to-end/src/e2e_synching.test.ts index 1a2952b30bb..f635a7ada80 100644 --- a/yarn-project/end-to-end/src/e2e_synching.test.ts +++ b/yarn-project/end-to-end/src/e2e_synching.test.ts @@ -469,7 +469,7 @@ describe('e2e_synching', () => { // All the blocks have been "re-played" and we are now to simply get a new node up to speed const timer = new Timer(); const freshNode = await AztecNodeService.createAndSync( - { ...opts.config!, disableSequencer: true, disableValidator: true }, + { ...opts.config!, disableValidator: true }, new NoopTelemetryClient(), ); const syncTime = timer.s(); diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts index e66dc17d510..6ccad86f600 100644 --- a/yarn-project/foundation/src/config/env_var.ts +++ b/yarn-project/foundation/src/config/env_var.ts @@ -76,10 +76,12 @@ export type EnvVar = | 'P2P_REQRESP_INDIVIDUAL_REQUEST_TIMEOUT_MS' | 'P2P_REQRESP_OVERALL_REQUEST_TIMEOUT_MS' | 'P2P_SEVERE_PEER_PENALTY_BLOCK_LENGTH' + | 'P2P_TCP_LISTEN_ADDR' | 'P2P_TCP_ANNOUNCE_ADDR' | 'P2P_TX_POOL_KEEP_PROVEN_FOR' | 'P2P_TX_PROTOCOL' | 'P2P_UDP_ANNOUNCE_ADDR' + | 'P2P_UDP_LISTEN_ADDR' | 'PEER_ID_PRIVATE_KEY' | 'PROOF_VERIFIER_L1_START_BLOCK' | 'PROOF_VERIFIER_POLL_INTERVAL_MS' @@ -112,7 +114,6 @@ export type EnvVar = | 'ROLLUP_CONTRACT_ADDRESS' | 'SEQ_ALLOWED_SETUP_FN' | 'SEQ_ALLOWED_TEARDOWN_FN' - | 'SEQ_DISABLED' | 'SEQ_MAX_BLOCK_SIZE_IN_BYTES' | 'SEQ_MAX_SECONDS_BETWEEN_BLOCKS' | 'SEQ_MAX_TX_PER_BLOCK' @@ -122,12 +123,10 @@ export type EnvVar = | 'SEQ_PUBLISHER_PRIVATE_KEY' | 'SEQ_REQUIRED_CONFIRMATIONS' | 'SEQ_TX_POLLING_INTERVAL_MS' - | 'TCP_LISTEN_ADDR' | 'TELEMETRY' | 'TEST_ACCOUNTS' | 'TX_GOSSIP_VERSION' | 'TXE_PORT' - | 'UDP_LISTEN_ADDR' | 'VALIDATOR_ATTESTATIONS_POOLING_INTERVAL_MS' | 'VALIDATOR_ATTESTATIONS_WAIT_TIMEOUT_MS' | 'VALIDATOR_DISABLED' diff --git a/yarn-project/p2p/src/config.ts b/yarn-project/p2p/src/config.ts index d6d4d743656..88ff1af0939 100644 --- a/yarn-project/p2p/src/config.ts +++ b/yarn-project/p2p/src/config.ts @@ -174,12 +174,12 @@ export const p2pConfigMappings: ConfigMappingsType = { ...numberConfigHelper(1_000), }, tcpListenAddress: { - env: 'TCP_LISTEN_ADDR', + env: 'P2P_TCP_LISTEN_ADDR', defaultValue: '0.0.0.0:40400', description: 'The listen address for TCP. Format: :.', }, udpListenAddress: { - env: 'UDP_LISTEN_ADDR', + env: 'P2P_UDP_LISTEN_ADDR', defaultValue: '0.0.0.0:40400', description: 'The listen address for UDP. Format: :.', },