From 8021eda6b5c6e6c518ff38bacdc828fcfab09465 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 8 Aug 2024 10:43:23 +0100 Subject: [PATCH] feat: merge devnet chagnes to master (#7822) Brings the latest set of devnet fixes into the master branch --------- Co-authored-by: PhilWindle Co-authored-by: Aztec Bot <49558828+AztecBot@users.noreply.github.com> Co-authored-by: sirasistant Co-authored-by: Maxim Vezenov Co-authored-by: Facundo Co-authored-by: AztecBot Co-authored-by: PhilWindle <60546371+PhilWindle@users.noreply.github.com> Co-authored-by: Santiago Palladino Co-authored-by: spypsy Co-authored-by: spypsy --- .github/workflows/devnet-deploys.yml | 62 ++-- aztec-up/bin/.aztec-run | 37 ++- aztec-up/bin/aztec | 38 ++- aztec-up/bin/aztec-install | 31 +- cspell.json | 1 + docker-compose.yml | 9 +- iac/mainnet-fork/redeploy | 2 +- .../archiver/src/archiver/archiver.ts | 9 +- yarn-project/archiver/src/archiver/config.ts | 70 ++--- .../archiver/src/archiver/instrumentation.ts | 27 +- yarn-project/archiver/src/index.ts | 2 +- .../aztec-node/src/aztec-node/config.ts | 47 +-- .../aztec-node/src/aztec-node/server.ts | 2 +- yarn-project/aztec/package.json | 2 + .../aztec/src/cli/aztec_start_options.ts | 294 ++++++++++++++++++ yarn-project/aztec/src/cli/cli.ts | 190 +++++------ .../aztec/src/cli/cmds/start_archiver.ts | 13 +- yarn-project/aztec/src/cli/cmds/start_bot.ts | 9 +- yarn-project/aztec/src/cli/cmds/start_node.ts | 94 +++--- .../aztec/src/cli/cmds/start_p2p_bootstrap.ts | 8 +- .../aztec/src/cli/cmds/start_prover_agent.ts | 52 ++-- .../aztec/src/cli/cmds/start_prover_node.ts | 26 +- yarn-project/aztec/src/cli/cmds/start_pxe.ts | 46 +-- yarn-project/aztec/src/cli/cmds/start_txe.ts | 4 +- yarn-project/aztec/src/cli/texts.ts | 107 ------- yarn-project/aztec/src/cli/util.ts | 232 +++++++++----- yarn-project/aztec/src/sandbox.ts | 14 +- yarn-project/aztec/terraform/node/main.tf | 12 +- yarn-project/aztec/terraform/prover/main.tf | 192 ++++++------ .../bb-prover/src/prover/bb_prover.ts | 2 +- .../bb-prover/src/test/test_circuit_prover.ts | 1 - yarn-project/bot/src/config.ts | 119 ++++--- yarn-project/bot/src/index.ts | 2 +- .../circuit-types/src/interfaces/configs.ts | 4 +- .../src/interfaces/prover-client.ts | 51 ++- .../cli/src/cmds/devnet/bootstrap_network.ts | 1 - yarn-project/cli/src/cmds/pxe/call.ts | 20 +- yarn-project/cli/src/cmds/pxe/index.ts | 5 +- .../composed/integration_l1_publisher.test.ts | 6 +- .../uniswap_trade_on_l1_from_l2.test.ts | 2 +- yarn-project/end-to-end/src/e2e_bot.test.ts | 8 +- .../end-to-end/src/e2e_fees/fees_test.ts | 6 +- .../end-to-end/src/e2e_p2p_network.test.ts | 9 +- ...lic_cross_chain_messaging_contract_test.ts | 4 +- .../src/fixtures/snapshot_manager.ts | 12 +- yarn-project/end-to-end/src/fixtures/utils.ts | 16 +- yarn-project/ethereum/src/index.ts | 1 + .../ethereum/src/l1_contract_addresses.ts | 65 ++-- yarn-project/ethereum/src/l1_reader.ts | 40 +++ yarn-project/foundation/package.json | 1 + yarn-project/foundation/src/config/env_var.ts | 97 ++++++ yarn-project/foundation/src/config/index.ts | 115 +++++++ yarn-project/foundation/src/index.ts | 1 + yarn-project/kv-store/src/utils.ts | 11 +- yarn-project/p2p-bootstrap/src/index.ts | 4 +- yarn-project/p2p/src/bootstrap/bootstrap.ts | 11 +- yarn-project/p2p/src/client/p2p_client.ts | 2 +- yarn-project/p2p/src/config.ts | 162 +++++++--- .../p2p/src/service/discv5_service.test.ts | 6 +- .../p2p/src/service/libp2p_service.ts | 16 +- .../p2p/src/tx_pool/aztec_kv_tx_pool.ts | 8 +- yarn-project/prover-client/src/config.ts | 84 ++--- .../src/prover-agent/prover-agent.ts | 37 ++- yarn-project/prover-node/src/config.ts | 30 +- .../prover-node/src/tx-provider/config.ts | 14 +- yarn-project/pxe/src/config/index.ts | 101 ++++-- yarn-project/sequencer-client/src/config.ts | 156 ++++++---- .../src/global_variable_builder/config.ts | 20 -- .../src/global_variable_builder/index.ts | 6 +- .../global_variable_builder/viem-reader.ts | 9 +- .../sequencer-client/src/publisher/config.ts | 74 +++-- .../sequencer-client/src/publisher/index.ts | 2 +- .../src/publisher/viem-tx-sender.ts | 2 +- .../sequencer-client/src/sequencer/metrics.ts | 57 +++- .../src/sequencer/sequencer.ts | 8 +- yarn-project/telemetry-client/package.json | 1 + .../telemetry-client/src/attributes.ts | 2 +- .../src/aztec_resource_detector.ts | 19 ++ yarn-project/telemetry-client/src/config.ts | 29 ++ yarn-project/telemetry-client/src/metrics.ts | 10 +- yarn-project/telemetry-client/src/otel.ts | 45 ++- yarn-project/telemetry-client/src/start.ts | 32 +- .../telemetry-client/src/telemetry.ts | 18 ++ yarn-project/types/src/network/index.ts | 7 + .../world-state/src/synchronizer/config.ts | 31 +- yarn-project/yarn.lock | 62 ++++ 86 files changed, 2164 insertions(+), 1132 deletions(-) create mode 100644 yarn-project/aztec/src/cli/aztec_start_options.ts delete mode 100644 yarn-project/aztec/src/cli/texts.ts create mode 100644 yarn-project/ethereum/src/l1_reader.ts create mode 100644 yarn-project/foundation/src/config/env_var.ts create mode 100644 yarn-project/foundation/src/config/index.ts delete mode 100644 yarn-project/sequencer-client/src/global_variable_builder/config.ts create mode 100644 yarn-project/telemetry-client/src/aztec_resource_detector.ts create mode 100644 yarn-project/telemetry-client/src/config.ts diff --git a/.github/workflows/devnet-deploys.yml b/.github/workflows/devnet-deploys.yml index cb297afb6b2..b6d4509538f 100644 --- a/.github/workflows/devnet-deploys.yml +++ b/.github/workflows/devnet-deploys.yml @@ -77,7 +77,7 @@ jobs: setup: uses: ./.github/workflows/setup-runner.yml with: - username: master + username: ${{ github.actor }} runner_type: builder-x86 secrets: inherit @@ -99,7 +99,6 @@ jobs: pxe_lb_priority_range_start: ${{ steps.set_network_vars.outputs.pxe_lb_priority_range_start }} faucet_lb_priority: ${{ steps.set_network_vars.outputs.faucet_lb_priority }} bot_no_wait: ${{ steps.set_network_vars.outputs.bot_no_wait }} - max_txs_per_block: ${{ steps.set_network_vars.outputs.max_txs_per_block }} steps: - name: Set network vars shell: bash @@ -119,13 +118,12 @@ jobs: echo "pxe_lb_priority_range_start=5100" >> $GITHUB_OUTPUT echo "faucet_lb_priority=601" >> $GITHUB_OUTPUT echo "bot_no_wait=false" >> $GITHUB_OUTPUT - echo "max_txs_per_block=64" >> $GITHUB_OUTPUT elif [ "$BRANCH_NAME" = "provernet" ] then echo "deploy_tag=provernet" >> $GITHUB_OUTPUT echo "branch_name=provernet" >> $GITHUB_OUTPUT echo "network_api_key=PROVERNET_API_KEY" >> $GITHUB_OUTPUT - echo "agents_per_prover=2" >> $GITHUB_OUTPUT + echo "agents_per_prover=4" >> $GITHUB_OUTPUT echo "bot_interval=300" >> $GITHUB_OUTPUT echo "node_tcp_range_start=40200" >> $GITHUB_OUTPUT echo "node_udp_range_start=45200" >> $GITHUB_OUTPUT @@ -133,7 +131,6 @@ jobs: echo "pxe_lb_priority_range_start=5200" >> $GITHUB_OUTPUT echo "faucet_lb_priority=602" >> $GITHUB_OUTPUT echo "bot_no_wait=true" >> $GITHUB_OUTPUT - echo "max_txs_per_block=4" >> $GITHUB_OUTPUT elif [ "$BRANCH_NAME" = "alphanet" ] then echo "deploy_tag=alphanet" >> $GITHUB_OUTPUT @@ -147,7 +144,6 @@ jobs: echo "pxe_lb_priority_range_start=5000" >> $GITHUB_OUTPUT echo "faucet_lb_priority=600" >> $GITHUB_OUTPUT echo "bot_no_wait=false" >> $GITHUB_OUTPUT - echo "max_txs_per_block=64" >> $GITHUB_OUTPUT else echo "Unrecognized Branch!!" exit 1 @@ -190,31 +186,6 @@ jobs: earthly-ci \ --no-output --push ./iac/mainnet-fork+export-mainnet-fork --DIST_TAG=${{ env.DEPLOY_TAG }} - build-aztec-nargo: - needs: set-network - env: - BRANCH_NAME: ${{ needs.set-network.outputs.branch_name }} - DEPLOY_TAG: ${{ needs.set-network.outputs.deploy_tag }} - TF_VAR_DEPLOY_TAG: ${{ needs.set-network.outputs.deploy_tag }} - API_KEY: ${{ secrets[needs.set-network.outputs.network_api_key] }} - TF_VAR_API_KEY: ${{ secrets[needs.set-network.outputs.network_api_key] }} - API_KEY_NAME: ${{ needs.set-network.outputs.network_api_key }} - runs-on: ${{ github.actor }}-x86 - steps: - - uses: actions/checkout@v4 - with: - ref: "${{ env.GIT_COMMIT }}" - fetch-depth: 0 - - uses: ./.github/ci-setup-action - with: - concurrency_key: build-aztec-nargo-${{ github.actor }} - dockerhub_password: "${{ env.DOCKERHUB_PASSWORD }}" - - - name: Build & push aztec nargo image - run: | - earthly-ci --no-output --push ./aztec-nargo+export-aztec-nargo --DIST_TAG=${{ env.DEPLOY_TAG }} - earthly-ci --no-output --push ./aztec-nargo+export-aztec-nargo --DIST_TAG=${{ github.sha }} - build-aztec: needs: set-network env: @@ -268,8 +239,33 @@ jobs: docker push aztecprotocol/aztec:${{ github.sha }} + build-aztec-nargo: + needs: [set-network, build-aztec] + env: + BRANCH_NAME: ${{ needs.set-network.outputs.branch_name }} + DEPLOY_TAG: ${{ needs.set-network.outputs.deploy_tag }} + TF_VAR_DEPLOY_TAG: ${{ needs.set-network.outputs.deploy_tag }} + API_KEY: ${{ secrets[needs.set-network.outputs.network_api_key] }} + TF_VAR_API_KEY: ${{ secrets[needs.set-network.outputs.network_api_key] }} + API_KEY_NAME: ${{ needs.set-network.outputs.network_api_key }} + runs-on: ${{ github.actor }}-x86 + steps: + - uses: actions/checkout@v4 + with: + ref: "${{ env.GIT_COMMIT }}" + fetch-depth: 0 + - uses: ./.github/ci-setup-action + with: + concurrency_key: build-aztec-nargo-${{ github.actor }} + dockerhub_password: "${{ env.DOCKERHUB_PASSWORD }}" + + - name: Build & push aztec nargo image + run: | + earthly-ci --no-output --push ./aztec-nargo+export-aztec-nargo --DIST_TAG=${{ env.DEPLOY_TAG }} + earthly-ci --no-output --push ./aztec-nargo+export-aztec-nargo --DIST_TAG=${{ github.sha }} + build-faucet: - needs: set-network + needs: [set-network, build-aztec] env: BRANCH_NAME: ${{ needs.set-network.outputs.branch_name }} DEPLOY_TAG: ${{ needs.set-network.outputs.deploy_tag }} @@ -415,7 +411,6 @@ jobs: TF_VAR_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.node_lb_priority_range_start }} TF_VAR_PXE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.pxe_lb_priority_range_start }} TF_VAR_BOT_NO_WAIT_FOR_TRANSFERS: ${{ needs.set-network.outputs.bot_no_wait }} - TF_VAR_SEQ_MAX_TX_PER_BLOCK: ${{ needs.set-network.outputs.max_txs_per_block }} steps: - uses: actions/checkout@v4 with: @@ -616,7 +611,6 @@ jobs: TF_VAR_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.node_lb_priority_range_start }} TF_VAR_PXE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.pxe_lb_priority_range_start }} TF_VAR_BOT_NO_WAIT_FOR_TRANSFERS: ${{ needs.set-network.outputs.bot_no_wait }} - TF_VAR_SEQ_MAX_TX_PER_BLOCK: ${{ needs.set-network.outputs.max_txs_per_block }} TF_VAR_PROVING_ENABLED: true TF_VAR_BOT_NO_START: false steps: diff --git a/aztec-up/bin/.aztec-run b/aztec-up/bin/.aztec-run index 8e318fc4c63..1ba0673cf08 100755 --- a/aztec-up/bin/.aztec-run +++ b/aztec-up/bin/.aztec-run @@ -10,6 +10,13 @@ shift DEFAULT_PORT=8080 VERSION=${VERSION:-"latest"} AZTEC_PORT=${AZTEC_PORT:-$DEFAULT_PORT} +INHERIT_USER=${INHERIT_USER:-1} + +if [[ -n "${NETWORK:-}" ]]; then + VERSION=$NETWORK + # enable proving if connecting to a network by default + export PXE_PROVER_ENABLED=${PXE_PROVER_ENABLED:-1} +fi # preserve arguments to pass to docker run declare -a preserved_args @@ -39,7 +46,7 @@ fi # If running in rootless docker, don't set uid/gid in container, as it breaks the write perms to the home mount. # Otherwise we set the uid/gid to be that of user running the container to ensure write perms to the home mount. -if [ -z "${ROOTLESS:-}" ]; then +if [[ -z "${ROOTLESS:-}" && "${INHERIT_USER:-}" == "1" ]]; then DOCKER_USER="--user $(id -u):$(id -g)" fi @@ -73,16 +80,6 @@ for i in "${!args[@]}"; do fi done -DOCKER_ENV="-e HOME=$HOME" -for env in ${ENV_VARS_TO_INJECT:-}; do - if [ -n "${!env:-}" ]; then - # First substitute any reference to localhost with our host gateway. - env=${env//localhost/host.docker.internal} - # Inject into container. - DOCKER_ENV+=" -e $env=${!env:-}" - fi -done - # Parse command-line arguments while [[ "$#" -gt 0 ]]; do case $1 in @@ -91,6 +88,14 @@ while [[ "$#" -gt 0 ]]; do preserved_args+=("$1" "$2") # Store both argument and value shift 2 # Move past argument and value ;; + --pxe.network) + VERSION="$2" # Set version to user-specified network (e.g. 'devnet') + echo "Using aztecprotocol/aztec: $VERSION" + preserved_args+=("$1" "$2") # Store both argument and value + # turn on proving if connecting to a network by default + export PXE_PROVER_ENABLED=${PXE_PROVER_ENABLED:-1} + shift 2 + ;; # Move past argument and value *) preserved_args+=("$1") # Store unrecognized/other arguments shift # Move to next argument @@ -98,6 +103,16 @@ while [[ "$#" -gt 0 ]]; do esac done +DOCKER_ENV="-e HOME=$HOME" +for env in ${ENV_VARS_TO_INJECT:-}; do + if [ -n "${!env:-}" ]; then + # First substitute any reference to localhost with our host gateway. + env=${env//localhost/host.docker.internal} + # Inject into container. + DOCKER_ENV+=" -e $env=${!env:-}" + fi +done + # Dynamic port assignment port_assignment="" if [[ -z "${SKIP_PORT_ASSIGNMENT:-}" ]]; then diff --git a/aztec-up/bin/aztec b/aztec-up/bin/aztec index 7268142f8ae..eff925be279 100755 --- a/aztec-up/bin/aztec +++ b/aztec-up/bin/aztec @@ -1,6 +1,13 @@ #!/usr/bin/env bash set -euo pipefail +# Directory of env_var file +TS_ENV_VAR_FILE=/usr/src/yarn-project/foundation/src/config/env_var.ts +LOCAL_TS_FILE=./env_var.ts +NETWORK=${NETWORK:-} +VERSION=${VERSION:-${NETWORK:-"latest"}} +LOCAL_ENV_VAR_FILE="./.${VERSION}_aztec_cli_vars" + function get_compose { # Favour 'docker compose', falling back on docker-compose. CMD="docker compose" @@ -8,6 +15,11 @@ function get_compose { $CMD $@ } +# Function to parse the TypeScript file +function parse_ts_file { + grep -oE "\| '[^']+'" "$LOCAL_TS_FILE" | sed "s/| '//; s/'//g" >"$LOCAL_ENV_VAR_FILE" +} + CALLED_FROM=$PWD if [ "${1:-}" == "test" ]; then @@ -24,7 +36,7 @@ elif [ $# == 2 ] && [ "$1" == "start" ] && [ "$2" == "--sandbox" ]; then # Compose file to use FILE_ARG="-f $HOME/.aztec/docker-compose.sandbox.yml" - # Function to be executed when SIGINT is received. + # Function to be executed when SIGINT is received. cleanup() { get_compose $FILE_ARG down } @@ -32,9 +44,29 @@ elif [ $# == 2 ] && [ "$1" == "start" ] && [ "$2" == "--sandbox" ]; then # Set trap to catch SIGINT and call the cleanup function. trap cleanup SIGINT - get_compose -p sandbox $FILE_ARG up --force-recreate --remove-orphans + get_compose -p sandbox $FILE_ARG up --force-recreate --remove-orphans elif [ "${1:-}" == "start" ]; then - $(dirname $0)/.aztec-run aztecprotocol/aztec "$@" + # export ENV_VARS_TO_INJECT=$(<.aztec_cli_vars) + + # Grab env vars from aztec project + if [[ ! -f "$LOCAL_ENV_VAR_FILE" ]]; then + echo "No env vars file found, creating one..." + # Create temp container + temp_container_id=$(docker create aztecprotocol/aztec:$VERSION) + # Copy env vars file from container + docker cp ${temp_container_id}:$TS_ENV_VAR_FILE $LOCAL_TS_FILE &> /dev/null + # Remove temp container + docker rm $temp_container_id &> /dev/null + + # Parse the TypeScript file + parse_ts_file + rm $LOCAL_TS_FILE + fi + + # Read env vars from file + readarray -t ENV_VARS_TO_INJECT <"$LOCAL_ENV_VAR_FILE" + export ENV_VARS_TO_INJECT="${ENV_VARS_TO_INJECT[*]}" + ENV_VARS_TO_INJECT="${ENV_VARS_TO_INJECT[*]}" INHERIT_USER=0 $(dirname $0)/.aztec-run aztecprotocol/aztec "$@" else SKIP_PORT_ASSIGNMENT=1 $(dirname $0)/.aztec-run aztecprotocol/aztec "$@" fi diff --git a/aztec-up/bin/aztec-install b/aztec-up/bin/aztec-install index 4d158894149..028d798beab 100755 --- a/aztec-up/bin/aztec-install +++ b/aztec-up/bin/aztec-install @@ -2,19 +2,19 @@ set -euo pipefail # Colors -g="\033[32m" # Green -y="\033[33m" # Yellow -b="\033[34m" # Blue -p="\033[35m" # Purple -r="\033[0m" # Reset +g="\033[32m" # Green +y="\033[33m" # Yellow +b="\033[34m" # Blue +p="\033[35m" # Purple +r="\033[0m" # Reset bold="\033[1m" # Function to replace characters and add color function print_colored() { - local b=$'\033[34m' # Blue - local y=$'\033[33m' # Yellow - local r=$'\033[0m' # Reset - echo "$1" | sed -E "s/(█+)/${b}\1${y}/g" + local b=$'\033[34m' # Blue + local y=$'\033[33m' # Yellow + local r=$'\033[0m' # Reset + echo "$1" | sed -E "s/(█+)/${b}\1${y}/g" } function title() { @@ -32,11 +32,11 @@ function title() { echo if [ "$(uname -s)" == "Darwin" ]; then echo -e "${y}WARNING: For best performance we recommend adjusting your default docker settings:" - echo -e " - Under general, enable VirtioFS." - echo -e " - Under resources, set CPUs to ~80% your maximum." - echo -e " - Under resources, set Memory to ~80% your maximum." - echo -e "You may receive a warning about your home directory being mounted into a container." - echo -e "This is requested so we can read and write project files, that is all." + echo -e " - Under general, enable VirtioFS." + echo -e " - Under resources, set CPUs to ~80% your maximum." + echo -e " - Under resources, set Memory to ~80% your maximum." + echo -e "You may receive a warning about your home directory being mounted into a container." + echo -e "This is requested so we can read and write project files, that is all." echo -e "${r}" fi echo -e "This will install the following scripts and update your PATH if necessary:" @@ -126,6 +126,7 @@ install_bin aztec install_bin aztec-up install_bin aztec-nargo install_bin aztec-wallet +install_bin .aztec_cli_vars function update_path_env_var { TARGET_DIR="${1}" @@ -159,7 +160,7 @@ function update_path_env_var { fi # Add the target directory to the user's PATH in their profile. - echo "export PATH=\"\$PATH:$TARGET_DIR\"" >> "$SHELL_PROFILE" + echo "export PATH=\"\$PATH:$TARGET_DIR\"" >>"$SHELL_PROFILE" if [ -z "${NON_INTERACTIVE:-}" ]; then info "Done! Starting fresh shell..." diff --git a/cspell.json b/cspell.json index dda4954485e..c4ae01a32e5 100644 --- a/cspell.json +++ b/cspell.json @@ -5,6 +5,7 @@ "acir", "acvm", "addrs", + "alphanet", "archiver", "assignement", "asyncify", diff --git a/docker-compose.yml b/docker-compose.yml index 8cc9bda4c66..82a33648b5a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,7 @@ services: PXE_DATA_DIRECTORY: /var/lib/aztec/pxe NODE_NO_WARNINGS: 1 AZTEC_PORT: 8080 + MNEMONIC: "test test test test test test test test test test test junk" entrypoint: > sh -c ' export AZTEC_NODE_URL=$$(cat /var/run/secrets/aztec-node-url); @@ -61,7 +62,7 @@ services: P2P_ENABLED: true PEER_ID_PRIVATE_KEY: AZTEC_PORT: 8999 - TEL_COLLECTOR_BASE_URL: ${TEL_COLLECTOR_BASE_URL:-http://otel-collector:4318} + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318} secrets: - ethereum-host - p2p-boot-node @@ -76,13 +77,13 @@ services: # if the stack is started with --profile metrics --profile node, give the collector a chance to start before the node i=0 max=3 - while ! curl --head --silent $$TEL_COLLECTOR_BASE_URL > /dev/null; do + while ! curl --head --silent $$OTEL_EXPORTER_OTLP_ENDPOINT > /dev/null; do echo "OpenTelemetry collector not up. Retrying after 1s"; sleep 1; i=$$((i+1)); if [ $$i -eq $$max ]; then - echo "OpenTelemetry collector at $$TEL_COLLECTOR_BASE_URL not up after $${max}s. Running without metrics"; - unset TEL_COLLECTOR_BASE_URL; + echo "OpenTelemetry collector at $$OTEL_EXPORTER_OTLP_ENDPOINT not up after $${max}s. Running without metrics"; + unset OTEL_EXPORTER_OTLP_ENDPOINT; break fi; done; diff --git a/iac/mainnet-fork/redeploy b/iac/mainnet-fork/redeploy index e440e5c8425..00750edc07d 100644 --- a/iac/mainnet-fork/redeploy +++ b/iac/mainnet-fork/redeploy @@ -1 +1 @@ -3 \ No newline at end of file +3 diff --git a/yarn-project/archiver/src/archiver/archiver.ts b/yarn-project/archiver/src/archiver/archiver.ts index 006bccdfd4d..a276f7cc3a1 100644 --- a/yarn-project/archiver/src/archiver/archiver.ts +++ b/yarn-project/archiver/src/archiver/archiver.ts @@ -28,6 +28,7 @@ import { type EthAddress } from '@aztec/foundation/eth-address'; import { Fr } from '@aztec/foundation/fields'; import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; import { RunningPromise } from '@aztec/foundation/running-promise'; +import { Timer } from '@aztec/foundation/timer'; import { ClassRegistererAddress } from '@aztec/protocol-contracts/class-registerer'; import { type TelemetryClient } from '@aztec/telemetry-client'; import { @@ -109,7 +110,7 @@ export class Archiver implements ArchiveSource { telemetry: TelemetryClient, blockUntilSynced = true, ): Promise { - const chain = createEthereumChain(config.rpcUrl, config.l1ChainId); + const chain = createEthereumChain(config.l1RpcUrl, config.l1ChainId); const publicClient = createPublicClient({ chain: chain.chainInfo, transport: http(chain.rpcUrl), @@ -291,8 +292,12 @@ export class Archiver implements ArchiveSource { ); if (retrievedBlocks.retrievedData.length > 0) { + const timer = new Timer(); await this.store.addBlocks(retrievedBlocks); - this.instrumentation.processNewBlocks(retrievedBlocks.retrievedData); + this.instrumentation.processNewBlocks( + timer.ms() / retrievedBlocks.retrievedData.length, + retrievedBlocks.retrievedData, + ); const lastL2BlockNumber = retrievedBlocks.retrievedData[retrievedBlocks.retrievedData.length - 1].number; this.log.verbose(`Processed ${retrievedBlocks.retrievedData.length} new L2 blocks up to ${lastL2BlockNumber}`); } diff --git a/yarn-project/archiver/src/archiver/config.ts b/yarn-project/archiver/src/archiver/config.ts index c59e6d1789c..f4ec61d3106 100644 --- a/yarn-project/archiver/src/archiver/config.ts +++ b/yarn-project/archiver/src/archiver/config.ts @@ -1,4 +1,5 @@ -import { type L1ContractAddresses, getL1ContractAddressesFromEnv } from '@aztec/ethereum'; +import { type L1ContractAddresses, type L1ReaderConfig, l1ReaderConfigMappings } from '@aztec/ethereum'; +import { type ConfigMappingsType, getConfigFromMappings, numberConfigHelper } from '@aztec/foundation/config'; /** * There are 2 polling intervals used in this configuration. The first is the archiver polling interval, archiverPollingIntervalMS. @@ -10,27 +11,12 @@ import { type L1ContractAddresses, getL1ContractAddressesFromEnv } from '@aztec/ /** * The archiver configuration. */ -export interface ArchiverConfig { +export type ArchiverConfig = { /** * URL for an archiver service. If set, will return an archiver client as opposed to starting a new one. */ archiverUrl?: string; - /** - * The url of the Ethereum RPC node. - */ - rpcUrl: string; - - /** - * The key for the ethereum node. - */ - apiKey?: string; - - /** - * The L1 chain's ID - */ - l1ChainId: number; - /** * The polling interval in ms for retrieving new L2 blocks and encrypted logs. */ @@ -53,7 +39,35 @@ export interface ArchiverConfig { /** The max number of logs that can be obtained in 1 "getUnencryptedLogs" call. */ maxLogs?: number; -} +} & L1ReaderConfig; + +export const archiverConfigMappings: ConfigMappingsType = { + archiverUrl: { + env: 'ARCHIVER_URL', + description: + 'URL for an archiver service. If set, will return an archiver client as opposed to starting a new one.', + }, + archiverPollingIntervalMS: { + env: 'ARCHIVER_POLLING_INTERVAL_MS', + description: 'The polling interval in ms for retrieving new L2 blocks and encrypted logs.', + ...numberConfigHelper(1000), + }, + viemPollingIntervalMS: { + env: 'ARCHIVER_VIEM_POLLING_INTERVAL_MS', + description: 'The polling interval viem uses in ms', + ...numberConfigHelper(1000), + }, + dataDirectory: { + env: 'DATA_DIRECTORY', + description: 'Optional dir to store data. If omitted will store in memory.', + }, + maxLogs: { + env: 'ARCHIVER_MAX_LOGS', + description: 'The max number of logs that can be obtained in 1 "getUnencryptedLogs" call.', + ...numberConfigHelper(1_000), + }, + ...l1ReaderConfigMappings, +}; /** * Returns the archiver configuration from the environment variables. @@ -61,23 +75,5 @@ export interface ArchiverConfig { * @returns The archiver configuration. */ export function getArchiverConfigFromEnv(): ArchiverConfig { - const { - ETHEREUM_HOST, - L1_CHAIN_ID, - ARCHIVER_POLLING_INTERVAL_MS, - ARCHIVER_VIEM_POLLING_INTERVAL_MS, - API_KEY, - DATA_DIRECTORY, - ARCHIVER_URL, - } = process.env; - return { - rpcUrl: ETHEREUM_HOST || '', - l1ChainId: L1_CHAIN_ID ? +L1_CHAIN_ID : 31337, // 31337 is the default chain id for anvil - archiverPollingIntervalMS: ARCHIVER_POLLING_INTERVAL_MS ? +ARCHIVER_POLLING_INTERVAL_MS : 1_000, - viemPollingIntervalMS: ARCHIVER_VIEM_POLLING_INTERVAL_MS ? +ARCHIVER_VIEM_POLLING_INTERVAL_MS : 1_000, - apiKey: API_KEY, - l1Contracts: getL1ContractAddressesFromEnv(), - dataDirectory: DATA_DIRECTORY, - archiverUrl: ARCHIVER_URL, - }; + return getConfigFromMappings(archiverConfigMappings); } diff --git a/yarn-project/archiver/src/archiver/instrumentation.ts b/yarn-project/archiver/src/archiver/instrumentation.ts index 837b00af7f2..ccfaf063de8 100644 --- a/yarn-project/archiver/src/archiver/instrumentation.ts +++ b/yarn-project/archiver/src/archiver/instrumentation.ts @@ -1,9 +1,17 @@ import { type L2Block } from '@aztec/circuit-types'; -import { type Gauge, type Histogram, Metrics, type TelemetryClient, ValueType } from '@aztec/telemetry-client'; +import { + type Gauge, + type Histogram, + Metrics, + type TelemetryClient, + ValueType, + exponentialBuckets, +} from '@aztec/telemetry-client'; export class ArchiverInstrumentation { private blockHeight: Gauge; - private blockSize: Histogram; + private blockSize: Gauge; + private syncDuration: Histogram; constructor(telemetry: TelemetryClient) { const meter = telemetry.getMeter('Archiver'); @@ -12,16 +20,23 @@ export class ArchiverInstrumentation { valueType: ValueType.INT, }); - this.blockSize = meter.createHistogram(Metrics.ARCHIVER_BLOCK_SIZE, { - description: 'The number of transactions processed per block', + this.blockSize = meter.createGauge(Metrics.ARCHIVER_BLOCK_SIZE, { + description: 'The number of transactions in a block', + valueType: ValueType.INT, + }); + + this.syncDuration = meter.createHistogram(Metrics.ARCHIVER_SYNC_DURATION, { + unit: 'ms', + description: 'Duration to sync a block', valueType: ValueType.INT, advice: { - explicitBucketBoundaries: [2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192], + explicitBucketBoundaries: exponentialBuckets(1, 16), }, }); } - public processNewBlocks(blocks: L2Block[]) { + public processNewBlocks(syncTimePerBlock: number, blocks: L2Block[]) { + this.syncDuration.record(syncTimePerBlock); this.blockHeight.record(Math.max(...blocks.map(b => b.number))); for (const block of blocks) { this.blockSize.record(block.body.txEffects.length); diff --git a/yarn-project/archiver/src/index.ts b/yarn-project/archiver/src/index.ts index e47c3ebb5c5..7b8fa78a625 100644 --- a/yarn-project/archiver/src/index.ts +++ b/yarn-project/archiver/src/index.ts @@ -23,7 +23,7 @@ const log = createDebugLogger('aztec:archiver'); // eslint-disable-next-line require-await async function main() { const config = getArchiverConfigFromEnv(); - const { rpcUrl, l1Contracts } = config; + const { l1RpcUrl: rpcUrl, l1Contracts } = config; const publicClient = createPublicClient({ chain: localhost, diff --git a/yarn-project/aztec-node/src/aztec-node/config.ts b/yarn-project/aztec-node/src/aztec-node/config.ts index 8d2f883f1bf..79f705ab417 100644 --- a/yarn-project/aztec-node/src/aztec-node/config.ts +++ b/yarn-project/aztec-node/src/aztec-node/config.ts @@ -1,13 +1,16 @@ -import { type ArchiverConfig, getArchiverConfigFromEnv as getArchiverVars } from '@aztec/archiver'; -import { type P2PConfig, getP2PConfigEnvVars } from '@aztec/p2p'; -import { type ProverClientConfig, getProverEnvVars } from '@aztec/prover-client'; -import { type SequencerClientConfig, getConfigEnvVars as getSequencerVars } from '@aztec/sequencer-client'; -import { type WorldStateConfig, getWorldStateConfigFromEnv as getWorldStateVars } from '@aztec/world-state'; +import { type ArchiverConfig, archiverConfigMappings } from '@aztec/archiver'; +import { type ConfigMappingsType, getConfigFromMappings } from '@aztec/foundation/config'; +import { type P2PConfig, p2pConfigMappings } from '@aztec/p2p'; +import { type ProverClientConfig, proverClientConfigMappings } from '@aztec/prover-client'; +import { type SequencerClientConfig, sequencerClientConfigMappings } from '@aztec/sequencer-client'; +import { type WorldStateConfig, worldStateConfigMappings } from '@aztec/world-state'; import { readFileSync } from 'fs'; import { dirname, resolve } from 'path'; import { fileURLToPath } from 'url'; +export { sequencerClientConfigMappings, SequencerClientConfig } from '@aztec/sequencer-client'; + /** * The configuration the aztec node. */ @@ -23,24 +26,32 @@ export type AztecNodeConfig = ArchiverConfig & disableProver: boolean; }; +export const aztecNodeConfigMappings: ConfigMappingsType = { + ...archiverConfigMappings, + ...sequencerClientConfigMappings, + ...proverClientConfigMappings, + ...worldStateConfigMappings, + ...p2pConfigMappings, + disableSequencer: { + env: 'SEQ_DISABLED', + parseEnv: (val: string) => ['1', 'true'].includes(val), + default: false, + description: 'Whether the sequencer is disabled for this node.', + }, + disableProver: { + env: 'PROVER_DISABLED', + parseEnv: (val: string) => ['1', 'true'].includes(val), + default: false, + description: 'Whether the prover is disabled for this node.', + }, +}; + /** * Returns the config of the aztec node from environment variables with reasonable defaults. * @returns A valid aztec node config. */ export function getConfigEnvVars(): AztecNodeConfig { - const { SEQ_DISABLED, PROVER_DISABLED = '' } = process.env; - - const allEnvVars: AztecNodeConfig = { - ...getSequencerVars(), - ...getArchiverVars(), - ...getP2PConfigEnvVars(), - ...getWorldStateVars(), - ...getProverEnvVars(), - disableSequencer: !!SEQ_DISABLED, - disableProver: ['1', 'true'].includes(PROVER_DISABLED), - }; - - return allEnvVars; + return getConfigFromMappings(aztecNodeConfigMappings); } /** diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 931e33dc032..cd5b8b8b097 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -135,7 +135,7 @@ export class AztecNodeService implements AztecNode { storeLog = createDebugLogger('aztec:node:lmdb'), ): Promise { telemetry ??= new NoopTelemetryClient(); - const ethereumChain = createEthereumChain(config.rpcUrl, config.l1ChainId); + const ethereumChain = createEthereumChain(config.l1RpcUrl, config.l1ChainId); //validate that the actual chain id matches that specified in configuration if (config.l1ChainId !== ethereumChain.chainInfo.id) { throw new Error( diff --git a/yarn-project/aztec/package.json b/yarn-project/aztec/package.json index 879963d1ca2..ccd6261e0a5 100644 --- a/yarn-project/aztec/package.json +++ b/yarn-project/aztec/package.json @@ -55,7 +55,9 @@ "@aztec/telemetry-client": "workspace:^", "@aztec/txe": "workspace:^", "@aztec/types": "workspace:^", + "@types/chalk": "^2.2.0", "abitype": "^0.8.11", + "chalk": "^5.3.0", "commander": "^12.1.0", "koa": "^2.14.2", "koa-router": "^12.0.0", diff --git a/yarn-project/aztec/src/cli/aztec_start_options.ts b/yarn-project/aztec/src/cli/aztec_start_options.ts new file mode 100644 index 00000000000..663734e3bbd --- /dev/null +++ b/yarn-project/aztec/src/cli/aztec_start_options.ts @@ -0,0 +1,294 @@ +import { type ArchiverConfig, archiverConfigMappings } from '@aztec/archiver'; +import { sequencerClientConfigMappings } from '@aztec/aztec-node'; +import { botConfigMappings } from '@aztec/bot'; +import { type ConfigMapping, filterConfigMappings, isBooleanConfigValue } from '@aztec/foundation/config'; +import { bootnodeConfigMappings, p2pConfigMappings } from '@aztec/p2p'; +import { proverClientConfigMappings } from '@aztec/prover-client'; +import { proverNodeConfigMappings } from '@aztec/prover-node'; +import { allPxeConfigMappings } from '@aztec/pxe'; +import { telemetryClientConfigMappings } from '@aztec/telemetry-client/start'; + +// Define an interface for options +export interface AztecStartOption { + flag: string; + description: string; + defaultValue: any | undefined; + printDefault?: (val: any) => string; + envVar: string | undefined; + parseVal?: (val: string) => any; +} + +export const getOptions = (namespace: string, configMappings: Record) => { + const options: AztecStartOption[] = []; + for (const [key, { env, default: def, parseEnv, description, printDefault }] of Object.entries(configMappings)) { + if (universalOptions.includes(key)) { + continue; + } + const isBoolean = isBooleanConfigValue(configMappings, key as keyof typeof configMappings); + options.push({ + flag: `--${namespace}.${key}${isBoolean ? '' : ' '}`, + description, + defaultValue: def, + printDefault, + envVar: env, + parseVal: parseEnv, + }); + } + return options; +}; + +// These are options used by multiple modules so should be inputted once +export const universalOptions = ['l1RpcUrl', 'l1ChainId', 'l1Contracts', 'p2pEnabled']; + +// Define categories and options +export const aztecStartOptions: { [key: string]: AztecStartOption[] } = { + SANDBOX: [ + { + flag: '--sandbox', + description: 'Starts Aztec Sandbox', + defaultValue: undefined, + envVar: undefined, + }, + { + flag: '--sandbox.testAccounts', + description: 'Deploy test accounts on sandbox start', + defaultValue: true, + envVar: 'TEST_ACCOUNTS', + parseVal: val => ['1', true].includes(val), + }, + { + flag: '--sandbox.enableGas', + description: 'Enable gas on sandbox start', + defaultValue: false, + envVar: 'ENABLE_GAS', + parseVal: val => ['1', true].includes(val), + }, + ], + API: [ + { + flag: '--port', + description: 'Port to run the Aztec Services on on', + defaultValue: 8080, + envVar: 'AZTEC_PORT', + parseVal: val => parseInt(val, 10), + }, + { + flag: '--api-prefix', + description: 'Prefix for API routes on any service that is started', + defaultValue: '', + envVar: 'API_PREFIX', + }, + ], + ETHEREUM: [ + { + flag: '--l1-rpc-url ', + description: 'URL of the Ethereum RPC node that services will connect to', + defaultValue: 'http://localhost:8545', + envVar: 'ETHEREUM_HOST', + }, + { + flag: '--l1-chain-id ', + description: 'The L1 chain ID', + defaultValue: 1337, + envVar: 'L1_CHAIN_ID', + parseVal: val => parseInt(val, 10), + }, + { + flag: '--l1-mnemonic ', + description: 'Mnemonic for L1 accounts. Will be used if no publisher private keys are provided', + defaultValue: undefined, + envVar: 'MNEMONIC', + }, + ], + 'L1 CONTRACT ADDRESSES': [ + { + flag: '--rollup-address ', + description: 'The deployed L1 rollup contract address', + defaultValue: undefined, + envVar: 'ROLLUP_CONTRACT_ADDRESS', + }, + { + flag: '--registry-address ', + description: 'The deployed L1 registry contract address', + defaultValue: undefined, + envVar: 'REGISTRY_CONTRACT_ADDRESS', + }, + { + flag: '--inbox-address ', + description: 'The deployed L1 -> L2 inbox contract address', + defaultValue: undefined, + envVar: 'INBOX_CONTRACT_ADDRESS', + }, + { + flag: '--outbox-address ', + description: 'The deployed L2 -> L1 outbox contract address', + defaultValue: undefined, + envVar: 'OUTBOX_CONTRACT_ADDRESS', + }, + { + flag: '--availability-oracle-address ', + description: 'The deployed L1 availability oracle contract address', + defaultValue: undefined, + envVar: 'AVAILABILITY_ORACLE_CONTRACT_ADDRESS', + }, + { + flag: '--gas-token-address ', + description: 'The deployed L1 gas token contract address', + defaultValue: undefined, + envVar: 'GAS_TOKEN_CONTRACT_ADDRESS', + }, + { + flag: '--gas-portal-address ', + description: 'The deployed L1 gas portal contract address', + defaultValue: undefined, + envVar: 'GAS_PORTAL_CONTRACT_ADDRESS', + }, + ], + // We can't easily auto-generate node options as they're parts of modules defined below + 'AZTEC NODE': [ + { + flag: '--node', + description: 'Starts Aztec Node with options', + defaultValue: undefined, + envVar: undefined, + }, + { + flag: '--node.archiverUrl ', + description: 'URL for an archiver service', + defaultValue: undefined, + envVar: 'ARCHIVER_URL', + }, + { + flag: '--node.dataDirectory ', + description: 'Where to store node data. If not set, will store temporarily', + defaultValue: undefined, + envVar: 'NODE_DATA_DIRECTORY', + }, + { + flag: '--node.deployAztecContracts', + description: 'Deploys L1 Aztec contracts before starting the node. Needs mnemonic or private key to be set', + defaultValue: false, + envVar: 'DEPLOY_AZTEC_CONTRACTS', + }, + { + flag: '--node.publisherPrivateKey ', + description: 'Private key of account for publishing L1 contracts', + defaultValue: undefined, + envVar: 'L1_PRIVATE_KEY', + }, + { + flag: '--node.l2QueueSize ', + description: 'Size of queue of L2 blocks to store in world state', + defaultValue: 1000, + envVar: 'L2_QUEUE_SIZE', + parseVal: val => parseInt(val, 10), + }, + { + flag: '--node.worldStateBlockCheckIntervalMS ', + description: 'Frequency in which to check for blocks in ms', + defaultValue: 100, + envVar: 'WS_BLOCK_CHECK_INTERVAL_MS', + parseVal: val => parseInt(val, 10), + }, + ], + 'P2P SUBSYSTEM': [ + { + flag: '--p2p-enabled', + description: 'Enable P2P subsystem', + defaultValue: false, + envVar: 'P2P_ENABLED', + parseVal: val => ['1', true].includes(val), + }, + ...getOptions('p2p', p2pConfigMappings), + ], + TELEMETRY: [...getOptions('tel', telemetryClientConfigMappings)], + PXE: [ + { + flag: '--pxe', + description: 'Starts Aztec PXE with options', + defaultValue: undefined, + envVar: undefined, + }, + ...getOptions('pxe', allPxeConfigMappings), + ], + ARCHIVER: [ + { + flag: '--archiver', + description: 'Starts Aztec Archiver with options', + defaultValue: undefined, + envVar: undefined, + }, + // filter out archiverUrl as it's passed separately in --node & --prover-node + ...getOptions('archiver', archiverConfigMappings).filter(opt => !opt.flag.includes('archiverUrl')), + ], + SEQUENCER: [ + { + flag: '--sequencer', + description: 'Starts Aztec Sequencer with options', + defaultValue: undefined, + envVar: undefined, + }, + ...getOptions('sequencer', sequencerClientConfigMappings), + ], + 'PROVER AGENT': [ + { + flag: '--prover', + description: 'Starts Aztec Prover Agent with options', + defaultValue: undefined, + envVar: undefined, + }, + ...getOptions('prover', proverClientConfigMappings), + ], + 'PROVER NODE': [ + { + flag: '--prover-node', + description: 'Starts Aztec Prover Node with options', + defaultValue: undefined, + envVar: undefined, + }, + { + flag: '--proverNode.archiverUrl ', + description: 'URL for an archiver service', + defaultValue: undefined, + envVar: 'ARCHIVER_URL', + }, + ...getOptions( + 'proverNode', + // filter out archiver options from prover node options as they're passed separately in --archiver + filterConfigMappings(proverNodeConfigMappings, Object.keys(archiverConfigMappings) as (keyof ArchiverConfig)[]), + ), + ], + 'P2P BOOTSTRAP': [ + { + flag: '--p2p-bootstrap', + description: 'Starts Aztec P2P Bootstrap with options', + defaultValue: undefined, + envVar: undefined, + }, + ...getOptions('p2pBootstrap', bootnodeConfigMappings), + ], + BOT: [ + { + flag: '--bot', + description: 'Starts Aztec Bot with options', + defaultValue: undefined, + envVar: undefined, + }, + ...getOptions('bot', botConfigMappings), + ], + TXE: [ + { + flag: '--txe', + description: 'Starts Aztec TXE with options', + defaultValue: undefined, + envVar: undefined, + }, + { + flag: '--txe.port ', + description: 'Port to run TXE on', + defaultValue: 8081, + envVar: 'TXE_PORT', + parseVal: val => parseInt(val, 10), + }, + ], +}; diff --git a/yarn-project/aztec/src/cli/cli.ts b/yarn-project/aztec/src/cli/cli.ts index d5b92f2b566..4242e07c644 100644 --- a/yarn-project/aztec/src/cli/cli.ts +++ b/yarn-project/aztec/src/cli/cli.ts @@ -4,116 +4,118 @@ import { type ServerList, createNamespacedJsonRpcServer, createStatusRouter } fr import { type DebugLogger, type LogFn } from '@aztec/foundation/log'; import { createPXERpcServer } from '@aztec/pxe'; -import { type Command } from 'commander'; +import { Command } from 'commander'; import http from 'http'; import { createSandbox } from '../sandbox.js'; import { github, splash } from '../splash.js'; -import { cliTexts } from './texts.js'; -import { createAccountLogs, installSignalHandlers } from './util.js'; - -const { AZTEC_PORT = '8080', API_PREFIX = '', TEST_ACCOUNTS = 'true', ENABLE_GAS = '' } = process.env; +import { aztecStartOptions } from './aztec_start_options.js'; +import { + addOptions, + createAccountLogs, + extractNamespacedOptions, + installSignalHandlers, + printAztecStartHelpText, +} from './util.js'; /** * Returns commander program that defines the 'aztec' command line interface. * @param userLog - log function for logging user output. * @param debugLogger - logger for logging debug messages. */ -export function injectAztecCommands(program: Command, userLog: LogFn, debugLogger: DebugLogger) { - // Start Aztec modules with options - program - .command('start') - .description( - 'Starts Aztec modules. Options for each module can be set as key-value pairs (e.g. "option1=value1,option2=value2") or as environment variables.', - ) - .option('-sb, --sandbox', 'Starts Aztec Sandbox.') - .option('-p, --port ', 'Port to run Aztec on.', AZTEC_PORT) - .option('-n, --node [options]', cliTexts.node) - .option('-px, --pxe [options]', cliTexts.pxe) - .option('-a, --archiver [options]', cliTexts.archiver) - .option('-s, --sequencer [options]', cliTexts.sequencer) - .option('-r, --prover [options]', cliTexts.proverAgent) - .option('-o, --prover-node [options]', cliTexts.proverNode) - .option('-p2p, --p2p-bootstrap [options]', cliTexts.p2pBootstrap) - .option('-t, --txe [options]', cliTexts.txe) - .option('--bot [options]', cliTexts.bot) - .action(async options => { - // list of 'stop' functions to call when process ends - const signalHandlers: Array<() => Promise> = []; - let services: ServerList = []; - - if (options.sandbox) { - userLog(`${splash}\n${github}\n\n`); - userLog(`Setting up Aztec Sandbox, please stand by...`); - const { aztecNodeConfig, node, pxe, stop } = await createSandbox({ - enableGas: ['true', '1'].includes(ENABLE_GAS), - }); - - // Deploy test accounts by default - if (TEST_ACCOUNTS === 'true') { - if (aztecNodeConfig.p2pEnabled) { - userLog(`Not setting up test accounts as we are connecting to a network`); - } else { - userLog('Setting up test accounts...'); - const accounts = await deployInitialTestAccounts(pxe); - const accLogs = await createAccountLogs(accounts, pxe); - userLog(accLogs.join('')); - } - } +export function injectAztecCommands(program: Command, userLog: LogFn, debugLogger: DebugLogger): Command { + const startCmd = new Command('start').description( + 'Starts Aztec modules. Options for each module can be set as key-value pairs (e.g. "option1=value1,option2=value2") or as environment variables.', + ); - // Start Node and PXE JSON-RPC server - const nodeServer = createAztecNodeRpcServer(node); - const pxeServer = createPXERpcServer(pxe); - signalHandlers.push(stop); - services = [{ node: nodeServer }, { pxe: pxeServer }]; - } else { - if (options.node) { - const { startNode } = await import('./cmds/start_node.js'); - services = await startNode(options, signalHandlers, userLog); - } else if (options.bot) { - const { startBot } = await import('./cmds/start_bot.js'); - services = await startBot(options, signalHandlers, userLog); - } else if (options.proverNode) { - const { startProverNode } = await import('./cmds/start_prover_node.js'); - services = await startProverNode(options, signalHandlers, userLog); - } else if (options.pxe) { - const { startPXE } = await import('./cmds/start_pxe.js'); - services = await startPXE(options, signalHandlers, userLog); - } else if (options.archiver) { - const { startArchiver } = await import('./cmds/start_archiver.js'); - services = await startArchiver(options, signalHandlers); - } else if (options.p2pBootstrap) { - const { startP2PBootstrap } = await import('./cmds/start_p2p_bootstrap.js'); - await startP2PBootstrap(options, userLog, debugLogger); - } else if (options.prover) { - const { startProverAgent } = await import('./cmds/start_prover_agent.js'); - services = await startProverAgent(options, signalHandlers, userLog); - } else if (options.txe) { - const { startTXE } = await import('./cmds/start_txe.js'); - startTXE(options, debugLogger); - } else if (options.sequencer) { - userLog(`Cannot run a standalone sequencer without a node`); - process.exit(1); + // Assuming commands are added elsewhere, here we just add options to the main program + Object.keys(aztecStartOptions).forEach(category => { + addOptions(startCmd, aztecStartOptions[category]); + }); + + startCmd.helpInformation = printAztecStartHelpText; + + startCmd.action(async options => { + // list of 'stop' functions to call when process ends + const signalHandlers: Array<() => Promise> = []; + let services: ServerList = []; + + if (options.sandbox) { + const sandboxOptions = extractNamespacedOptions(options, 'sandbox'); + userLog(`${splash}\n${github}\n\n`); + userLog(`Setting up Aztec Sandbox, please stand by...`); + const { aztecNodeConfig, node, pxe, stop } = await createSandbox({ + enableGas: sandboxOptions.enableGas, + l1Mnemonic: options.l1Mnemonic, + }); + + // Deploy test accounts by default + if (sandboxOptions.testAccounts) { + if (aztecNodeConfig.p2pEnabled) { + userLog(`Not setting up test accounts as we are connecting to a network`); } else { - userLog(`No module specified to start ${JSON.stringify(options, null, 2)}`); - process.exit(1); + userLog('Setting up test accounts...'); + const accounts = await deployInitialTestAccounts(pxe); + const accLogs = await createAccountLogs(accounts, pxe); + userLog(accLogs.join('')); } } - installSignalHandlers(debugLogger.info, signalHandlers); - if (services.length) { - const rpcServer = createNamespacedJsonRpcServer(services, debugLogger); + // Start Node and PXE JSON-RPC server + const nodeServer = createAztecNodeRpcServer(node); + const pxeServer = createPXERpcServer(pxe); + signalHandlers.push(stop); + services = [{ node: nodeServer }, { pxe: pxeServer }]; + } else { + if (options.node) { + const { startNode } = await import('./cmds/start_node.js'); + services = await startNode(options, signalHandlers, userLog); + } else if (options.bot) { + const { startBot } = await import('./cmds/start_bot.js'); + services = await startBot(options, signalHandlers, userLog); + } else if (options.proverNode) { + const { startProverNode } = await import('./cmds/start_prover_node.js'); + services = await startProverNode(options, signalHandlers, userLog); + } else if (options.pxe) { + const { startPXE } = await import('./cmds/start_pxe.js'); + services = await startPXE(options, signalHandlers, userLog); + } else if (options.archiver) { + const { startArchiver } = await import('./cmds/start_archiver.js'); + services = await startArchiver(options, signalHandlers); + } else if (options.p2pBootstrap) { + const { startP2PBootstrap } = await import('./cmds/start_p2p_bootstrap.js'); + await startP2PBootstrap(options, userLog, debugLogger); + } else if (options.prover) { + const { startProverAgent } = await import('./cmds/start_prover_agent.js'); + services = await startProverAgent(options, signalHandlers, userLog); + } else if (options.txe) { + const { startTXE } = await import('./cmds/start_txe.js'); + startTXE(options, debugLogger); + } else if (options.sequencer) { + userLog(`Cannot run a standalone sequencer without a node`); + process.exit(1); + } else { + userLog(`No module specified to start ${JSON.stringify(options, null, 2)}`); + process.exit(1); + } + } + installSignalHandlers(debugLogger.info, signalHandlers); + + if (services.length) { + const rpcServer = createNamespacedJsonRpcServer(services, debugLogger); - const app = rpcServer.getApp(API_PREFIX); - // add status route - const statusRouter = createStatusRouter(API_PREFIX); - app.use(statusRouter.routes()).use(statusRouter.allowedMethods()); + const app = rpcServer.getApp(options.apiPrefix); + // add status route + const statusRouter = createStatusRouter(options.apiPrefix); + app.use(statusRouter.routes()).use(statusRouter.allowedMethods()); - const httpServer = http.createServer(app.callback()); - httpServer.listen(options.port); - userLog(`Aztec Server listening on port ${options.port}`); - } - }); + const httpServer = http.createServer(app.callback()); + httpServer.listen(options.port); + userLog(`Aztec Server listening on port ${options.port}`); + } + }); + + program.addCommand(startCmd); program.configureHelp({ sortSubcommands: true }); @@ -125,7 +127,7 @@ export function injectAztecCommands(program: Command, userLog: LogFn, debugLogge test [options]: starts a dockerized TXE node via $ aztec start --txe - then runs + then runs $ aztec-nargo test --silence-warnings --oracle-resolver= [options] `, ); diff --git a/yarn-project/aztec/src/cli/cmds/start_archiver.ts b/yarn-project/aztec/src/cli/cmds/start_archiver.ts index 4d6e487e532..c3464fdbad9 100644 --- a/yarn-project/aztec/src/cli/cmds/start_archiver.ts +++ b/yarn-project/aztec/src/cli/cmds/start_archiver.ts @@ -2,8 +2,8 @@ import { Archiver, type ArchiverConfig, KVArchiverDataStore, + archiverConfigMappings, createArchiverRpcServer, - getArchiverConfigFromEnv as getArchiverConfigEnvVars, } from '@aztec/archiver'; import { createDebugLogger } from '@aztec/aztec.js'; import { type ServerList } from '@aztec/foundation/json-rpc/server'; @@ -14,21 +14,16 @@ import { getConfigEnvVars as getTelemetryClientConfig, } from '@aztec/telemetry-client/start'; -import { mergeEnvVarsAndCliOptions, parseModuleOptions } from '../util.js'; +import { extractRelevantOptions } from '../util.js'; export const startArchiver = async (options: any, signalHandlers: (() => Promise)[]) => { const services: ServerList = []; // Start a standalone archiver. - // get env vars first - const archiverConfigEnvVars = getArchiverConfigEnvVars(); - // get config from options - const archiverCliOptions = parseModuleOptions(options.archiver); - // merge env vars and cli options - const archiverConfig = mergeEnvVarsAndCliOptions(archiverConfigEnvVars, archiverCliOptions, true); + const archiverConfig = extractRelevantOptions(options, archiverConfigMappings); const storeLog = createDebugLogger('aztec:archiver:lmdb'); const store = await initStoreForRollup( - AztecLmdbStore.open(archiverConfig.dataDirectory, false, storeLog), + AztecLmdbStore.open(archiverConfig.dataDirectory, false), archiverConfig.l1Contracts.rollupAddress, storeLog, ); diff --git a/yarn-project/aztec/src/cli/cmds/start_bot.ts b/yarn-project/aztec/src/cli/cmds/start_bot.ts index 0d33f0bf10c..0888b9dbf63 100644 --- a/yarn-project/aztec/src/cli/cmds/start_bot.ts +++ b/yarn-project/aztec/src/cli/cmds/start_bot.ts @@ -1,9 +1,9 @@ -import { type BotConfig, BotRunner, createBotRunnerRpcServer, getBotConfigFromEnv } from '@aztec/bot'; +import { type BotConfig, BotRunner, botConfigMappings, createBotRunnerRpcServer } from '@aztec/bot'; import { type PXE } from '@aztec/circuit-types'; import { type ServerList } from '@aztec/foundation/json-rpc/server'; import { type LogFn } from '@aztec/foundation/log'; -import { mergeEnvVarsAndCliOptions, parseModuleOptions } from '../util.js'; +import { extractRelevantOptions } from '../util.js'; export async function startBot( options: any, @@ -20,7 +20,6 @@ export async function startBot( ); process.exit(1); } - // Start a PXE client that is used by the bot if required let pxe: PXE | undefined; if (options.pxe) { @@ -38,9 +37,7 @@ export function addBot( signalHandlers: (() => Promise)[], deps: { pxe?: PXE } = {}, ) { - const envVars = getBotConfigFromEnv(); - const cliOptions = parseModuleOptions(options.bot); - const config = mergeEnvVarsAndCliOptions(envVars, cliOptions); + const config = extractRelevantOptions(options, botConfigMappings); const botRunner = new BotRunner(config, { pxe: deps.pxe }); const botServer = createBotRunnerRpcServer(botRunner); diff --git a/yarn-project/aztec/src/cli/cmds/start_node.ts b/yarn-project/aztec/src/cli/cmds/start_node.ts index 33dd2a2489b..a3666539abb 100644 --- a/yarn-project/aztec/src/cli/cmds/start_node.ts +++ b/yarn-project/aztec/src/cli/cmds/start_node.ts @@ -1,38 +1,35 @@ -import { - type AztecNodeConfig, - createAztecNodeRpcServer, - getConfigEnvVars as getNodeConfigEnvVars, -} from '@aztec/aztec-node'; +import { aztecNodeConfigMappings, createAztecNodeRpcServer } from '@aztec/aztec-node'; import { type PXE } from '@aztec/circuit-types'; -import { NULL_KEY } from '@aztec/ethereum'; import { type ServerList } from '@aztec/foundation/json-rpc/server'; import { type LogFn } from '@aztec/foundation/log'; import { createProvingJobSourceServer } from '@aztec/prover-client/prover-agent'; import { + type TelemetryClientConfig, createAndStartTelemetryClient, - getConfigEnvVars as getTelemetryClientConfig, + telemetryClientConfigMappings, } from '@aztec/telemetry-client/start'; import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts'; -import { MNEMONIC, createAztecNode, deployContractsToL1 } from '../../sandbox.js'; -import { mergeEnvVarsAndCliOptions, parseModuleOptions } from '../util.js'; - -const { DEPLOY_AZTEC_CONTRACTS } = process.env; +import { createAztecNode, deployContractsToL1 } from '../../sandbox.js'; +import { extractL1ContractAddresses, extractNamespacedOptions, extractRelevantOptions } from '../util.js'; export const startNode = async ( options: any, signalHandlers: (() => Promise)[], userLog: LogFn, -): Promise => { + // ): Promise => { +) => { // Services that will be started in a single multi-rpc server const services: ServerList = []; - // get env vars first - const aztecNodeConfigEnvVars = getNodeConfigEnvVars(); - // get config from options - const nodeCliOptions = parseModuleOptions(options.node); - // merge env vars and cli options - let nodeConfig = mergeEnvVarsAndCliOptions(aztecNodeConfigEnvVars, nodeCliOptions); + + // options specifically namespaced with --node.