feat(archiver): read blobs from cl

.github/workflows/devnet-deploy.yml

@@ -111,7 +111,7 @@ jobs:
 
           PXE_PORT="$(jq -r .pxe.service.nodePort helm_values.json)"
           FAUCET_PORT="$(jq -r .faucet.apiServerPort helm_values.json)"
-          ETHEREUM_PORT="$(jq -r .ethereum.service.port helm_values.json)"
+          ETHEREUM_PORT="$(jq -r .ethereum.execution.service.port helm_values.json)"
           L1_CHAIN_ID="$(jq -r .ethereum.chainId helm_values.json)"
 
           MNEMONIC="$(jq -r .aztec.l1DeploymentMnemonic helm_values.json)"

package.json

@@ -0,0 +1,3 @@
+{
+  "packageManager": "yarn@4.5.2"
+}

scripts/run_native_testnet.sh

@@ -32,7 +32,11 @@ PROVER_SCRIPT="\"./prover-node.sh 8078 false\""
 NUM_VALIDATORS=3
 INTERLEAVED=false
 METRICS=false
+DISABLE_BLOB_SINK=false
 LOG_LEVEL="info"
+ETHEREUM_HOST="http://localhost:8545"
+L1_CONSENSUS_HOST_URL=
+
 OTEL_COLLECTOR_ENDPOINT=${OTEL_COLLECTOR_ENDPOINT:-"http://localhost:4318"}
 
 # Function to display help message
@@ -49,6 +53,9 @@ display_help() {
     echo "  -i     Run interleaved (default: $INTERLEAVED)"
     echo "  -m     Run with metrics (default: $METRICS) will use $OTEL_COLLECTOR_ENDPOINT as default otel endpoint"
     echo "  -c     Specify the otel collector endpoint (default: $OTEL_COLLECTOR_ENDPOINT)"
+    echo "  -b     Disable the blob sink (default: false)"
+    echo "  -e     Specify the ethereum host url (default: $ETHEREUM_HOST)"
+    echo "  -cl    Specify the l1 consensus host url (default: $L1_CONSENSUS_HOST_URL)"
     echo
     echo "Example:"
     echo "  $0 -t ./test-4epochs.sh -val 5 -v"
@@ -97,6 +104,18 @@ while [[ $# -gt 0 ]]; do
       OTEL_COLLECTOR_ENDPOINT="$2"
       shift 2
       ;;
+    -e)
+      ETHEREUM_HOST="$2"
+      shift 2
+      ;;
+    -cl)
+      L1_CONSENSUS_HOST_URL="$2"
+      shift 2
+      ;;
+    -b)
+      DISABLE_BLOB_SINK=true
+      shift
+      ;;
     *)
       echo "Invalid option: $1" >&2
       display_help
@@ -115,6 +134,24 @@ if $METRICS; then
   export LOG_JSON=1
 fi
 
+# If an ethereum rpc url is provided, use it
+export ETHEREUM_HOST
+export L1_CONSENSUS_HOST_URL
+
+# If an ethereum url has been provided, do not run the ethereum.sh script
+if [ -n "$ETHEREUM_HOST" ]; then
+  ETHEREUM_SCRIPT=""
+else
+  ETHEREUM_SCRIPT="./ethereum.sh"
+fi
+
+# If the blob sink is disabled, do not run the blob-sink.sh script
+if $DISABLE_BLOB_SINK; then
+  BLOB_SINK_SCRIPT=""
+else
+  BLOB_SINK_SCRIPT="./blob-sink.sh"
+fi
+
 # Go to repo root
 cd $(git rev-parse --show-toplevel)
 
@@ -124,11 +161,12 @@ BASE_CMD="INTERLEAVED=$INTERLEAVED ./yarn-project/end-to-end/scripts/native_netw
         \"./deploy-l1-contracts.sh $NUM_VALIDATORS\" \
         ./deploy-l2-contracts.sh \
         ./boot-node.sh \
-        ./ethereum.sh \
+        $ETHEREUM_SCRIPT \
         \"./validators.sh $NUM_VALIDATORS\" \
         $PROVER_SCRIPT \
         ./pxe.sh \
-        ./transaction-bot.sh"
+        ./transaction-bot.sh \
+        $BLOB_SINK_SCRIPT"
 
 # Execute the command
 eval $BASE_CMD

spartan/aztec-network/.gitignore

@@ -0,0 +1 @@
+out/

spartan/aztec-network/eth-devnet/README.md

@@ -0,0 +1,62 @@
+## Eth Devnet
+
+### Usage
+
+```bash
+./create.sh
+```
+
+## Args
+
+Args can be supplied via environment variables.
+
+### NUMBER_OF_KEYS
+
+This determines the number of accounts that will be prefunded with eth on the execution layer.
+
+### MNEMONIC
+
+The seed phrase from which the keys will be derived.
+
+### BLOCK_TIME
+
+The time in seconds between blocks.
+
+### GAS_LIMIT
+
+The gas limit for the execution layer.
+
+### CHAIN_ID
+
+The chain id for the execution layer.
+
+---
+
+## Common Pitfalls
+
+If you are struggling to get the network up and running, it is usually due to the genesis.json file having different values from the config.yaml + genesis.ssz file. Make sure that you do not edit any of them by accident after ./create.sh is run.
+Note that this script places the configuration values within the /out folder, it will not actually run the testnet.
+
+SSZ files are not passable thorugh config maps, so they must be base64 encoded, then decoded in the container before running.
+
+Generating an Ethereum testnet requires a few ingredients:
+
+## Genesis.json file
+
+The genesis.json file configures the initial state of the execution layer, it defines what accounts are preloaded with what balances, what hardforks are active etc.
+In this case the most important values to set are the deposit contract (ensuring that it is filled with empty state ( and an empty deposit tree )), and the allocation accounts we would like to have preloaded with funds.
+
+## Config.yaml
+
+The config.yaml file is used to configure a beacon chain client. It configures what contract address the deposit contract should be read on, as well as configuring when hardforks should be activated.
+
+## Genesis.ssz
+
+This file contains the state of the beacon chain at the genesis block, and it is used to bootstrap the network, such as the validator registry at the time of genesis, the deposit root from eth1 at the time of genesis etc.
+
+## Other files
+
+### Jwt secret
+
+The jwt secret is used to authenticate the beacon chain client to the execution layer.
+The execution api ports should not be exposed to the open internet.

spartan/aztec-network/eth-devnet/config/config.yaml

@@ -0,0 +1,144 @@
+# Minimal config - from github.com/ethereum-optimism/optimism
+
+# Extends the minimal preset
+PRESET_BASE: "minimal"
+
+# Free-form short name of the network that this configuration applies to - known
+# canonical network names include:
+# * 'mainnet' - there can be only one
+# * 'prater' - testnet
+# Must match the regex: [a-z0-9\-]
+CONFIG_NAME: "minimal"
+
+# Transition
+# ---------------------------------------------------------------
+# 2**256-2**10 for testing minimal network
+TERMINAL_TOTAL_DIFFICULTY: 115792089237316195423570985008687907853269984665640564039457584007913129638912
+# By default, don't use these params
+TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000
+TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615
+
+# Genesis
+# ---------------------------------------------------------------
+# [customized]
+MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 64
+# Jan 3, 2020
+MIN_GENESIS_TIME: 1578009600
+# Highest byte set to 0xFF, this is dev network
+GENESIS_FORK_VERSION: 0x000000FF
+# [customized] don't wait with genesis if we don't have to
+GENESIS_DELAY: 0
+
+# Forking
+# ---------------------------------------------------------------
+# Values provided for illustrative purposes.
+# Individual tests/testnets may set different values.
+
+# Altair
+ALTAIR_FORK_VERSION: 0x01000001
+ALTAIR_FORK_EPOCH: 0
+# Bellatrix
+BELLATRIX_FORK_VERSION: 0x02000001
+BELLATRIX_FORK_EPOCH: 0
+# Capella
+CAPELLA_FORK_VERSION: 0x03000001
+CAPELLA_FORK_EPOCH: 0
+# DENEB
+DENEB_FORK_VERSION: 0x04000001
+DENEB_FORK_EPOCH: 0
+
+# EIP6110
+EIP6110_FORK_VERSION: 0x05000001
+EIP6110_FORK_EPOCH: 18446744073709551615
+# EIP7002
+EIP7002_FORK_VERSION: 0x05000001
+EIP7002_FORK_EPOCH: 18446744073709551615
+# WHISK
+WHISK_FORK_VERSION: 0x06000001
+WHISK_FORK_EPOCH: 18446744073709551615
+
+# Time parameters
+# ---------------------------------------------------------------
+# [customized] Faster for testing purposes
+SECONDS_PER_SLOT: 12
+# 14 (estimate from Eth1 mainnet)
+SECONDS_PER_ETH1_BLOCK: 12
+# 2**8 (= 256) epochs
+MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256
+# [customized] higher frequency of committee turnover and faster time to acceptable voluntary exit
+SHARD_COMMITTEE_PERIOD: 64
+# [customized] process deposits more quickly, but insecure
+ETH1_FOLLOW_DISTANCE: 16
+
+# Validator cycle
+# ---------------------------------------------------------------
+# 2**2 (= 4)
+INACTIVITY_SCORE_BIAS: 4
+# 2**4 (= 16)
+INACTIVITY_SCORE_RECOVERY_RATE: 16
+# 2**4 * 10**9 (= 16,000,000,000) Gwei
+EJECTION_BALANCE: 16000000000
+# [customized] more easily demonstrate the difference between this value and the activation churn limit
+MIN_PER_EPOCH_CHURN_LIMIT: 2
+# [customized] scale queue churn at much lower validator counts for testing
+CHURN_LIMIT_QUOTIENT: 32
+# [New in Deneb:EIP7514] [customized]
+MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 4
+
+# Fork choice
+# ---------------------------------------------------------------
+# 40%
+PROPOSER_SCORE_BOOST: 40
+# 20%
+REORG_HEAD_WEIGHT_THRESHOLD: 20
+# 160%
+REORG_PARENT_WEIGHT_THRESHOLD: 160
+# `2` epochs
+REORG_MAX_EPOCHS_SINCE_FINALIZATION: 2
+
+# Deposit contract
+# ---------------------------------------------------------------
+# Local Optimism monorepo devnet
+DEPOSIT_CHAIN_ID: 1337
+DEPOSIT_NETWORK_ID: 1337
+# Configured on a per testnet basis
+DEPOSIT_CONTRACT_ADDRESS: 0x4242424242424242424242424242424242424242
+
+# Networking
+# ---------------------------------------------------------------
+# `10 * 2**20` (= 10485760, 10 MiB)
+GOSSIP_MAX_SIZE: 10485760
+# `2**10` (= 1024)
+MAX_REQUEST_BLOCKS: 1024
+# `2**8` (= 256)
+EPOCHS_PER_SUBNET_SUBSCRIPTION: 256
+# [customized] `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 272)
+MIN_EPOCHS_FOR_BLOCK_REQUESTS: 272
+# `10 * 2**20` (=10485760, 10 MiB)
+MAX_CHUNK_SIZE: 10485760
+# 5s
+TTFB_TIMEOUT: 5
+# 10s
+RESP_TIMEOUT: 10
+ATTESTATION_PROPAGATION_SLOT_RANGE: 32
+# 500ms
+MAXIMUM_GOSSIP_CLOCK_DISPARITY: 500
+MESSAGE_DOMAIN_INVALID_SNAPPY: 0x00000000
+MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000
+# 2 subnets per node
+SUBNETS_PER_NODE: 2
+# 2**8 (= 64)
+ATTESTATION_SUBNET_COUNT: 64
+ATTESTATION_SUBNET_EXTRA_BITS: 0
+# ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS
+ATTESTATION_SUBNET_PREFIX_BITS: 6
+
+# Deneb
+# `2**7` (=128)
+MAX_REQUEST_BLOCKS_DENEB: 128
+# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK
+MAX_REQUEST_BLOB_SIDECARS: 768
+# `2**12` (= 4096 epochs, ~18 days)
+MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096
+# `6`
+BLOB_SIDECAR_SUBNET_COUNT: 6