Snap sync moving root/healing implementation

packages/blockchain/src/blockchain.ts

@@ -1319,6 +1319,13 @@ export class Blockchain implements BlockchainInterface {
       number: 0,
       stateRoot,
       withdrawalsRoot: common.isActivatedEIP(4895) ? KECCAK256_RLP : undefined,
+      ...(common.isActivatedEIP(4844)
+        ? {
+            blobGasUsed: 0,
+            excessBlobGas: 0,
+            parentBeaconBlockRoot: new Uint8Array(32),
+          }
+        : undefined),
     }
     if (common.consensusType() === 'poa') {
       if (common.genesis().extraData) {

packages/client/src/config.ts

@@ -326,6 +326,7 @@ export interface ConfigOptions {
   pruneEngineCache?: boolean
   snapAvailabilityDepth?: bigint
   snapTransitionSafeDepth?: bigint
+  snapLookbackWindow?: number
 
   /**
    * Save account keys preimages in the meta db (default: false)
@@ -397,6 +398,7 @@ export class Config {
   // distance from head at which we can safely transition from a synced snapstate to vmexecution
   // randomly kept it at 5 for fast testing purposes but ideally should be >=32 slots
   public static readonly SNAP_TRANSITION_SAFE_DEPTH = BigInt(5)
+  public static readonly SNAP_LOOKBACK_WINDOW = 1
 
   // support blobs and proofs cache for CL getBlobs for upto 1 epoch of data
   public static readonly BLOBS_AND_PROOFS_CACHE_BLOCKS = 32
@@ -448,6 +450,7 @@ export class Config {
   public readonly engineNewpayloadMaxTxsExecute: number
   public readonly snapAvailabilityDepth: bigint
   public readonly snapTransitionSafeDepth: bigint
+  public readonly snapLookbackWindow: number
 
   public readonly prefixStorageTrieKeys: boolean
   // Defaulting to false as experimental as of now
@@ -543,6 +546,7 @@ export class Config {
     this.snapAvailabilityDepth = options.snapAvailabilityDepth ?? Config.SNAP_AVAILABILITY_DEPTH
     this.snapTransitionSafeDepth =
       options.snapTransitionSafeDepth ?? Config.SNAP_TRANSITION_SAFE_DEPTH
+    this.snapLookbackWindow = options.snapLookbackWindow ?? Config.SNAP_LOOKBACK_WINDOW
 
     this.prefixStorageTrieKeys = options.prefixStorageTrieKeys ?? true
     this.enableSnapSync = options.enableSnapSync ?? false

packages/client/src/net/server/rlpxserver.ts

@@ -65,7 +65,7 @@ export class RlpxServer extends Server {
     super(options)
     // As of now, the devp2p dpt server listens on the ip4 protocol by default and hence the ip in the
     // bootnode needs to be of ip4 by default
-    this.ip = options.config.extIP ?? '0.0.0.0'
+    this.ip = options.config.extIP ?? '127.0.0.1'
     this.discovery = options.config.discV4 || options.config.discDns
     this.clientFilter = options.clientFilter ?? [
       'go1.5',

packages/client/src/sync/fetcher/accountfetcher.ts

@@ -43,8 +43,8 @@ type AccountDataResponse = AccountData[] & { completed?: boolean }
  * @memberof module:sync/fetcher
  */
 export interface AccountFetcherOptions extends FetcherOptions {
-  /** Root hash of the account trie to serve */
-  root: Uint8Array
+  // height of block being targeted for snap sync
+  height: bigint
 
   /** The origin to start account fetcher from (including), by default starts from 0 (0x0000...) */
   first: bigint
@@ -72,7 +72,7 @@ export class AccountFetcher extends Fetcher<JobTask, AccountData[], AccountData>
   stateManager: MerkleStateManager
   accountTrie: MerklePatriciaTrie
 
-  root: Uint8Array
+  height: bigint
   highestKnownHash: Uint8Array | undefined
 
   /** The origin to start account fetcher from (including), by default starts from 0 (0x0000...) */
@@ -92,7 +92,7 @@ export class AccountFetcher extends Fetcher<JobTask, AccountData[], AccountData>
     super(options)
     this.fetcherDoneFlags = options.fetcherDoneFlags ?? getInitFetcherDoneFlags()
 
-    this.root = options.root
+    this.height = options.height
     this.first = options.first
     this.count = options.count ?? BIGINT_2EXP256 - this.first
 
@@ -104,7 +104,7 @@ export class AccountFetcher extends Fetcher<JobTask, AccountData[], AccountData>
     this.storageFetcher = new StorageFetcher({
       config: this.config,
       pool: this.pool,
-      root: this.root,
+      height: this.height,
       storageRequests: [],
       first: BIGINT_1,
       destroyWhenDone: false,
@@ -122,7 +122,7 @@ export class AccountFetcher extends Fetcher<JobTask, AccountData[], AccountData>
     this.trieNodeFetcher = new TrieNodeFetcher({
       config: this.config,
       pool: this.pool,
-      root: this.root,
+      height: this.height,
       stateManager: this.stateManager,
       destroyWhenDone: false,
       fetcherDoneFlags: this.fetcherDoneFlags,
@@ -138,7 +138,7 @@ export class AccountFetcher extends Fetcher<JobTask, AccountData[], AccountData>
 
     this.DEBUG &&
       this.debug(
-        `Account fetcher instantiated root=${short(this.root)} origin=${short(origin)} limit=${short(
+        `Account fetcher instantiated root=${short(this.fetcherDoneFlags.snapTargetRoot!)} origin=${short(origin)} limit=${short(
           limit,
         )} destroyWhenDone=${this.destroyWhenDone}`,
       )
@@ -277,6 +277,12 @@ export class AccountFetcher extends Fetcher<JobTask, AccountData[], AccountData>
         }
         break
       case TrieNodeFetcher:
+        // TODO update to check if heal phase completed successfully and then continue with next
+        // healing phase
+        if (fetcherDoneFlags.trieNodeFetcher.healCycleRepeatCount > 1) {
+          fetcherDoneFlags.trieNodeFetcher.healCycleRepeatCount--
+          break
+        }
         fetcherDoneFlags.trieNodeFetcher.done = true
         break
     }
@@ -286,15 +292,21 @@ export class AccountFetcher extends Fetcher<JobTask, AccountData[], AccountData>
       accountFetcher.done && storageFetcher.done && byteCodeFetcher.done && trieNodeFetcher.done
 
     this.config.superMsg(
-      `snapFetchersCompletion root=${short(this.root)} accountsRoot=${short(
-        fetcherDoneFlags.stateRoot ?? 'na',
-      )} done=${this.fetcherDoneFlags.done} accountsDone=${accountFetcher.done} storageDone=${
-        storageFetcher.done
-      } byteCodesDone=${byteCodeFetcher.done} trieNodesDone=${trieNodeFetcher.done}`,
+      `snapFetchersCompletion root=${short(
+        this.fetcherDoneFlags.snapTargetRoot!,
+      )} accountsRoot=${short(fetcherDoneFlags.stateRoot ?? 'na')} done=${
+        this.fetcherDoneFlags.done
+      } accountsDone=${accountFetcher.done} storageDone=${storageFetcher.done} byteCodesDone=${
+        byteCodeFetcher.done
+      } trieNodesDone=${trieNodeFetcher.done}`,
     )
 
     if (this.fetcherDoneFlags.done) {
-      this.config.events.emit(Event.SYNC_SNAPSYNC_COMPLETE, this.root, this.stateManager)
+      this.config.events.emit(
+        Event.SYNC_SNAPSYNC_COMPLETE,
+        this.fetcherDoneFlags.snapTargetRoot!,
+        this.stateManager,
+      )
     }
   }
 
@@ -378,18 +390,30 @@ export class AccountFetcher extends Fetcher<JobTask, AccountData[], AccountData>
     // TODOs:
     // 1. Properly rewrite Fetcher with async/await -> allow to at least place in Fetcher.next()
     // 2. Properly implement ETH request IDs -> allow to call on non-idle in Peer Pool
-    await peer?.latest()
+    const latest = await peer?.latest()
+    if (latest !== undefined && latest.stateRoot !== undefined) {
+      // TODO currently doing this check and update both in account and trienode fetchers since they
+      // could be running independently of eachother in some cases
+      const currentHeight = this.height
+      const newestHeight = latest.number
+      if (newestHeight - currentHeight >= this.config.snapLookbackWindow) {
+        this.fetcherDoneFlags.snapTargetHeight = latest.number
+        this.fetcherDoneFlags.snapTargetRoot = latest.stateRoot
+        this.fetcherDoneFlags.snapTargetHash = latest.hash()
+        this.height = newestHeight
+      }
+    }
+
     const origin = this.getOrigin(job)
     const limit = this.getLimit(job)
-
     if (this.highestKnownHash && compareBytes(limit, this.highestKnownHash) < 0) {
       // skip this job and don't rerequest it if it's limit is lower than the highest known key hash
       this.DEBUG && this.debug(`skipping request with limit lower than highest known hash`)
       return Object.assign([], [{ skipped: true }], { completed: true })
     }
 
     const rangeResult = await peer!.snap!.getAccountRange({
-      root: this.root,
+      root: this.fetcherDoneFlags.snapTargetRoot!,
       origin,
       limit,
       bytes: BigInt(this.config.maxRangeBytes),
@@ -406,7 +430,7 @@ export class AccountFetcher extends Fetcher<JobTask, AccountData[], AccountData>
       if (rangeResult.proof.length > 0) {
         try {
           const isMissingRightRange = await verifyMerkleRangeProof(
-            this.root,
+            this.fetcherDoneFlags.snapTargetRoot!,
             origin,
             null,
             [],
@@ -433,7 +457,11 @@ export class AccountFetcher extends Fetcher<JobTask, AccountData[], AccountData>
     try {
       // verifyRangeProof will also verify validate there are no missed states between origin and
       // response data
-      const isMissingRightRange = await this.verifyRangeProof(this.root, origin, rangeResult)
+      const isMissingRightRange = await this.verifyRangeProof(
+        this.fetcherDoneFlags.snapTargetRoot!,
+        origin,
+        rangeResult,
+      )
 
       // Check if there is any pending data to be synced to the right
       let completed: boolean
@@ -586,10 +614,6 @@ export class AccountFetcher extends Fetcher<JobTask, AccountData[], AccountData>
     return tasks
   }
 
-  updateStateRoot(stateRoot: Uint8Array) {
-    this.root = stateRoot
-  }
-
   nextTasks(): void {
     if (
       this.in.length === 0 &&

packages/client/src/sync/fetcher/storagefetcher.ts

@@ -34,15 +34,17 @@ export type StorageRequest = {
   storageRoot: Uint8Array
   first: bigint
   count: bigint
+  stateRoot: Uint8Array // needed for verifying request if targetRoot has been updated since the creation of this request
+  height: bigint // needed for checking if request is still safely within stateRoot-lookback-window before requesting to avoid being banned by peer
 }
 
 /**
  * Implements a snap1 based storage fetcher
  * @memberof module:sync/fetcher
  */
 export interface StorageFetcherOptions extends FetcherOptions {
-  /** Root hash of the account trie to serve */
-  root: Uint8Array
+  // height of block being targeted for snap sync
+  height: bigint
 
   /** Storage requests to fetch */
   storageRequests?: StorageRequest[]
@@ -64,11 +66,13 @@ export interface StorageFetcherOptions extends FetcherOptions {
 export type JobTask = {
   storageRequests: StorageRequest[]
   multi: boolean
+  stateRoot: Uint8Array // needed for verifying request if targetRoot has been updated since the creation of this request
+  height: bigint // needed for checking if request is still safely within stateRoot-lookback-window before requesting to avoid being banned by peer
 }
 
 export class StorageFetcher extends Fetcher<JobTask, StorageData[][], StorageData[]> {
   protected debug: Debugger
-  root: Uint8Array
+  height: bigint
   stateManager: MerkleStateManager
   fetcherDoneFlags: SnapFetcherDoneFlags
 
@@ -87,7 +91,7 @@ export class StorageFetcher extends Fetcher<JobTask, StorageData[][], StorageDat
     super(options)
     this.fragmentedRequests = []
 
-    this.root = options.root
+    this.height = options.height
     this.stateManager = options.stateManager ?? new MerkleStateManager()
     this.fetcherDoneFlags = options.fetcherDoneFlags ?? getInitFetcherDoneFlags()
     this.storageRequests = options.storageRequests ?? []
@@ -105,7 +109,7 @@ export class StorageFetcher extends Fetcher<JobTask, StorageData[][], StorageDat
         this.debug(
           `Storage fetcher instantiated with ${
             fullJob.task.storageRequests.length
-          } accounts requested and root=${short(this.root)} origin=${short(origin)} limit=${short(
+          } accounts requested and root=${short(this.fetcherDoneFlags.snapTargetRoot!)} origin=${short(origin)} limit=${short(
             limit,
           )} destroyWhenDone=${this.destroyWhenDone}`,
         )
@@ -223,12 +227,23 @@ export class StorageFetcher extends Fetcher<JobTask, StorageData[][], StorageDat
     // TODOs:
     // 1. Properly rewrite Fetcher with async/await -> allow to at least place in Fetcher.next()
     // 2. Properly implement ETH request IDs -> allow to call on non-idle in Peer Pool
-    await peer?.latest()
+    const latest = await peer?.latest()
+    if (latest !== undefined && latest.stateRoot !== undefined) {
+      // TODO currently doing this check and update both in account and trienode fetchers since they
+      // could be running independently of eachother in some cases
+      const currentHeight = this.height
+      const newestHeight = latest.number
+      if (newestHeight - currentHeight >= this.config.snapLookbackWindow) {
+        this.fetcherDoneFlags.snapTargetHeight = latest.number
+        this.fetcherDoneFlags.snapTargetRoot = latest.stateRoot
+        this.fetcherDoneFlags.snapTargetHash = latest.hash()
+      }
+    }
 
     const origin = this.getOrigin(job)
     const limit = this.getLimit(job)
 
-    this.DEBUG && this.debug(`requested root: ${bytesToHex(this.root)}`)
+    this.DEBUG && this.debug(`requested root: ${bytesToHex(this.fetcherDoneFlags.snapTargetRoot!)}`)
     this.DEBUG && this.debug(`requested origin: ${bytesToHex(origin)}`)
     this.DEBUG && this.debug(`requested limit: ${bytesToHex(limit)}`)
     this.DEBUG &&
@@ -251,8 +266,17 @@ export class StorageFetcher extends Fetcher<JobTask, StorageData[][], StorageDat
       }
     }
 
+    if (
+      this.fetcherDoneFlags.snapTargetHeight! - (task.height ?? task.storageRequests[0].height) >=
+      this.config.snapLookbackWindow
+    ) {
+      // skip request if we are close to being outside of lookback range to avoid getting banned
+      this.debug(`skipping request that is close to being outside of lookback range`)
+      return Object.assign([], [{ skipped: true }], { completed: true })
+    }
+
     const rangeResult = await peer!.snap!.getStorageRanges({
-      root: this.root,
+      root: task.stateRoot,
       accounts: task.storageRequests.map((req) => req.accountHash),
       origin,
       limit,
@@ -393,6 +417,8 @@ export class StorageFetcher extends Fetcher<JobTask, StorageData[][], StorageDat
               // start fetching from next hash after last slot hash of last account received
               first: bytesToBigInt(highestReceivedhash),
               count: TOTAL_RANGE_END - bytesToBigInt(highestReceivedhash),
+              stateRoot: this.fetcherDoneFlags.snapTargetRoot!, // TODO make sure the flags root or height being undefined isn't a possiblity here
+              height: this.fetcherDoneFlags.snapTargetHeight!,
             } as StorageRequest)
           }
           // finally, we have to requeue account requests after fragmented account that were ignored
@@ -555,6 +581,8 @@ export class StorageFetcher extends Fetcher<JobTask, StorageData[][], StorageDat
       tasks.unshift({
         storageRequests: this.storageRequests, // TODO limit max number of accounts per single fetch request
         multi: true,
+        stateRoot: this.fetcherDoneFlags.snapTargetRoot!, // TODO make sure the flags root or height being undefined isn't a possiblity here
+        height: this.fetcherDoneFlags.snapTargetHeight!,
       })
       this.storageRequests = [] // greedily request as many account slots by requesting all known ones
       return tasks
@@ -579,8 +607,12 @@ export class StorageFetcher extends Fetcher<JobTask, StorageData[][], StorageDat
             storageRoot: storageRequest!.storageRoot,
             first: myFirst,
             count: max,
+            stateRoot: this.fetcherDoneFlags.snapTargetRoot!, // TODO make sure the flags root or height being undefined isn't a possiblity here
+            height: this.fetcherDoneFlags.snapTargetHeight!,
           },
         ],
+        stateRoot: this.fetcherDoneFlags.snapTargetRoot!, // TODO make sure the flags root or height being undefined isn't a possiblity here
+        height: this.fetcherDoneFlags.snapTargetHeight!,
         first: myFirst,
         count: max,
         multi: false,
@@ -598,8 +630,12 @@ export class StorageFetcher extends Fetcher<JobTask, StorageData[][], StorageDat
             storageRoot: storageRequest!.storageRoot,
             first: myFirst,
             count: myCount,
+            stateRoot: this.fetcherDoneFlags.snapTargetRoot!, // TODO make sure the flags root or height being undefined isn't a possiblity here
+            height: this.fetcherDoneFlags.snapTargetHeight!,
           },
         ],
+        stateRoot: this.fetcherDoneFlags.snapTargetRoot!, // TODO make sure the flags root or height being undefined isn't a possiblity here
+        height: this.fetcherDoneFlags.snapTargetHeight!,
         first: myFirst,
         count: myCount,
         multi: false,
@@ -615,6 +651,8 @@ export class StorageFetcher extends Fetcher<JobTask, StorageData[][], StorageDat
     if (myCount !== BIGINT_0 && startedWith === whereFirstWas) {
       // create new fragmented request to keep track of where to start building the next set of tasks for fetching the same account
       this.fragmentedRequests.unshift({
+        stateRoot: this.fetcherDoneFlags.snapTargetRoot!, // TODO make sure the flags root or height being undefined isn't a possiblity here
+        height: this.fetcherDoneFlags.snapTargetHeight!,
         accountHash: storageRequest!.accountHash,
         storageRoot: storageRequest!.storageRoot,
         first: myFirst,