Client: use same Cache Setup for normal and executeBlocks-triggered E…

…xecution (#3063)

* Trie: generalize trie shallowCopy() options to allow for more options from the original options to be passed, adopt StateManager usage

* Trie: add additional cache size default 0 for shallowCopy() test

* Trie: add adopt cache size on shallowCopy() test

* StateManager: add downlevelCaches option to shallowCopy(), add tests

* Add downlevelCaches option in VM shallowCopy(), use option for client when run with --executeBlocks flag

* replace any usages in StateManager tests to retain typing

* Apply code review suggestions

---------

Co-authored-by: acolytec3 <17355484+acolytec3@users.noreply.github.com>

packages/client/src/execution/vmexecution.ts

@@ -613,7 +613,7 @@ export class VMExecution extends Execution {
    */
   async executeBlocks(first: number, last: number, txHashes: string[]) {
     this.config.logger.info('Preparing for block execution (debug mode, no services started)...')
-    const vm = await this.vm.shallowCopy()
+    const vm = await this.vm.shallowCopy(false)
 
     for (let blockNumber = first; blockNumber <= last; blockNumber++) {
       const block = await vm.blockchain.getBlock(blockNumber)

packages/common/src/interfaces.ts

@@ -82,7 +82,7 @@ export interface StateManagerInterface {
   setStateRoot(stateRoot: Uint8Array, clearCache?: boolean): Promise<void>
   getProof?(address: Address, storageSlots: Uint8Array[]): Promise<Proof>
   hasStateRoot(root: Uint8Array): Promise<boolean> // only used in client
-  shallowCopy(): StateManagerInterface
+  shallowCopy(downlevelCaches?: boolean): StateManagerInterface
 }
 
 export interface EVMStateManagerInterface extends StateManagerInterface {
@@ -96,5 +96,5 @@ export interface EVMStateManagerInterface extends StateManagerInterface {
   generateCanonicalGenesis(initState: any): Promise<void> // TODO make input more typesafe
   getProof(address: Address, storageSlots?: Uint8Array[]): Promise<Proof>
 
-  shallowCopy(): EVMStateManagerInterface
+  shallowCopy(downlevelCaches?: boolean): EVMStateManagerInterface
 }

packages/statemanager/src/stateManager.ts

@@ -397,7 +397,7 @@ export class DefaultStateManager implements EVMStateManagerInterface {
       const keyPrefix = this._prefixStorageTrieKeys
         ? keccak256(address.bytes).slice(0, 7)
         : undefined
-      const storageTrie = this._trie.shallowCopy(false, keyPrefix)
+      const storageTrie = this._trie.shallowCopy(false, { keyPrefix })
       storageTrie.root(account.storageRoot)
       storageTrie.flushCheckpoints()
       this._storageTries[addressHex] = storageTrie
@@ -948,27 +948,37 @@ export class DefaultStateManager implements EVMStateManagerInterface {
    * at the last fully committed point, i.e. as if all current
    * checkpoints were reverted.
    *
-   * Note on caches:
+   * Caches are downleveled (so: adopted for short-term usage)
+   * by default.
+   *
+   * This means in particular:
    * 1. For caches instantiated as an LRU cache type
    * the copy() method will instantiate with an ORDERED_MAP cache
    * instead, since copied instantances are mostly used in
    * short-term usage contexts and LRU cache instantation would create
    * a large overhead here.
-   * 2. Cache values are generally not copied along
+   * 2. The underlying trie object is initialized with 0 cache size
+   *
+   * Both adoptions can be deactivated by setting `downlevelCaches` to
+   * `false`.
+   *
+   * Cache values are generally not copied along regardless of the
+   * `downlevelCaches` setting.
    */
-  shallowCopy(): DefaultStateManager {
+  shallowCopy(downlevelCaches = true): DefaultStateManager {
     const common = this.common.copy()
     common.setHardfork(this.common.hardfork())
 
-    const trie = this._trie.shallowCopy(false)
+    const cacheSize = !downlevelCaches ? this._trie['_opts']['cacheSize'] : 0
+    const trie = this._trie.shallowCopy(false, { cacheSize })
     const prefixCodeHashes = this._prefixCodeHashes
     const prefixStorageTrieKeys = this._prefixStorageTrieKeys
     let accountCacheOpts = { ...this._accountCacheSettings }
-    if (!this._accountCacheSettings.deactivate) {
+    if (downlevelCaches && !this._accountCacheSettings.deactivate) {
       accountCacheOpts = { ...accountCacheOpts, type: CacheType.ORDERED_MAP }
     }
     let storageCacheOpts = { ...this._storageCacheSettings }
-    if (!this._storageCacheSettings.deactivate) {
+    if (downlevelCaches && !this._storageCacheSettings.deactivate) {
       storageCacheOpts = { ...storageCacheOpts, type: CacheType.ORDERED_MAP }
     }
 

packages/statemanager/test/stateManager.spec.ts

@@ -1,3 +1,4 @@
+import { Trie } from '@ethereumjs/trie'
 import { Address, KECCAK256_RLP, bigIntToBytes, setLengthLeft, utf8ToBytes } from '@ethereumjs/util'
 import { assert, describe, it } from 'vitest'
 
@@ -36,18 +37,21 @@ describe('StateManager -> General', () => {
   })
 
   it(`copy()`, async () => {
+    const trie = new Trie({ cacheSize: 1000 })
     let sm = new DefaultStateManager({
+      trie,
       prefixCodeHashes: false,
     })
 
     let smCopy = sm.shallowCopy()
     assert.equal(
-      (smCopy as any)._prefixCodeHashes,
-      (sm as any)._prefixCodeHashes,
+      smCopy['_prefixCodeHashes'],
+      sm['_prefixCodeHashes'],
       'should retain non-default values'
     )
 
     sm = new DefaultStateManager({
+      trie,
       accountCacheOpts: {
         type: CacheType.LRU,
       },
@@ -58,14 +62,32 @@ describe('StateManager -> General', () => {
 
     smCopy = sm.shallowCopy()
     assert.equal(
-      (smCopy as any)._accountCacheSettings.type,
+      smCopy['_accountCacheSettings'].type,
       CacheType.ORDERED_MAP,
       'should switch to ORDERED_MAP account cache on copy()'
     )
     assert.equal(
-      (smCopy as any)._storageCacheSettings.type,
+      smCopy['_storageCacheSettings'].type,
       CacheType.ORDERED_MAP,
       'should switch to ORDERED_MAP storage cache on copy()'
     )
+    assert.equal(smCopy['_trie']['_opts'].cacheSize, 0, 'should set trie cache size to 0')
+
+    smCopy = sm.shallowCopy(false)
+    assert.equal(
+      smCopy['_accountCacheSettings'].type,
+      CacheType.LRU,
+      'should retain account cache type when deactivate cache downleveling'
+    )
+    assert.equal(
+      smCopy['_storageCacheSettings'].type,
+      CacheType.LRU,
+      'should retain storage cache type when deactivate cache downleveling'
+    )
+    assert.equal(
+      smCopy['_trie']['_opts'].cacheSize,
+      1000,
+      'should retain trie cache size when deactivate cache downleveling'
+    )
   })
 })

packages/trie/src/trie.ts

@@ -38,6 +38,7 @@ import type {
   TrieNode,
   TrieOpts,
   TrieOptsWithDefaults,
+  TrieShallowCopyOpts,
 } from './types.js'
 import type { OnFound } from './util/asyncWalk.js'
 import type { BatchDBOp, DB, PutBatch } from '@ethereumjs/util'
@@ -1043,19 +1044,20 @@ export class Trie {
    * Note on db: the copy will create a reference to the
    * same underlying database.
    *
-   * Note on cache: for memory reasons a copy will not
-   * recreate a new LRU cache but initialize with cache
-   * being deactivated.
+   * Note on cache: for memory reasons a copy will by default
+   * not recreate a new LRU cache but initialize with cache
+   * being deactivated. This behavior can be overwritten by
+   * explicitly setting `cacheSize` as an option on the method.
    *
    * @param includeCheckpoints - If true and during a checkpoint, the copy will contain the checkpointing metadata and will use the same scratch as underlying db.
    */
-  shallowCopy(includeCheckpoints = true, keyPrefix?: Uint8Array): Trie {
+  shallowCopy(includeCheckpoints = true, opts?: TrieShallowCopyOpts): Trie {
     const trie = new Trie({
       ...this._opts,
       db: this._db.db.shallowCopy(),
       root: this.root(),
-      keyPrefix: keyPrefix ?? this._opts.keyPrefix,
       cacheSize: 0,
+      ...(opts ?? {}),
     })
     if (includeCheckpoints && this.hasCheckpoints()) {
       trie._db.setCheckpoints(this._db.checkpoints)

packages/trie/src/types.ts

@@ -88,6 +88,11 @@ export type TrieOptsWithDefaults = TrieOpts & {
   cacheSize: number
 }
 
+export interface TrieShallowCopyOpts {
+  keyPrefix?: Uint8Array
+  cacheSize?: number
+}
+
 export interface CheckpointDBOpts {
   /**
    * A database instance.

packages/trie/test/trie/secure.spec.ts

@@ -31,10 +31,19 @@ describe('SecureTrie', () => {
     assert.isUndefined(t['_opts']['keyPrefix'])
   })
 
-  it('copy trie (new key prefix)', async () => {
-    const prefix = hexToBytes('0x1234')
-    const t = trie.shallowCopy(true, prefix)
-    assert.ok(equalsBytes(t['_opts']['keyPrefix'] as Uint8Array, prefix))
+  it('copy trie (new key prefix / default 0 size cache)', async () => {
+    const keyPrefix = hexToBytes('0x1234')
+    const t = trie.shallowCopy(true, { keyPrefix })
+    assert.ok(equalsBytes(t['_opts']['keyPrefix'] as Uint8Array, keyPrefix))
+    assert.equal(t['_opts']['cacheSize'] as number, 0)
+    assert.equal(trie['_opts']['cacheSize'] as number, 0)
+  })
+
+  it('copy trie (new cache size)', async () => {
+    const cacheSize = 1000
+    const t = trie.shallowCopy(true, { cacheSize })
+    assert.equal(t['_opts']['cacheSize'] as number, cacheSize)
+    assert.equal(trie['_opts']['cacheSize'] as number, 0)
   })
 })
 

packages/vm/src/vm.ts

@@ -239,13 +239,21 @@ export class VM {
   /**
    * Returns a copy of the {@link VM} instance.
    *
-   * Note that the returned copy will share the same db as the original for the blockchain and the statemanager
+   * Note that the returned copy will share the same db as the original for the blockchain and the statemanager.
+   *
+   * Associated caches will be deleted and caches will be re-initialized for a more short-term focused
+   * usage, being less memory intense (the statemanager caches will switch to using an ORDERED_MAP cache
+   * datastructure more suitable for short-term usage, the trie node LRU cache will not be activated at all).
+   * To fine-tune this behavior (if the shallow-copy-returned object has a longer life span e.g.) you can set
+   * the `downlevelCaches` option to `false`.
+   *
+   * @param downlevelCaches Downlevel (so: adopted for short-term usage) associated state caches (default: true)
    */
-  async shallowCopy(): Promise<VM> {
+  async shallowCopy(downlevelCaches = true): Promise<VM> {
     const common = this.common.copy()
     common.setHardfork(this.common.hardfork())
     const blockchain = this.blockchain.shallowCopy()
-    const stateManager = this.stateManager.shallowCopy()
+    const stateManager = this.stateManager.shallowCopy(downlevelCaches)
     const evmOpts = {
       ...(this.evm as any)._optsCached,
       common,