From 56d11b32b8144a609f789a2daa6f0b9a0eeb9eb6 Mon Sep 17 00:00:00 2001 From: holgerd77 Date: Fri, 22 Jan 2021 11:43:12 +0100 Subject: [PATCH 01/12] client -> execution refactor: added new execution module, new Execution abstract base class and dedicated VMExecution class, moved over execution logic from FullSynchronizer --- .../client/lib/sync/execution/execution.ts | 44 +++++++ packages/client/lib/sync/execution/index.ts | 0 .../client/lib/sync/execution/vmexecution.ts | 107 ++++++++++++++++ packages/client/lib/sync/fetcher/fetcher.ts | 2 +- packages/client/lib/sync/fullsync.ts | 115 +++--------------- packages/client/lib/sync/sync.ts | 2 - 6 files changed, 166 insertions(+), 104 deletions(-) create mode 100644 packages/client/lib/sync/execution/execution.ts create mode 100644 packages/client/lib/sync/execution/index.ts create mode 100644 packages/client/lib/sync/execution/vmexecution.ts diff --git a/packages/client/lib/sync/execution/execution.ts b/packages/client/lib/sync/execution/execution.ts new file mode 100644 index 0000000000..0a2db06bf9 --- /dev/null +++ b/packages/client/lib/sync/execution/execution.ts @@ -0,0 +1,44 @@ +import { EventEmitter } from 'events' +import { Config } from '../../config' +import { LevelUp } from 'levelup' +import { Chain } from '../../blockchain' + +export interface ExecutionOptions { + /* Config */ + config: Config + + /* State database */ + stateDB?: LevelUp + + /** Chain */ + chain: Chain +} + +export abstract class Execution extends EventEmitter { + public config: Config + + protected stateDB?: LevelUp + protected chain: Chain + + protected running: boolean = false + + /** + * Create new excution module + * @memberof module:sync/execution + */ + constructor(options: ExecutionOptions) { + super() + + this.config = options.config + this.chain = options.chain + this.stateDB = options.stateDB + } + + /** + * Stop execution. Returns a promise that resolves once stopped. + */ + async stop(): Promise { + this.running = false + return true + } +} diff --git a/packages/client/lib/sync/execution/index.ts b/packages/client/lib/sync/execution/index.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/client/lib/sync/execution/vmexecution.ts b/packages/client/lib/sync/execution/vmexecution.ts new file mode 100644 index 0000000000..3ea632a3b5 --- /dev/null +++ b/packages/client/lib/sync/execution/vmexecution.ts @@ -0,0 +1,107 @@ +import { Execution, ExecutionOptions } from './execution' +import { short } from '../../util' +import VM from '@ethereumjs/vm' +import { DefaultStateManager } from '@ethereumjs/vm/dist/state' +import { SecureTrie as Trie } from '@ethereumjs/trie' + +export class VMExecution extends Execution { + public vm: VM + + private vmPromise?: Promise + private stopSyncing = false + + /** + * Create new VM excution module + */ + constructor(options: ExecutionOptions) { + super(options) + + if (!this.config.vm) { + const trie = new Trie(this.stateDB) + + const stateManager = new DefaultStateManager({ + common: this.config.common, + trie, + }) + + this.vm = new VM({ + common: this.config.common, + blockchain: this.chain.blockchain, + stateManager, + }) + } else { + this.vm = this.config.vm + //@ts-ignore blockchain has readonly property + this.vm.blockchain = this.chain.blockchain + } + } + + /** + * This updates the VM once blocks were put in the VM + */ + async runBlocks() { + if (this.running) { + return + } + this.running = true + let blockCounter = 0 + let txCounter = 0 + const NUM_BLOCKS_PER_LOG_MSG = 50 + try { + let oldHead = Buffer.alloc(0) + const newHeadBlock = await this.vm.blockchain.getHead() + let newHead = newHeadBlock.hash() + let firstHeadBlock = newHeadBlock + let lastHeadBlock = newHeadBlock + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + while (!newHead.equals(oldHead) && !this.stopSyncing) { + oldHead = newHead + this.vmPromise = this.vm.runBlockchain(this.vm.blockchain, 1) + await this.vmPromise + const headBlock = await this.vm.blockchain.getHead() + newHead = headBlock.hash() + if (blockCounter === 0) { + firstHeadBlock = headBlock + } + // check if we did run a new block: + if (!newHead.equals(oldHead)) { + blockCounter += 1 + txCounter += headBlock.transactions.length + lastHeadBlock = headBlock + + if (blockCounter >= NUM_BLOCKS_PER_LOG_MSG) { + const firstNumber = firstHeadBlock.header.number.toNumber() + const firstHash = short(firstHeadBlock.hash()) + const lastNumber = lastHeadBlock.header.number.toNumber() + const lastHash = short(lastHeadBlock.hash()) + this.config.logger.info( + `Executed blocks count=${blockCounter} first=${firstNumber} hash=${firstHash} last=${lastNumber} hash=${lastHash} with txs=${txCounter}` + ) + blockCounter = 0 + txCounter = 0 + } + } + } + } catch (error) { + this.emit('error', error) + } finally { + this.running = false + } + return blockCounter + } + + /** + * Stop VM execution. Returns a promise that resolves once its stopped. + * @returns {Promise} + */ + async stop(): Promise { + this.stopSyncing = true + if (this.vmPromise) { + // ensure that we wait that the VM finishes executing the block (and flushing the trie cache) + await this.vmPromise + } + await this.stateDB?.close() + await super.stop() + return true + } +} diff --git a/packages/client/lib/sync/fetcher/fetcher.ts b/packages/client/lib/sync/fetcher/fetcher.ts index 44d1856ee4..784247f4e2 100644 --- a/packages/client/lib/sync/fetcher/fetcher.ts +++ b/packages/client/lib/sync/fetcher/fetcher.ts @@ -123,7 +123,7 @@ export abstract class Fetcher extends Readable * @param result fetch result * @return {Promise} */ - abstract async store(_result: StorageItem[]): Promise + abstract store(_result: StorageItem[]): Promise /** * Generate list of tasks to fetch diff --git a/packages/client/lib/sync/fullsync.ts b/packages/client/lib/sync/fullsync.ts index 09e98b81d0..6cb4d33092 100644 --- a/packages/client/lib/sync/fullsync.ts +++ b/packages/client/lib/sync/fullsync.ts @@ -4,9 +4,7 @@ import { short } from '../util' import { Synchronizer, SynchronizerOptions } from './sync' import { BlockFetcher } from './fetcher' import { Block } from '@ethereumjs/block' -import VM from '@ethereumjs/vm' -import { DefaultStateManager } from '@ethereumjs/vm/dist/state' -import { SecureTrie as Trie } from '@ethereumjs/trie' +import { VMExecution } from './execution/vmexecution' /** * Implements an ethereum full sync synchronizer @@ -14,110 +12,30 @@ import { SecureTrie as Trie } from '@ethereumjs/trie' */ export class FullSynchronizer extends Synchronizer { private blockFetcher: BlockFetcher | null - - public vm: VM - public runningBlocks: boolean - - private stopSyncing: boolean - private vmPromise?: Promise + private execution: VMExecution constructor(options: SynchronizerOptions) { super(options) this.blockFetcher = null - if (!this.config.vm) { - const trie = new Trie(this.stateDB) - - const stateManager = new DefaultStateManager({ - common: this.config.common, - trie, - }) - - this.vm = new VM({ - common: this.config.common, - blockchain: this.chain.blockchain, - stateManager, - }) - } else { - this.vm = this.config.vm - //@ts-ignore blockchain has readonly property - this.vm.blockchain = this.chain.blockchain - } - - this.runningBlocks = false - this.stopSyncing = false + this.execution = new VMExecution({ + config: options.config, + stateDB: options.stateDB, + chain: options.chain, + }) const self = this this.chain.on('updated', async function () { - // for some reason, if we use .on('updated', this.runBlocks), it runs in the context of the Chain and not in the FullSync context..? - await self.runBlocks() + // for some reason, if we use .on('updated', this.runBlocks) + // it runs in the context of the Chain and not in the FullSync context..? + if (self.running) { + await self.execution.runBlocks() + } }) // eslint-disable-next-line @typescript-eslint/no-floating-promises this.chain.update() } - /** - * This updates the VM once blocks were put in the VM - */ - async runBlocks() { - if (!this.running || this.runningBlocks) { - return - } - this.runningBlocks = true - let blockCounter = 0 - let txCounter = 0 - const NUM_BLOCKS_PER_LOG_MSG = 50 - try { - let oldHead = Buffer.alloc(0) - const newHeadBlock = await this.vm.blockchain.getHead() - let newHead = newHeadBlock.hash() - let firstHeadBlock = newHeadBlock - let lastHeadBlock = newHeadBlock - while (!newHead.equals(oldHead) && !this.stopSyncing) { - oldHead = newHead - this.vmPromise = this.vm.runBlockchain(this.vm.blockchain, 1) - const numExecuted = (await this.vmPromise) as number - if (numExecuted === 0) { - this.config.logger.warn( - `No blocks executed past chain head hash=${short( - newHead - )} number=${newHeadBlock.header.number.toNumber()}` - ) - this.runningBlocks = false - return - } - const headBlock = await this.vm.blockchain.getHead() - newHead = headBlock.hash() - if (blockCounter === 0) { - firstHeadBlock = headBlock - } - // check if we did run a new block: - if (!newHead.equals(oldHead)) { - blockCounter += 1 - txCounter += headBlock.transactions.length - lastHeadBlock = headBlock - - if (blockCounter >= NUM_BLOCKS_PER_LOG_MSG) { - const firstNumber = firstHeadBlock.header.number.toNumber() - const firstHash = short(firstHeadBlock.hash()) - const lastNumber = lastHeadBlock.header.number.toNumber() - const lastHash = short(lastHeadBlock.hash()) - this.config.logger.info( - `Executed blocks count=${blockCounter} first=${firstNumber} hash=${firstHash} last=${lastNumber} hash=${lastHash} with txs=${txCounter}` - ) - blockCounter = 0 - txCounter = 0 - } - } - } - } catch (error) { - this.emit('error', error) - } finally { - this.runningBlocks = false - } - return blockCounter - } - /** * Returns synchronizer type * @return {string} type @@ -254,16 +172,11 @@ export class FullSynchronizer extends Synchronizer { * @return {Promise} */ async stop(): Promise { - this.stopSyncing = true - if (this.vmPromise) { - // ensure that we wait that the VM finishes executing the block (and flushes the trie cache) - await this.vmPromise - } - await this.stateDB?.close() - if (!this.running) { return false } + await this.execution.stop() + if (this.blockFetcher) { this.blockFetcher.destroy() // TODO: Should this be deleted? diff --git a/packages/client/lib/sync/sync.ts b/packages/client/lib/sync/sync.ts index 6190f6db76..2e9cc93dac 100644 --- a/packages/client/lib/sync/sync.ts +++ b/packages/client/lib/sync/sync.ts @@ -35,7 +35,6 @@ export abstract class Synchronizer extends EventEmitter { protected pool: PeerPool protected chain: Chain - protected stateDB?: LevelUp protected flow: FlowControl protected interval: number public running: boolean @@ -52,7 +51,6 @@ export abstract class Synchronizer extends EventEmitter { this.pool = options.pool this.chain = options.chain - this.stateDB = options.stateDB this.flow = options.flow ?? new FlowControl() this.interval = options.interval ?? 1000 this.running = false From 0b4420a7f64ea22732dfe364346176af2fe59a87 Mon Sep 17 00:00:00 2001 From: holgerd77 Date: Fri, 22 Jan 2021 11:56:26 +0100 Subject: [PATCH 02/12] client -> execution refactor: moved tests over to dedicated execution test file --- .../client/lib/sync/execution/execution.ts | 2 +- .../client/lib/sync/execution/vmexecution.ts | 13 +++- packages/client/lib/sync/fullsync.ts | 2 +- .../test/sync/execution/vmexecution.spec.ts | 63 +++++++++++++++++++ packages/client/test/sync/fullsync.spec.ts | 60 ------------------ 5 files changed, 76 insertions(+), 64 deletions(-) create mode 100644 packages/client/test/sync/execution/vmexecution.spec.ts diff --git a/packages/client/lib/sync/execution/execution.ts b/packages/client/lib/sync/execution/execution.ts index 0a2db06bf9..4e8946c4a5 100644 --- a/packages/client/lib/sync/execution/execution.ts +++ b/packages/client/lib/sync/execution/execution.ts @@ -20,7 +20,7 @@ export abstract class Execution extends EventEmitter { protected stateDB?: LevelUp protected chain: Chain - protected running: boolean = false + public running: boolean = false /** * Create new excution module diff --git a/packages/client/lib/sync/execution/vmexecution.ts b/packages/client/lib/sync/execution/vmexecution.ts index 3ea632a3b5..58af9f953c 100644 --- a/packages/client/lib/sync/execution/vmexecution.ts +++ b/packages/client/lib/sync/execution/vmexecution.ts @@ -7,7 +7,7 @@ import { SecureTrie as Trie } from '@ethereumjs/trie' export class VMExecution extends Execution { public vm: VM - private vmPromise?: Promise + private vmPromise?: Promise private stopSyncing = false /** @@ -57,7 +57,16 @@ export class VMExecution extends Execution { while (!newHead.equals(oldHead) && !this.stopSyncing) { oldHead = newHead this.vmPromise = this.vm.runBlockchain(this.vm.blockchain, 1) - await this.vmPromise + const numExecuted = (await this.vmPromise) as number + if (numExecuted === 0) { + this.config.logger.warn( + `No blocks executed past chain head hash=${short( + newHead + )} number=${newHeadBlock.header.number.toNumber()}` + ) + this.running = false + return 0 + } const headBlock = await this.vm.blockchain.getHead() newHead = headBlock.hash() if (blockCounter === 0) { diff --git a/packages/client/lib/sync/fullsync.ts b/packages/client/lib/sync/fullsync.ts index 6cb4d33092..d61ee1fc4e 100644 --- a/packages/client/lib/sync/fullsync.ts +++ b/packages/client/lib/sync/fullsync.ts @@ -12,7 +12,7 @@ import { VMExecution } from './execution/vmexecution' */ export class FullSynchronizer extends Synchronizer { private blockFetcher: BlockFetcher | null - private execution: VMExecution + public execution: VMExecution constructor(options: SynchronizerOptions) { super(options) diff --git a/packages/client/test/sync/execution/vmexecution.spec.ts b/packages/client/test/sync/execution/vmexecution.spec.ts new file mode 100644 index 0000000000..b94a54da11 --- /dev/null +++ b/packages/client/test/sync/execution/vmexecution.spec.ts @@ -0,0 +1,63 @@ +import tape from 'tape-catch' +import td from 'testdouble' +import { BN } from 'ethereumjs-util' +import VM from '@ethereumjs/vm' +import Blockchain from '@ethereumjs/blockchain' +import { Config } from '../../../lib/config' +import { Chain } from '../../../lib/blockchain' +import { VMExecution } from '../../../lib/sync/execution/vmexecution' + +tape('[FullSynchronizer]', async (t) => { + t.test('should initialize with VM provided by config', async (t) => { + const vm = new VM() + const config = new Config({ vm, loglevel: 'error', transports: [] }) + const chain = new Chain({ config }) + const exec = new VMExecution({ + config, + chain, + }) + t.equals(exec.vm, vm, 'provided VM is used') + t.end() + }) + + t.test('should run blocks', async (t) => { + const vm = new VM() + vm.runBlockchain = td.func() + const config = new Config({ vm, loglevel: 'error', transports: [] }) + const blockchain = new Blockchain() as any + const chain = new Chain({ config, blockchain }) + const exec = new VMExecution({ + config, + chain, + }) + const oldHead = exec.vm.blockchain.getHead() + await exec.runBlocks() + t.deepEqual(exec.vm.blockchain.getHead(), oldHead, 'should not modify blockchain on emtpy run') + + blockchain.getHead = td.func() + const getHeadResponse: any = [] + for (let i = 2; i <= 100; i++) { + getHeadResponse.push({ + hash: () => { + return Buffer.from(`hash${i}`) + }, + header: { number: new BN(i) }, + transactions: [i], + }) + } + + td.when(blockchain.getHead()).thenResolve( + { + hash: () => { + return Buffer.from('hash0') + }, + header: { number: new BN(1) }, + transactions: [], + }, + ...getHeadResponse + ) + t.equal(await exec.runBlocks(), 49) + + t.end() + }) +}) diff --git a/packages/client/test/sync/fullsync.spec.ts b/packages/client/test/sync/fullsync.spec.ts index cdedbe697e..0b676d37fc 100644 --- a/packages/client/test/sync/fullsync.spec.ts +++ b/packages/client/test/sync/fullsync.spec.ts @@ -2,10 +2,8 @@ import { EventEmitter } from 'events' import tape from 'tape-catch' import td from 'testdouble' import { BN } from 'ethereumjs-util' -import VM from '@ethereumjs/vm' import { Config } from '../../lib/config' import { Chain } from '../../lib/blockchain' -import Blockchain from '@ethereumjs/blockchain' tape('[FullSynchronizer]', async (t) => { class PeerPool extends EventEmitter { @@ -33,20 +31,6 @@ tape('[FullSynchronizer]', async (t) => { t.end() }) - t.test('should initialize with VM provided by config', async (t) => { - const vm = new VM() - const config = new Config({ vm, loglevel: 'error', transports: [] }) - const pool = new PeerPool() as any - const chain = new Chain({ config }) - const sync = new FullSynchronizer({ - config, - pool, - chain, - }) - t.equals(sync.vm, vm, 'provided VM is used') - t.end() - }) - t.test('should open', async (t) => { const config = new Config({ loglevel: 'error', transports: [] }) const pool = new PeerPool() as any @@ -155,48 +139,4 @@ tape('[FullSynchronizer]', async (t) => { td.reset() t.end() }) - - t.test('should run blocks', async (t) => { - const vm = new VM() - vm.runBlockchain = td.func() - const config = new Config({ vm, loglevel: 'error', transports: [] }) - const pool = new PeerPool() as any - const blockchain = new Blockchain() as any - const chain = new Chain({ config, blockchain }) - const sync = new FullSynchronizer({ - config, - pool, - chain, - }) - const oldHead = sync.vm.blockchain.getHead() - sync.running = true - await sync.runBlocks() - t.deepEqual(sync.vm.blockchain.getHead(), oldHead, 'should not modify blockchain on emtpy run') - - blockchain.getHead = td.func() - const getHeadResponse: any = [] - for (let i = 2; i <= 100; i++) { - getHeadResponse.push({ - hash: () => { - return Buffer.from(`hash${i}`) - }, - header: { number: new BN(i) }, - transactions: [i], - }) - } - - td.when(blockchain.getHead()).thenResolve( - { - hash: () => { - return Buffer.from('hash0') - }, - header: { number: new BN(1) }, - transactions: [], - }, - ...getHeadResponse - ) - t.equal(await sync.runBlocks(), 49) - - t.end() - }) }) From 3bdea5cc94127333638a7494e030dd34969ac985 Mon Sep 17 00:00:00 2001 From: holgerd77 Date: Fri, 22 Jan 2021 13:16:16 +0100 Subject: [PATCH 03/12] client -> execution refactor: directly integrate VM.runBlockchain() code for the client runBlocks() logic --- packages/client/lib/sync/execution/index.ts | 6 + .../client/lib/sync/execution/vmexecution.ts | 127 ++++++++++-------- packages/client/lib/sync/fullsync.ts | 5 +- .../integration/fullethereumservice.spec.ts | 4 +- packages/client/test/integration/util.ts | 2 +- .../test/sync/execution/vmexecution.spec.ts | 17 ++- 6 files changed, 98 insertions(+), 63 deletions(-) diff --git a/packages/client/lib/sync/execution/index.ts b/packages/client/lib/sync/execution/index.ts index e69de29bb2..07baec5913 100644 --- a/packages/client/lib/sync/execution/index.ts +++ b/packages/client/lib/sync/execution/index.ts @@ -0,0 +1,6 @@ +/** + * @module execution + */ + +export * from './execution' +export * from './vmexecution' diff --git a/packages/client/lib/sync/execution/vmexecution.ts b/packages/client/lib/sync/execution/vmexecution.ts index 58af9f953c..3755641bd1 100644 --- a/packages/client/lib/sync/execution/vmexecution.ts +++ b/packages/client/lib/sync/execution/vmexecution.ts @@ -3,12 +3,15 @@ import { short } from '../../util' import VM from '@ethereumjs/vm' import { DefaultStateManager } from '@ethereumjs/vm/dist/state' import { SecureTrie as Trie } from '@ethereumjs/trie' +import { Block } from '@ethereumjs/block' export class VMExecution extends Execution { public vm: VM - private vmPromise?: Promise - private stopSyncing = false + public syncing = false + private vmPromise?: Promise + + private NUM_BLOCKS_PER_ITERATION = 50 /** * Create new VM excution module @@ -36,67 +39,84 @@ export class VMExecution extends Execution { } } - /** - * This updates the VM once blocks were put in the VM - */ async runBlocks() { - if (this.running) { + if (this.running || !this.syncing) { return } this.running = true - let blockCounter = 0 + let txCounter = 0 - const NUM_BLOCKS_PER_LOG_MSG = 50 - try { - let oldHead = Buffer.alloc(0) - const newHeadBlock = await this.vm.blockchain.getHead() - let newHead = newHeadBlock.hash() - let firstHeadBlock = newHeadBlock - let lastHeadBlock = newHeadBlock + let numExecuted: number | undefined + + const blockchain = this.vm.blockchain + let startHeadBlock = await this.vm.blockchain.getHead() + let canonicalHead = await this.vm.blockchain.getLatestBlock() + + let headBlock: Block | undefined + let parentState: Buffer | undefined + + while ( + (numExecuted === undefined || numExecuted === this.NUM_BLOCKS_PER_ITERATION) && + !startHeadBlock.hash().equals(canonicalHead.hash()) && // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - while (!newHead.equals(oldHead) && !this.stopSyncing) { - oldHead = newHead - this.vmPromise = this.vm.runBlockchain(this.vm.blockchain, 1) - const numExecuted = (await this.vmPromise) as number - if (numExecuted === 0) { - this.config.logger.warn( - `No blocks executed past chain head hash=${short( - newHead - )} number=${newHeadBlock.header.number.toNumber()}` - ) - this.running = false - return 0 - } - const headBlock = await this.vm.blockchain.getHead() - newHead = headBlock.hash() - if (blockCounter === 0) { - firstHeadBlock = headBlock - } - // check if we did run a new block: - if (!newHead.equals(oldHead)) { - blockCounter += 1 - txCounter += headBlock.transactions.length - lastHeadBlock = headBlock + this.syncing + ) { + headBlock = undefined + parentState = undefined - if (blockCounter >= NUM_BLOCKS_PER_LOG_MSG) { - const firstNumber = firstHeadBlock.header.number.toNumber() - const firstHash = short(firstHeadBlock.hash()) - const lastNumber = lastHeadBlock.header.number.toNumber() - const lastHash = short(lastHeadBlock.hash()) - this.config.logger.info( - `Executed blocks count=${blockCounter} first=${firstNumber} hash=${firstHash} last=${lastNumber} hash=${lastHash} with txs=${txCounter}` - ) - blockCounter = 0 - txCounter = 0 + this.vmPromise = blockchain.iterator( + 'vm', + async (block: Block, reorg: boolean) => { + // determine starting state for block run + // if we are just starting or if a chain re-org has happened + if (!headBlock || reorg) { + const parentBlock = await blockchain!.getBlock(block.header.parentHash) + parentState = parentBlock.header.stateRoot + // generate genesis state if we are at the genesis block + // we don't have the genesis state + if (!headBlock) { + await this.vm.stateManager.generateCanonicalGenesis() + } else { + parentState = headBlock.header.stateRoot + } } - } + // run block, update head if valid + try { + await this.vm.runBlock({ block, root: parentState }) + txCounter += block.transactions.length + // set as new head block + headBlock = block + } catch (error) { + // remove invalid block + await blockchain!.delBlock(block.header.hash()) + throw error + } + }, + this.NUM_BLOCKS_PER_ITERATION + ) + numExecuted = (await this.vmPromise) as number + + const endHeadBlock = await this.vm.blockchain.getHead() + if (numExecuted > 0) { + const firstNumber = startHeadBlock.header.number.toNumber() + const firstHash = short(startHeadBlock.hash()) + const lastNumber = endHeadBlock.header.number.toNumber() + const lastHash = short(endHeadBlock.hash()) + this.config.logger.info( + `Executed blocks count=${numExecuted} first=${firstNumber} hash=${firstHash} last=${lastNumber} hash=${lastHash} with txs=${txCounter}` + ) + } else { + this.config.logger.warn( + `No blocks executed past chain head hash=${short( + endHeadBlock.hash() + )} number=${endHeadBlock.header.number.toNumber()}` + ) } - } catch (error) { - this.emit('error', error) - } finally { - this.running = false + startHeadBlock = endHeadBlock + canonicalHead = await this.vm.blockchain.getLatestBlock() } - return blockCounter + this.running = false + return numExecuted } /** @@ -104,7 +124,6 @@ export class VMExecution extends Execution { * @returns {Promise} */ async stop(): Promise { - this.stopSyncing = true if (this.vmPromise) { // ensure that we wait that the VM finishes executing the block (and flushing the trie cache) await this.vmPromise diff --git a/packages/client/lib/sync/fullsync.ts b/packages/client/lib/sync/fullsync.ts index d61ee1fc4e..91d4dcc9bf 100644 --- a/packages/client/lib/sync/fullsync.ts +++ b/packages/client/lib/sync/fullsync.ts @@ -161,6 +161,7 @@ export class FullSynchronizer extends Synchronizer { async open(): Promise { await this.chain.open() await this.pool.open() + this.execution.syncing = true const number = this.chain.blocks.height.toString(10) const td = this.chain.blocks.td.toString(10) const hash = this.chain.blocks.latest!.hash() @@ -172,10 +173,12 @@ export class FullSynchronizer extends Synchronizer { * @return {Promise} */ async stop(): Promise { + this.execution.syncing = false + await this.execution.stop() + if (!this.running) { return false } - await this.execution.stop() if (this.blockFetcher) { this.blockFetcher.destroy() diff --git a/packages/client/test/integration/fullethereumservice.spec.ts b/packages/client/test/integration/fullethereumservice.spec.ts index 273495446b..4ae06b6e22 100644 --- a/packages/client/test/integration/fullethereumservice.spec.ts +++ b/packages/client/test/integration/fullethereumservice.spec.ts @@ -17,8 +17,8 @@ tape('[Integration:FullEthereumService]', async (t) => { config: serviceConfig, chain, }) - // Set runningBlocks to true to skip VM execution - service.synchronizer.runningBlocks = true + // Set syncing to false to skip VM execution + service.synchronizer.execution.syncing = false await service.open() await server.start() await service.start() diff --git a/packages/client/test/integration/util.ts b/packages/client/test/integration/util.ts index 5cfba6c976..ad7eda74b6 100644 --- a/packages/client/test/integration/util.ts +++ b/packages/client/test/integration/util.ts @@ -38,7 +38,7 @@ export async function setup( ...serviceOpts, lightserv: true, }) - service.synchronizer.runningBlocks = true + service.synchronizer.execution.syncing = false } await service.open() await service.start() diff --git a/packages/client/test/sync/execution/vmexecution.spec.ts b/packages/client/test/sync/execution/vmexecution.spec.ts index b94a54da11..abc67307fe 100644 --- a/packages/client/test/sync/execution/vmexecution.spec.ts +++ b/packages/client/test/sync/execution/vmexecution.spec.ts @@ -1,6 +1,6 @@ import tape from 'tape-catch' import td from 'testdouble' -import { BN } from 'ethereumjs-util' +//import { BN } from 'ethereumjs-util' import VM from '@ethereumjs/vm' import Blockchain from '@ethereumjs/blockchain' import { Config } from '../../../lib/config' @@ -30,11 +30,18 @@ tape('[FullSynchronizer]', async (t) => { config, chain, }) - const oldHead = exec.vm.blockchain.getHead() + exec.syncing = true + const oldHead = await exec.vm.blockchain.getHead() await exec.runBlocks() - t.deepEqual(exec.vm.blockchain.getHead(), oldHead, 'should not modify blockchain on emtpy run') + t.deepEqual( + (await exec.vm.blockchain.getHead()).hash(), + oldHead.hash(), + 'should not modify blockchain on emtpy run' + ) - blockchain.getHead = td.func() + //TODO: replace with testdata blockchain tests, mocking not feasible on + // block execution getting more complex + /*blockchain.getHead = td.func() const getHeadResponse: any = [] for (let i = 2; i <= 100; i++) { getHeadResponse.push({ @@ -56,7 +63,7 @@ tape('[FullSynchronizer]', async (t) => { }, ...getHeadResponse ) - t.equal(await exec.runBlocks(), 49) + t.equal(await exec.runBlocks(), 49)*/ t.end() }) From 2b6b0038a70332b8e89ff384ecbc0961a5775124 Mon Sep 17 00:00:00 2001 From: holgerd77 Date: Sat, 23 Jan 2021 10:35:41 +0100 Subject: [PATCH 04/12] blockchain: added new static factory helper method Blockchain.fromBlocksData(), added test --- packages/blockchain/src/index.ts | 53 ++++++--- packages/blockchain/test/blocksData.json | 143 +++++++++++++++++++++++ packages/blockchain/test/index.spec.ts | 15 +++ 3 files changed, 194 insertions(+), 17 deletions(-) create mode 100644 packages/blockchain/test/blocksData.json diff --git a/packages/blockchain/src/index.ts b/packages/blockchain/src/index.ts index bb6b378084..a1fb9b7fba 100644 --- a/packages/blockchain/src/index.ts +++ b/packages/blockchain/src/index.ts @@ -1,6 +1,6 @@ import Semaphore from 'semaphore-async-await' import { Address, BN, rlp } from 'ethereumjs-util' -import { Block, BlockHeader } from '@ethereumjs/block' +import { Block, BlockData, BlockHeader } from '@ethereumjs/block' import Ethash from '@ethereumjs/ethash' import Common from '@ethereumjs/common' import { DBManager } from './db/manager' @@ -177,6 +177,41 @@ export default class Blockchain implements BlockchainInterface { */ private _cliqueLatestBlockSigners: CliqueLatestBlockSigners = [] + /** + * Safe creation of a new Blockchain object awaiting the initialization function, + * encouraged method to use when creating a blockchain object. + * + * @param opts Constructor options, see [[BlockchainOptions]] + */ + + public static async create(opts: BlockchainOptions = {}) { + const blockchain = new Blockchain(opts) + await blockchain.initPromise!.catch((e) => { + throw e + }) + return blockchain + } + + /** + * Creates a blockchain from a list of block objects, + * objects must be readable by the `Block.fromBlockData()` method + * + * @param blockData List of block objects + * @param opts Constructor options, see [[BlockchainOptions]] + */ + public static async fromBlocksData(blocksData: BlockData[], opts: BlockchainOptions = {}) { + const blockchain = await Blockchain.create(opts) + for (const blockData of blocksData) { + const common = Object.assign( + Object.create(Object.getPrototypeOf(blockchain._common)), + blockchain._common + ) + const block = Block.fromBlockData(blockData, { common, hardforkByBlockNumber: true }) + await blockchain.putBlock(block) + } + return blockchain + } + /** * Creates new Blockchain object * @@ -236,22 +271,6 @@ export default class Blockchain implements BlockchainInterface { this.initPromise = this._init(opts.genesisBlock) } - /** - * This static constructor safely creates a new Blockchain object, which also - * awaits the initialization function. If this initialization function throws, - * then this constructor will throw as well, and is therefore the encouraged - * method to use when creating a blockchain object. - * @param opts Constructor options, see [[BlockchainOptions]] - */ - - public static async create(opts: BlockchainOptions = {}) { - const blockchain = new Blockchain(opts) - await blockchain.initPromise!.catch((e) => { - throw e - }) - return blockchain - } - /** * Returns an object with metadata about the Blockchain. It's defined for * backwards compatibility. diff --git a/packages/blockchain/test/blocksData.json b/packages/blockchain/test/blocksData.json new file mode 100644 index 0000000000..8b8a8566ed --- /dev/null +++ b/packages/blockchain/test/blocksData.json @@ -0,0 +1,143 @@ +[ + { + "header": { + "parentHash": "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3", + "uncleHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "coinbase": "0x05a56e2d52c817161883f50c441c3228cfe54d9f", + "stateRoot": "0xd67e4d450343046425ae4271474353857ab860dbc0a1dde64b41b5cd3a532bf3", + "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x3ff800000", + "number": "0x1", + "gasLimit": "0x1388", + "gasUsed": "0x0", + "timestamp": "0x55ba4224", + "extraData": "0x476574682f76312e302e302f6c696e75782f676f312e342e32", + "mixHash": "0x969b900de27b6ac6a67742365dd65f55a0526c41fd18e1b16f1a1215c2e66f59", + "nonce": "0x539bd4979fef1ec4" + }, + "transactions": [], + "uncleHeaders": [] + }, + { + "header": { + "parentHash": "0x88e96d4537bea4d9c05d12549907b32561d3bf31f45aae734cdc119f13406cb6", + "uncleHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "coinbase": "0xdd2f1e6e498202e86d8f5442af596580a4f03c2c", + "stateRoot": "0x4943d941637411107494da9ec8bc04359d731bfd08b72b4d0edcbd4cd2ecb341", + "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x3ff001000", + "number": "0x2", + "gasLimit": "0x1388", + "gasUsed": "0x0", + "timestamp": "0x55ba4241", + "extraData": "0x476574682f76312e302e302d30636463373634372f6c696e75782f676f312e34", + "mixHash": "0x2f0790c5aa31ab94195e1f6443d645af5b75c46c04fbf9911711198a0ce8fdda", + "nonce": "0xb853fa261a86aa9e" + }, + "transactions": [], + "uncleHeaders": [] + }, + { + "header": { + "parentHash": "0xb495a1d7e6663152ae92708da4843337b958146015a2802f4193a410044698c9", + "uncleHash": "0x6b17b938c6e4ef18b26ad81b9ca3515f27fd9c4e82aac56a1fd8eab288785e41", + "coinbase": "0x5088d623ba0fcf0131e0897a91734a4d83596aa0", + "stateRoot": "0x76ab0b899e8387436ff2658e2988f83cbf1af1590b9fe9feca3714f8d1824940", + "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x3fe802ffe", + "number": "0x3", + "gasLimit": "0x1388", + "gasUsed": "0x0", + "timestamp": "0x55ba4260", + "extraData": "0x476574682f76312e302e302d66633739643332642f6c696e75782f676f312e34", + "mixHash": "0x65e12eec23fe6555e6bcdb47aa25269ae106e5f16b54e1e92dcee25e1c8ad037", + "nonce": "0x2e9344e0cbde83ce" + }, + "transactions": [], + "uncleHeaders": [ + { + "parentHash": "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3", + "uncleHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "coinbase": "0xc8ebccc5f5689fa8659d83713341e5ad19349448", + "stateRoot": "0x1e6e030581fd1873b4784280859cd3b3c04aa85520f08c304cf5ee63d3935add", + "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x3ff800000", + "number": "0x1", + "gasLimit": "0x1388", + "gasUsed": "0x0", + "timestamp": "0x55ba4242", + "extraData": "0x59617465732052616e64616c6c202d2045746865724e696e6a61", + "mixHash": "0xf8c94dfe61cf26dcdf8cffeda337cf6a903d65c449d7691a022837f6e2d99459", + "nonce": "0x68b769c5451a7aea" + } + ] + }, + { + "header": { + "parentHash": "0x3d6122660cc824376f11ee842f83addc3525e2dd6756b9bcf0affa6aa88cf741", + "uncleHash": "0x83a8da8965660cb6bdf0c37f1b111778e49753c4213bf7c3e280fccfde89f2b5", + "coinbase": "0xc8ebccc5f5689fa8659d83713341e5ad19349448", + "stateRoot": "0xe6d9f6e95a05ee69719c718c6157d0759049ef3dffdba2d48f015d7c8b9933d8", + "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x3fe005ff9", + "number": "0x4", + "gasLimit": "0x1388", + "gasUsed": "0x0", + "timestamp": "0x55ba427d", + "extraData": "0x59617465732052616e64616c6c202d2045746865724e696e6a61", + "mixHash": "0x06ba40902198357cbeac24a86b2ef11e9fdff48d28a421a0055e26476e3ac59f", + "nonce": "0xc2535b5efca9bee0" + }, + "transactions": [], + "uncleHeaders": [ + { + "parentHash": "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3", + "uncleHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "coinbase": "0x5088d623ba0fcf0131e0897a91734a4d83596aa0", + "stateRoot": "0x9a6597b26adc0e5915cfcca537ba493a647cad1c3c923d406cdec6ca49a0a06d", + "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x3ff800000", + "number": "0x1", + "gasLimit": "0x1388", + "gasUsed": "0x0", + "timestamp": "0x55ba4237", + "extraData": "0x476574682f76312e302e302d66633739643332642f6c696e75782f676f312e34", + "mixHash": "0xd045b852770160da169ec793ec0c6e6ff562e473b2bf3f8192dc59842e36f754", + "nonce": "0xdb821a775bf9dace" + } + ] + }, + { + "header": { + "parentHash": "0x23adf5a3be0f5235b36941bcb29b62504278ec5b9cdfa277b992ba4a7a3cd3a2", + "uncleHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "coinbase": "0x05a56e2d52c817161883f50c441c3228cfe54d9f", + "stateRoot": "0x4470f3dc1cc8097394a4ae85302eac3368462b3c1cfa523ffca942c1dd478220", + "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x3fe802004", + "number": "0x5", + "gasLimit": "0x1388", + "gasUsed": "0x0", + "timestamp": "0x55ba4283", + "extraData": "0x476574682f76312e302e302f6c696e75782f676f312e342e32", + "mixHash": "0x17b85b5ec310c4868249fa2f378c83b4f330e2d897e5373a8195946c71d1d19e", + "nonce": "0xfba9d0cff9dc5cf3" + }, + "transactions": [], + "uncleHeaders": [] + } +] \ No newline at end of file diff --git a/packages/blockchain/test/index.spec.ts b/packages/blockchain/test/index.spec.ts index 057ee3ba5f..4b1b670c4f 100644 --- a/packages/blockchain/test/index.spec.ts +++ b/packages/blockchain/test/index.spec.ts @@ -5,6 +5,7 @@ import tape from 'tape' import Blockchain from '../src' import { generateBlockchain, generateBlocks, isConsecutive, createTestDB } from './util' import * as testData from './testdata.json' +import blocksData from './blocksData.json' const level = require('level-mem') @@ -28,6 +29,20 @@ tape('blockchain test', (t) => { st.end() }) + t.test('should initialize correctly with Blockchain.fromBlocksData()', async (st) => { + const common = new Common({ chain: 'mainnet' }) + const blockchain = await Blockchain.fromBlocksData(blocksData, { + validateBlocks: true, + validateConsensus: false, + common, + }) + + const head = await blockchain.getHead() + + st.equals(head.header.number.toNumber(), 5, 'correct block number') + st.end() + }) + t.test('should only initialize with supported consensus validation options', (st) => { let common = new Common({ chain: 'mainnet' }) st.doesNotThrow(() => { From 524ffdac732bc8a37c62ef45758f99c23e318a5c Mon Sep 17 00:00:00 2001 From: holgerd77 Date: Mon, 25 Jan 2021 18:13:10 +0100 Subject: [PATCH 05/12] blockchain: dedicated testdata folder for test fixtures --- packages/blockchain/test/index.spec.ts | 4 ++-- .../test/{blocksData.json => testdata/blocks_mainnet.json} | 0 packages/blockchain/test/{ => testdata}/testdata.json | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename packages/blockchain/test/{blocksData.json => testdata/blocks_mainnet.json} (100%) rename packages/blockchain/test/{ => testdata}/testdata.json (100%) diff --git a/packages/blockchain/test/index.spec.ts b/packages/blockchain/test/index.spec.ts index 4b1b670c4f..9590a5f049 100644 --- a/packages/blockchain/test/index.spec.ts +++ b/packages/blockchain/test/index.spec.ts @@ -4,8 +4,8 @@ import { Block, BlockHeader, BlockOptions } from '@ethereumjs/block' import tape from 'tape' import Blockchain from '../src' import { generateBlockchain, generateBlocks, isConsecutive, createTestDB } from './util' -import * as testData from './testdata.json' -import blocksData from './blocksData.json' +import * as testData from './testdata/testdata.json' +import blocksData from './testdata/blocks_mainnet.json' const level = require('level-mem') diff --git a/packages/blockchain/test/blocksData.json b/packages/blockchain/test/testdata/blocks_mainnet.json similarity index 100% rename from packages/blockchain/test/blocksData.json rename to packages/blockchain/test/testdata/blocks_mainnet.json diff --git a/packages/blockchain/test/testdata.json b/packages/blockchain/test/testdata/testdata.json similarity index 100% rename from packages/blockchain/test/testdata.json rename to packages/blockchain/test/testdata/testdata.json From 21dc84ad98fef29b91da2b6f775e9f29c4dace88 Mon Sep 17 00:00:00 2001 From: holgerd77 Date: Mon, 25 Jan 2021 18:43:07 +0100 Subject: [PATCH 06/12] client -> execution refactor: added test fixture for blockchain creation, reactivated VM execution test --- .../test/sync/execution/vmexecution.spec.ts | 59 +++----- .../client/test/testdata/blocks_mainnet.json | 143 ++++++++++++++++++ 2 files changed, 166 insertions(+), 36 deletions(-) create mode 100644 packages/client/test/testdata/blocks_mainnet.json diff --git a/packages/client/test/sync/execution/vmexecution.spec.ts b/packages/client/test/sync/execution/vmexecution.spec.ts index abc67307fe..5001a2c888 100644 --- a/packages/client/test/sync/execution/vmexecution.spec.ts +++ b/packages/client/test/sync/execution/vmexecution.spec.ts @@ -1,13 +1,12 @@ import tape from 'tape-catch' -import td from 'testdouble' -//import { BN } from 'ethereumjs-util' import VM from '@ethereumjs/vm' import Blockchain from '@ethereumjs/blockchain' import { Config } from '../../../lib/config' import { Chain } from '../../../lib/blockchain' import { VMExecution } from '../../../lib/sync/execution/vmexecution' +import blocksData from './../../testdata/blocks_mainnet.json' -tape('[FullSynchronizer]', async (t) => { +tape('[VMExecution]', async (t) => { t.test('should initialize with VM provided by config', async (t) => { const vm = new VM() const config = new Config({ vm, loglevel: 'error', transports: [] }) @@ -20,50 +19,38 @@ tape('[FullSynchronizer]', async (t) => { t.end() }) - t.test('should run blocks', async (t) => { + function initWithBlockchain(blockchain: Blockchain) { const vm = new VM() - vm.runBlockchain = td.func() + //vm.runBlockchain = td.func() const config = new Config({ vm, loglevel: 'error', transports: [] }) - const blockchain = new Blockchain() as any const chain = new Chain({ config, blockchain }) const exec = new VMExecution({ config, chain, }) exec.syncing = true + return exec + } + + t.test('should run blocks', async (t) => { + let blockchain = new Blockchain({ + validateBlocks: true, + validateConsensus: false, + }) + let exec = initWithBlockchain(blockchain) const oldHead = await exec.vm.blockchain.getHead() await exec.runBlocks() - t.deepEqual( - (await exec.vm.blockchain.getHead()).hash(), - oldHead.hash(), - 'should not modify blockchain on emtpy run' - ) + let newHead = await exec.vm.blockchain.getHead() + t.deepEqual(newHead.hash(), oldHead.hash(), 'should not modify blockchain on emtpy run') - //TODO: replace with testdata blockchain tests, mocking not feasible on - // block execution getting more complex - /*blockchain.getHead = td.func() - const getHeadResponse: any = [] - for (let i = 2; i <= 100; i++) { - getHeadResponse.push({ - hash: () => { - return Buffer.from(`hash${i}`) - }, - header: { number: new BN(i) }, - transactions: [i], - }) - } - - td.when(blockchain.getHead()).thenResolve( - { - hash: () => { - return Buffer.from('hash0') - }, - header: { number: new BN(1) }, - transactions: [], - }, - ...getHeadResponse - ) - t.equal(await exec.runBlocks(), 49)*/ + blockchain = await Blockchain.fromBlocksData(blocksData, { + validateBlocks: true, + validateConsensus: false, + }) + exec = initWithBlockchain(blockchain) + await exec.runBlocks() + newHead = await exec.vm.blockchain.getHead() + t.deepEqual(newHead.header.number.toNumber(), 5, 'should run all blocks from testfile') t.end() }) diff --git a/packages/client/test/testdata/blocks_mainnet.json b/packages/client/test/testdata/blocks_mainnet.json new file mode 100644 index 0000000000..8b8a8566ed --- /dev/null +++ b/packages/client/test/testdata/blocks_mainnet.json @@ -0,0 +1,143 @@ +[ + { + "header": { + "parentHash": "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3", + "uncleHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "coinbase": "0x05a56e2d52c817161883f50c441c3228cfe54d9f", + "stateRoot": "0xd67e4d450343046425ae4271474353857ab860dbc0a1dde64b41b5cd3a532bf3", + "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x3ff800000", + "number": "0x1", + "gasLimit": "0x1388", + "gasUsed": "0x0", + "timestamp": "0x55ba4224", + "extraData": "0x476574682f76312e302e302f6c696e75782f676f312e342e32", + "mixHash": "0x969b900de27b6ac6a67742365dd65f55a0526c41fd18e1b16f1a1215c2e66f59", + "nonce": "0x539bd4979fef1ec4" + }, + "transactions": [], + "uncleHeaders": [] + }, + { + "header": { + "parentHash": "0x88e96d4537bea4d9c05d12549907b32561d3bf31f45aae734cdc119f13406cb6", + "uncleHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "coinbase": "0xdd2f1e6e498202e86d8f5442af596580a4f03c2c", + "stateRoot": "0x4943d941637411107494da9ec8bc04359d731bfd08b72b4d0edcbd4cd2ecb341", + "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x3ff001000", + "number": "0x2", + "gasLimit": "0x1388", + "gasUsed": "0x0", + "timestamp": "0x55ba4241", + "extraData": "0x476574682f76312e302e302d30636463373634372f6c696e75782f676f312e34", + "mixHash": "0x2f0790c5aa31ab94195e1f6443d645af5b75c46c04fbf9911711198a0ce8fdda", + "nonce": "0xb853fa261a86aa9e" + }, + "transactions": [], + "uncleHeaders": [] + }, + { + "header": { + "parentHash": "0xb495a1d7e6663152ae92708da4843337b958146015a2802f4193a410044698c9", + "uncleHash": "0x6b17b938c6e4ef18b26ad81b9ca3515f27fd9c4e82aac56a1fd8eab288785e41", + "coinbase": "0x5088d623ba0fcf0131e0897a91734a4d83596aa0", + "stateRoot": "0x76ab0b899e8387436ff2658e2988f83cbf1af1590b9fe9feca3714f8d1824940", + "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x3fe802ffe", + "number": "0x3", + "gasLimit": "0x1388", + "gasUsed": "0x0", + "timestamp": "0x55ba4260", + "extraData": "0x476574682f76312e302e302d66633739643332642f6c696e75782f676f312e34", + "mixHash": "0x65e12eec23fe6555e6bcdb47aa25269ae106e5f16b54e1e92dcee25e1c8ad037", + "nonce": "0x2e9344e0cbde83ce" + }, + "transactions": [], + "uncleHeaders": [ + { + "parentHash": "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3", + "uncleHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "coinbase": "0xc8ebccc5f5689fa8659d83713341e5ad19349448", + "stateRoot": "0x1e6e030581fd1873b4784280859cd3b3c04aa85520f08c304cf5ee63d3935add", + "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x3ff800000", + "number": "0x1", + "gasLimit": "0x1388", + "gasUsed": "0x0", + "timestamp": "0x55ba4242", + "extraData": "0x59617465732052616e64616c6c202d2045746865724e696e6a61", + "mixHash": "0xf8c94dfe61cf26dcdf8cffeda337cf6a903d65c449d7691a022837f6e2d99459", + "nonce": "0x68b769c5451a7aea" + } + ] + }, + { + "header": { + "parentHash": "0x3d6122660cc824376f11ee842f83addc3525e2dd6756b9bcf0affa6aa88cf741", + "uncleHash": "0x83a8da8965660cb6bdf0c37f1b111778e49753c4213bf7c3e280fccfde89f2b5", + "coinbase": "0xc8ebccc5f5689fa8659d83713341e5ad19349448", + "stateRoot": "0xe6d9f6e95a05ee69719c718c6157d0759049ef3dffdba2d48f015d7c8b9933d8", + "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x3fe005ff9", + "number": "0x4", + "gasLimit": "0x1388", + "gasUsed": "0x0", + "timestamp": "0x55ba427d", + "extraData": "0x59617465732052616e64616c6c202d2045746865724e696e6a61", + "mixHash": "0x06ba40902198357cbeac24a86b2ef11e9fdff48d28a421a0055e26476e3ac59f", + "nonce": "0xc2535b5efca9bee0" + }, + "transactions": [], + "uncleHeaders": [ + { + "parentHash": "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3", + "uncleHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "coinbase": "0x5088d623ba0fcf0131e0897a91734a4d83596aa0", + "stateRoot": "0x9a6597b26adc0e5915cfcca537ba493a647cad1c3c923d406cdec6ca49a0a06d", + "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x3ff800000", + "number": "0x1", + "gasLimit": "0x1388", + "gasUsed": "0x0", + "timestamp": "0x55ba4237", + "extraData": "0x476574682f76312e302e302d66633739643332642f6c696e75782f676f312e34", + "mixHash": "0xd045b852770160da169ec793ec0c6e6ff562e473b2bf3f8192dc59842e36f754", + "nonce": "0xdb821a775bf9dace" + } + ] + }, + { + "header": { + "parentHash": "0x23adf5a3be0f5235b36941bcb29b62504278ec5b9cdfa277b992ba4a7a3cd3a2", + "uncleHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "coinbase": "0x05a56e2d52c817161883f50c441c3228cfe54d9f", + "stateRoot": "0x4470f3dc1cc8097394a4ae85302eac3368462b3c1cfa523ffca942c1dd478220", + "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x3fe802004", + "number": "0x5", + "gasLimit": "0x1388", + "gasUsed": "0x0", + "timestamp": "0x55ba4283", + "extraData": "0x476574682f76312e302e302f6c696e75782f676f312e342e32", + "mixHash": "0x17b85b5ec310c4868249fa2f378c83b4f330e2d897e5373a8195946c71d1d19e", + "nonce": "0xfba9d0cff9dc5cf3" + }, + "transactions": [], + "uncleHeaders": [] + } +] \ No newline at end of file From 2667505e4566a840580d90f253a0d0b3aef96343 Mon Sep 17 00:00:00 2001 From: holgerd77 Date: Mon, 25 Jan 2021 18:58:05 +0100 Subject: [PATCH 07/12] client -> execution refactor: renamed runBlocks() to run(), abstract run() method in Execution class --- packages/client/lib/sync/execution/execution.ts | 7 +++++++ packages/client/lib/sync/execution/vmexecution.ts | 11 ++++++++--- packages/client/lib/sync/fullsync.ts | 2 +- .../client/test/sync/execution/vmexecution.spec.ts | 6 +++--- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/packages/client/lib/sync/execution/execution.ts b/packages/client/lib/sync/execution/execution.ts index 4e8946c4a5..b5fea45b7a 100644 --- a/packages/client/lib/sync/execution/execution.ts +++ b/packages/client/lib/sync/execution/execution.ts @@ -34,6 +34,13 @@ export abstract class Execution extends EventEmitter { this.stateDB = options.stateDB } + /** + * Runs an execution + * + * @returns number quantifying execution run + */ + abstract run(): Promise + /** * Stop execution. Returns a promise that resolves once stopped. */ diff --git a/packages/client/lib/sync/execution/vmexecution.ts b/packages/client/lib/sync/execution/vmexecution.ts index 3755641bd1..a2297b6733 100644 --- a/packages/client/lib/sync/execution/vmexecution.ts +++ b/packages/client/lib/sync/execution/vmexecution.ts @@ -39,9 +39,14 @@ export class VMExecution extends Execution { } } - async runBlocks() { + /** + * Runs the VM execution + * + * @returns number of blocks executed + */ + async run(): Promise { if (this.running || !this.syncing) { - return + return 0 } this.running = true @@ -116,7 +121,7 @@ export class VMExecution extends Execution { canonicalHead = await this.vm.blockchain.getLatestBlock() } this.running = false - return numExecuted + return numExecuted as number } /** diff --git a/packages/client/lib/sync/fullsync.ts b/packages/client/lib/sync/fullsync.ts index 91d4dcc9bf..8dd7f783d8 100644 --- a/packages/client/lib/sync/fullsync.ts +++ b/packages/client/lib/sync/fullsync.ts @@ -29,7 +29,7 @@ export class FullSynchronizer extends Synchronizer { // for some reason, if we use .on('updated', this.runBlocks) // it runs in the context of the Chain and not in the FullSync context..? if (self.running) { - await self.execution.runBlocks() + await self.execution.run() } }) // eslint-disable-next-line @typescript-eslint/no-floating-promises diff --git a/packages/client/test/sync/execution/vmexecution.spec.ts b/packages/client/test/sync/execution/vmexecution.spec.ts index 5001a2c888..e9ae787494 100644 --- a/packages/client/test/sync/execution/vmexecution.spec.ts +++ b/packages/client/test/sync/execution/vmexecution.spec.ts @@ -39,7 +39,7 @@ tape('[VMExecution]', async (t) => { }) let exec = initWithBlockchain(blockchain) const oldHead = await exec.vm.blockchain.getHead() - await exec.runBlocks() + await exec.run() let newHead = await exec.vm.blockchain.getHead() t.deepEqual(newHead.hash(), oldHead.hash(), 'should not modify blockchain on emtpy run') @@ -48,9 +48,9 @@ tape('[VMExecution]', async (t) => { validateConsensus: false, }) exec = initWithBlockchain(blockchain) - await exec.runBlocks() + await exec.run() newHead = await exec.vm.blockchain.getHead() - t.deepEqual(newHead.header.number.toNumber(), 5, 'should run all blocks from testfile') + t.deepEqual(newHead.header.number.toNumber(), 5, 'should run all blocks') t.end() }) From a1a24c5c195efcedc96d43874abe2d68929a4c4b Mon Sep 17 00:00:00 2001 From: holgerd77 Date: Mon, 25 Jan 2021 19:26:51 +0100 Subject: [PATCH 08/12] client -> execution refactor: separated generic common into chainCommon and execCommon to decouple on HF states --- packages/client/bin/cli.ts | 2 +- packages/client/lib/blockchain/chain.ts | 10 +++---- packages/client/lib/config.ts | 28 +++++++++++-------- packages/client/lib/net/peer/rlpxpeer.ts | 2 +- packages/client/lib/net/server/rlpxserver.ts | 2 +- .../client/lib/sync/execution/execution.ts | 4 +-- .../client/lib/sync/execution/vmexecution.ts | 8 +++--- .../client/lib/sync/fetcher/blockfetcher.ts | 4 ++- 8 files changed, 33 insertions(+), 27 deletions(-) diff --git a/packages/client/bin/cli.ts b/packages/client/bin/cli.ts index a655d208ea..7ff031bb5b 100755 --- a/packages/client/bin/cli.ts +++ b/packages/client/bin/cli.ts @@ -114,7 +114,7 @@ async function runNode(config: Config) { client.on('synchronized', () => { config.logger.info('Synchronized') }) - config.logger.info(`Connecting to network: ${config.common.chainName()}`) + config.logger.info(`Connecting to network: ${config.chainCommon.chainName()}`) await client.open() config.logger.info('Synchronizing blockchain...') await client.start() diff --git a/packages/client/lib/blockchain/chain.ts b/packages/client/lib/blockchain/chain.ts index 926e36ec91..334fde457c 100644 --- a/packages/client/lib/blockchain/chain.ts +++ b/packages/client/lib/blockchain/chain.ts @@ -108,7 +108,7 @@ export class Chain extends EventEmitter { options.blockchain ?? new Blockchain({ db: options.chainDB, - common: this.config.common, + common: this.config.chainCommon, validateBlocks: false, validateConsensus: false, }) @@ -138,14 +138,14 @@ export class Chain extends EventEmitter { * @return {number} */ get networkId(): number { - return this.config.common.networkId() + return this.config.chainCommon.networkId() } /** * Genesis block parameters */ get genesis(): GenesisBlockParams { - const genesis = this.config.common.genesis() + const genesis = this.config.chainCommon.genesis() Object.entries(genesis).forEach(([k, v]) => { genesis[k] = toBuffer(v as string) }) @@ -267,7 +267,7 @@ export class Chain extends EventEmitter { } await this.open() blocks = blocks.map((b: Block) => - Block.fromValuesArray(b.raw(), { common: this.config.common }) + Block.fromValuesArray(b.raw(), { common: this.config.chainCommon }) ) await this.blockchain.putBlocks(blocks) await this.update() @@ -302,7 +302,7 @@ export class Chain extends EventEmitter { } await this.open() headers = headers.map((h) => - BlockHeader.fromValuesArray(h.raw(), { common: this.config.common }) + BlockHeader.fromValuesArray(h.raw(), { common: this.config.chainCommon }) ) await this.blockchain.putHeaders(headers) await this.update() diff --git a/packages/client/lib/config.ts b/packages/client/lib/config.ts index 7922086334..ceb2ef0a47 100644 --- a/packages/client/lib/config.ts +++ b/packages/client/lib/config.ts @@ -6,9 +6,10 @@ import { parseTransports } from './util' export interface ConfigOptions { /** - * Specify the chain and hardfork by passing a Common instance. + * Specify the chain by providing a common instance, + * common instance will not be modified by client * - * Default: chain 'mainnet' and hardfork 'chainstart' + * Default: 'mainnet' Common */ common?: Common @@ -104,10 +105,7 @@ export interface ConfigOptions { } export class Config { - // Initialize Common with an explicit 'chainstart' HF set until - // hardfork awareness is implemented within the library - // Also a fix for https://github.com/ethereumjs/ethereumjs-vm/issues/757 - public static readonly COMMON_DEFAULT = new Common({ chain: 'mainnet', hardfork: 'chainstart' }) + public static readonly CHAIN_DEFAULT = 'mainnet' public static readonly SYNCMODE_DEFAULT = 'full' public static readonly LIGHTSERV_DEFAULT = false public static readonly DATADIR_DEFAULT = `./datadir` @@ -119,7 +117,6 @@ export class Config { public static readonly MINPEERS_DEFAULT = 2 public static readonly MAXPEERS_DEFAULT = 25 - public readonly common: Common public readonly logger: Logger public readonly syncmode: string public readonly vm?: VM @@ -133,11 +130,12 @@ export class Config { public readonly minPeers: number public readonly maxPeers: number + public readonly chainCommon: Common + public readonly execCommon: Common + public readonly servers: (RlpxServer | Libp2pServer)[] = [] constructor(options: ConfigOptions = {}) { - // TODO: map chainParams (and lib/util.parseParams) to new Common format - this.common = options.common ?? Config.COMMON_DEFAULT this.syncmode = options.syncmode ?? Config.SYNCMODE_DEFAULT this.vm = options.vm this.lightserv = options.lightserv ?? Config.LIGHTSERV_DEFAULT @@ -150,6 +148,12 @@ export class Config { this.minPeers = options.minPeers ?? Config.MINPEERS_DEFAULT this.maxPeers = options.maxPeers ?? Config.MAXPEERS_DEFAULT + // TODO: map chainParams (and lib/util.parseParams) to new Common format + const common = + options.common ?? new Common({ chain: Config.CHAIN_DEFAULT, hardfork: 'chainstart' }) + this.chainCommon = Object.assign(Object.create(Object.getPrototypeOf(common)), common) + this.execCommon = Object.assign(Object.create(Object.getPrototypeOf(common)), common) + if (options.logger) { if (options.loglevel) { throw new Error('Config initialization with both logger and loglevel options not allowed') @@ -174,7 +178,7 @@ export class Config { // Otherwise parse transports from transports option this.servers = parseTransports(this.transports).map((t) => { if (t.name === 'rlpx') { - t.options.bootnodes = t.options.bootnodes || this.common.bootstrapNodes() + t.options.bootnodes = t.options.bootnodes || this.chainCommon.bootstrapNodes() return new RlpxServer({ config: this, ...t.options }) } else { return new Libp2pServer({ config: this, ...t.options }) @@ -188,7 +192,7 @@ export class Config { * based on syncmode and selected chain (subdirectory of 'datadir') */ getChainDataDirectory(): string { - const networkDirName = this.common.chainName() + const networkDirName = this.chainCommon.chainName() const chainDataDirName = this.syncmode === 'light' ? 'lightchain' : 'chain' const dataDir = `${this.datadir}/${networkDirName}/${chainDataDirName}` @@ -200,7 +204,7 @@ export class Config { * based selected chain (subdirectory of 'datadir') */ getStateDataDirectory(): string { - const networkDirName = this.common.chainName() + const networkDirName = this.chainCommon.chainName() const dataDir = `${this.datadir}/${networkDirName}/state` return dataDir diff --git a/packages/client/lib/net/peer/rlpxpeer.ts b/packages/client/lib/net/peer/rlpxpeer.ts index 860519a1a1..ad9742caf9 100644 --- a/packages/client/lib/net/peer/rlpxpeer.ts +++ b/packages/client/lib/net/peer/rlpxpeer.ts @@ -103,7 +103,7 @@ export class RlpxPeer extends Peer { await Promise.all(this.protocols.map((p) => p.open())) this.rlpx = new Devp2pRLPx(key, { capabilities: RlpxPeer.capabilities(this.protocols), - common: this.config.common, + common: this.config.chainCommon, }) await this.rlpx.connect({ id: Buffer.from(this.id, 'hex'), diff --git a/packages/client/lib/net/server/rlpxserver.ts b/packages/client/lib/net/server/rlpxserver.ts index 102c14ca40..a73c14c1a1 100644 --- a/packages/client/lib/net/server/rlpxserver.ts +++ b/packages/client/lib/net/server/rlpxserver.ts @@ -212,7 +212,7 @@ export class RlpxServer extends Server { capabilities: RlpxPeer.capabilities(Array.from(this.protocols)), remoteClientIdFilter: this.clientFilter, listenPort: this.port, - common: this.config.common, + common: this.config.chainCommon, }) this.rlpx.on('peer:added', async (rlpxPeer: Devp2pRLPxPeer) => { diff --git a/packages/client/lib/sync/execution/execution.ts b/packages/client/lib/sync/execution/execution.ts index b5fea45b7a..811e827dd1 100644 --- a/packages/client/lib/sync/execution/execution.ts +++ b/packages/client/lib/sync/execution/execution.ts @@ -36,9 +36,9 @@ export abstract class Execution extends EventEmitter { /** * Runs an execution - * + * * @returns number quantifying execution run - */ + */ abstract run(): Promise /** diff --git a/packages/client/lib/sync/execution/vmexecution.ts b/packages/client/lib/sync/execution/vmexecution.ts index a2297b6733..bc976d476d 100644 --- a/packages/client/lib/sync/execution/vmexecution.ts +++ b/packages/client/lib/sync/execution/vmexecution.ts @@ -23,12 +23,12 @@ export class VMExecution extends Execution { const trie = new Trie(this.stateDB) const stateManager = new DefaultStateManager({ - common: this.config.common, + common: this.config.chainCommon, trie, }) this.vm = new VM({ - common: this.config.common, + common: this.config.chainCommon, blockchain: this.chain.blockchain, stateManager, }) @@ -41,9 +41,9 @@ export class VMExecution extends Execution { /** * Runs the VM execution - * + * * @returns number of blocks executed - */ + */ async run(): Promise { if (this.running || !this.syncing) { return 0 diff --git a/packages/client/lib/sync/fetcher/blockfetcher.ts b/packages/client/lib/sync/fetcher/blockfetcher.ts index 465d1c1142..52dba0a5f3 100644 --- a/packages/client/lib/sync/fetcher/blockfetcher.ts +++ b/packages/client/lib/sync/fetcher/blockfetcher.ts @@ -51,7 +51,9 @@ export class BlockFetcher extends BlockFetcherBase { await peer!.eth!.getBlockBodies(headers.map((h) => h.hash())) ) const blocks: Block[] = bodies.map(([txsData, unclesData]: BlockBodyBuffer, i: number) => - Block.fromValuesArray([headers[i].raw(), txsData, unclesData], { common: this.config.common }) + Block.fromValuesArray([headers[i].raw(), txsData, unclesData], { + common: this.config.chainCommon, + }) ) return blocks } From 47d160d9ab21ed02d1a401cb6a66e666dd7f5f71 Mon Sep 17 00:00:00 2001 From: holgerd77 Date: Mon, 25 Jan 2021 21:49:10 +0100 Subject: [PATCH 09/12] client, common -> refactor execution: added chain and execution HF logic along initialization and processing, new Common getHardforkByBlockNumber() function --- .../client/lib/sync/execution/vmexecution.ts | 40 +++++++++++++++++-- packages/client/lib/sync/fullsync.ts | 25 +++++++++--- packages/common/src/index.ts | 16 ++++++-- packages/common/tests/hardforks.spec.ts | 13 +++++- 4 files changed, 81 insertions(+), 13 deletions(-) diff --git a/packages/client/lib/sync/execution/vmexecution.ts b/packages/client/lib/sync/execution/vmexecution.ts index bc976d476d..ec2773bb6f 100644 --- a/packages/client/lib/sync/execution/vmexecution.ts +++ b/packages/client/lib/sync/execution/vmexecution.ts @@ -7,6 +7,7 @@ import { Block } from '@ethereumjs/block' export class VMExecution extends Execution { public vm: VM + public hardfork: string = '' public syncing = false private vmPromise?: Promise @@ -23,12 +24,12 @@ export class VMExecution extends Execution { const trie = new Trie(this.stateDB) const stateManager = new DefaultStateManager({ - common: this.config.chainCommon, + common: this.config.execCommon, trie, }) this.vm = new VM({ - common: this.config.chainCommon, + common: this.config.execCommon, blockchain: this.chain.blockchain, stateManager, }) @@ -39,6 +40,17 @@ export class VMExecution extends Execution { } } + /** + * Initializes VM execution. Must be called before run() is called + */ + async open(): Promise { + const headBlock = await this.vm.blockchain.getHead() + const blockNumber = headBlock.header.number.toNumber() + this.config.execCommon.setHardforkByBlockNumber(blockNumber) + this.hardfork = this.config.execCommon.hardfork() + this.config.logger.info(`Initializing VM execution hardfork=${this.hardfork}`) + } + /** * Runs the VM execution * @@ -87,6 +99,15 @@ export class VMExecution extends Execution { } // run block, update head if valid try { + const blockNumber = block.header.number.toNumber() + const hardfork = this.config.execCommon.getHardforkByBlockNumber(blockNumber) + if (hardfork !== this.hardfork) { + const hash = short(block.hash()) + this.config.logger.info( + `Execution hardfork switch on block number=${blockNumber} hash=${hash} old=${this.hardfork} new=${hardfork}` + ) + this.config.execCommon.setHardforkByBlockNumber(blockNumber) + } await this.vm.runBlock({ block, root: parentState }) txCounter += block.transactions.length // set as new head block @@ -94,6 +115,19 @@ export class VMExecution extends Execution { } catch (error) { // remove invalid block await blockchain!.delBlock(block.header.hash()) + const blockNumber = block.header.number.toNumber() + const hash = short(block.hash()) + this.config.logger.warn( + `Deleted block number=${blockNumber} hash=${hash} on failed execution` + ) + + const hardfork = this.config.execCommon.getHardforkByBlockNumber(blockNumber) + if (hardfork !== this.hardfork) { + this.config.logger.warn( + `Set back hardfork along block deletion number=${blockNumber} hash=${hash} old=${this.hardfork} new=${hardfork}` + ) + this.config.execCommon.setHardforkByBlockNumber(blockNumber) + } throw error } }, @@ -108,7 +142,7 @@ export class VMExecution extends Execution { const lastNumber = endHeadBlock.header.number.toNumber() const lastHash = short(endHeadBlock.hash()) this.config.logger.info( - `Executed blocks count=${numExecuted} first=${firstNumber} hash=${firstHash} last=${lastNumber} hash=${lastHash} with txs=${txCounter}` + `Executed blocks count=${numExecuted} first=${firstNumber} hash=${firstHash} hardfork=${this.hardfork} last=${lastNumber} hash=${lastHash} with txs=${txCounter}` ) } else { this.config.logger.warn( diff --git a/packages/client/lib/sync/fullsync.ts b/packages/client/lib/sync/fullsync.ts index 8dd7f783d8..c181c865ab 100644 --- a/packages/client/lib/sync/fullsync.ts +++ b/packages/client/lib/sync/fullsync.ts @@ -12,6 +12,8 @@ import { VMExecution } from './execution/vmexecution' */ export class FullSynchronizer extends Synchronizer { private blockFetcher: BlockFetcher | null + + public hardfork: string = '' public execution: VMExecution constructor(options: SynchronizerOptions) { @@ -100,6 +102,14 @@ export class FullSynchronizer extends Synchronizer { const count = height.sub(first).addn(1) if (count.lten(0)) return false + const nextForkBlock = this.config.chainCommon.nextHardforkBlock() + if (nextForkBlock) { + if (first.gten(nextForkBlock)) { + this.config.chainCommon.setHardforkByBlockNumber(first.toNumber()) + this.hardfork = this.config.chainCommon.hardfork() + } + } + this.config.logger.debug( `Syncing with peer: ${peer.toString(true)} height=${height.toString(10)}` ) @@ -120,9 +130,9 @@ export class FullSynchronizer extends Synchronizer { const first = new BN(blocks[0].header.number) const hash = short(blocks[0].hash()) this.config.logger.info( - `Imported blocks count=${blocks.length} number=${first.toString(10)} hash=${hash} peers=${ - this.pool.size - }` + `Imported blocks count=${blocks.length} number=${first.toString( + 10 + )} hash=${hash} hardfork=${this.hardfork} peers=${this.pool.size}` ) }) await this.blockFetcher.fetch() @@ -160,12 +170,17 @@ export class FullSynchronizer extends Synchronizer { */ async open(): Promise { await this.chain.open() + await this.execution.open() await this.pool.open() this.execution.syncing = true - const number = this.chain.blocks.height.toString(10) + const number = this.chain.blocks.height.toNumber() const td = this.chain.blocks.td.toString(10) const hash = this.chain.blocks.latest!.hash() - this.config.logger.info(`Latest local block: number=${number} td=${td} hash=${short(hash)}`) + this.config.chainCommon.setHardforkByBlockNumber(number) + this.hardfork = this.config.chainCommon.hardfork() + this.config.logger.info( + `Latest local block: number=${number} td=${td} hash=${short(hash)} hardfork=${this.hardfork}` + ) } /** diff --git a/packages/common/src/index.ts b/packages/common/src/index.ts index afe4dbcbf7..a5a7ce5476 100644 --- a/packages/common/src/index.ts +++ b/packages/common/src/index.ts @@ -179,11 +179,11 @@ export default class Common { } /** - * Sets a new hardfork based on the block number provided + * Returns the hardfork based on the block number provided * @param blockNumber - * @returns The name of the HF set + * @returns The name of the HF */ - setHardforkByBlockNumber(blockNumber: number): string { + getHardforkByBlockNumber(blockNumber: number): string { let hardfork = 'chainstart' for (const hf of this.hardforks()) { const hardforkBlock = hf.block @@ -197,6 +197,16 @@ export default class Common { hardfork = hf.name } } + return hardfork + } + + /** + * Sets a new hardfork based on the block number provided + * @param blockNumber + * @returns The name of the HF set + */ + setHardforkByBlockNumber(blockNumber: number): string { + const hardfork = this.getHardforkByBlockNumber(blockNumber) this.setHardfork(hardfork) return hardfork } diff --git a/packages/common/tests/hardforks.spec.ts b/packages/common/tests/hardforks.spec.ts index 9e56ad2ae6..151f658bee 100644 --- a/packages/common/tests/hardforks.spec.ts +++ b/packages/common/tests/hardforks.spec.ts @@ -25,9 +25,18 @@ tape('[Common]: Hardfork logic', function (t: tape.Test) { st.end() }) - t.test('setHardforkByBlockNumber()', function (st: tape.Test) { + t.test('getHardforkByBlockNumber() / setHardforkByBlockNumber()', function (st: tape.Test) { let c = new Common({ chain: 'mainnet' }) - const msg = 'should set HF correctly' + let msg = 'should get HF correctly' + + st.equal(c.getHardforkByBlockNumber(0), 'chainstart', msg) + st.equal(c.getHardforkByBlockNumber(1149999), 'chainstart', msg) + st.equal(c.getHardforkByBlockNumber(1150000), 'homestead', msg) + st.equal(c.getHardforkByBlockNumber(1400000), 'homestead', msg) + st.equal(c.getHardforkByBlockNumber(9200000), 'muirGlacier', msg) + st.equal(c.getHardforkByBlockNumber(999999999999), 'muirGlacier', msg) + + msg = 'should set HF correctly' st.equal(c.setHardforkByBlockNumber(0), 'chainstart', msg) st.equal(c.setHardforkByBlockNumber(1149999), 'chainstart', msg) From e733ce108588238d8db11ae677dfb38c46c019a6 Mon Sep 17 00:00:00 2001 From: holgerd77 Date: Tue, 26 Jan 2021 10:41:55 +0100 Subject: [PATCH 10/12] client -> refactor execution: added HF update test for VM execution --- .../test/sync/execution/vmexecution.spec.ts | 25 +++++--- packages/client/test/sync/fullsync.spec.ts | 9 --- packages/client/test/testdata/testnet.json | 59 +++++++++++++++++++ 3 files changed, 75 insertions(+), 18 deletions(-) create mode 100644 packages/client/test/testdata/testnet.json diff --git a/packages/client/test/sync/execution/vmexecution.spec.ts b/packages/client/test/sync/execution/vmexecution.spec.ts index e9ae787494..34d125ae4d 100644 --- a/packages/client/test/sync/execution/vmexecution.spec.ts +++ b/packages/client/test/sync/execution/vmexecution.spec.ts @@ -5,9 +5,11 @@ import { Config } from '../../../lib/config' import { Chain } from '../../../lib/blockchain' import { VMExecution } from '../../../lib/sync/execution/vmexecution' import blocksData from './../../testdata/blocks_mainnet.json' +import testnet from './../../testdata/testnet.json' +import Common from '@ethereumjs/common' tape('[VMExecution]', async (t) => { - t.test('should initialize with VM provided by config', async (t) => { + t.test('Initialization', async (t) => { const vm = new VM() const config = new Config({ vm, loglevel: 'error', transports: [] }) const chain = new Chain({ config }) @@ -15,43 +17,48 @@ tape('[VMExecution]', async (t) => { config, chain, }) - t.equals(exec.vm, vm, 'provided VM is used') + t.equals(exec.vm, vm, 'should use provided') t.end() }) - function initWithBlockchain(blockchain: Blockchain) { + async function testSetup(blockchain: Blockchain, common?: Common) { const vm = new VM() - //vm.runBlockchain = td.func() - const config = new Config({ vm, loglevel: 'error', transports: [] }) + const config = new Config({ common, vm, loglevel: 'error', transports: [] }) const chain = new Chain({ config, blockchain }) const exec = new VMExecution({ config, chain, }) exec.syncing = true + await exec.open() return exec } - t.test('should run blocks', async (t) => { + t.test('Block execution / Hardforks', async (t) => { let blockchain = new Blockchain({ validateBlocks: true, validateConsensus: false, }) - let exec = initWithBlockchain(blockchain) + let exec = await testSetup(blockchain) const oldHead = await exec.vm.blockchain.getHead() await exec.run() let newHead = await exec.vm.blockchain.getHead() - t.deepEqual(newHead.hash(), oldHead.hash(), 'should not modify blockchain on emtpy run') + t.deepEqual(newHead.hash(), oldHead.hash(), 'should not modify blockchain on empty run') blockchain = await Blockchain.fromBlocksData(blocksData, { validateBlocks: true, validateConsensus: false, }) - exec = initWithBlockchain(blockchain) + exec = await testSetup(blockchain) await exec.run() newHead = await exec.vm.blockchain.getHead() t.deepEqual(newHead.header.number.toNumber(), 5, 'should run all blocks') + const common = new Common({ chain: 'testnet', customChains: [testnet] }) + exec = await testSetup(blockchain, common) + await exec.run() + t.equal(exec.hardfork, 'byzantium', 'should update HF on block run') + t.end() }) }) diff --git a/packages/client/test/sync/fullsync.spec.ts b/packages/client/test/sync/fullsync.spec.ts index 0b676d37fc..d315c39558 100644 --- a/packages/client/test/sync/fullsync.spec.ts +++ b/packages/client/test/sync/fullsync.spec.ts @@ -40,16 +40,7 @@ tape('[FullSynchronizer]', async (t) => { pool, chain, }) - ;(sync as any).chain = { - open: td.func(), - blocks: { - height: '1', - td: '10', - latest: { hash: () => '1234567890' }, - }, - } ;(sync as any).pool.open = td.func() - td.when((sync as any).chain.open()).thenResolve(null) td.when((sync as any).pool.open()).thenResolve(null) await sync.open() t.pass('opened') diff --git a/packages/client/test/testdata/testnet.json b/packages/client/test/testdata/testnet.json new file mode 100644 index 0000000000..d14e88d5d4 --- /dev/null +++ b/packages/client/test/testdata/testnet.json @@ -0,0 +1,59 @@ +{ + "name": "testnet", + "chainId": 12345, + "networkId": 12345, + "defaultHardfork": "byzantium", + "consensus": { + "type": "pow", + "algorithm": "ethash" + }, + "comment": "Private test network", + "url": "[TESTNET_URL]", + "genesis": { + "hash": "0xaa00000000000000000000000000000000000000000000000000000000000000", + "timestamp": null, + "gasLimit": 1000000, + "difficulty": 1, + "nonce": "0xbb00000000000000", + "extraData": "0xcc000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "stateRoot": "0xdd00000000000000000000000000000000000000000000000000000000000000" + }, + "hardforks": [ + { + "name": "chainstart", + "block": 0 + }, + { + "name": "homestead", + "block": 1 + }, + { + "name": "tangerineWhistle", + "block": 2 + }, + { + "name": "spuriousDragon", + "block": 3 + }, + { + "name": "byzantium", + "block": 4 + } + ], + "bootstrapNodes": [ + { + "ip": "10.0.0.1", + "port": 30303, + "id": "11000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "location": "", + "comment": "" + }, + { + "ip": "10.0.0.2", + "port": 30303, + "id": "22000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "location": "", + "comment": "" + } + ] +} From 6d9d632ac41ea07f1aed9fc6f1343e045097fe20 Mon Sep 17 00:00:00 2001 From: holgerd77 Date: Wed, 27 Jan 2021 13:20:54 +0100 Subject: [PATCH 11/12] blockchain, client -> refactor execution: added getIteratorHead(), setIteratorHead() functions, deprecated getHead(), setHead() functions, reactivated VM execution tests --- packages/blockchain/src/index.ts | 43 ++++++++++++++++++- packages/blockchain/test/index.spec.ts | 16 +++++-- .../client/lib/sync/execution/vmexecution.ts | 3 +- packages/client/test/integration/util.ts | 4 +- .../test/sync/execution/vmexecution.spec.ts | 6 +-- 5 files changed, 62 insertions(+), 10 deletions(-) diff --git a/packages/blockchain/src/index.ts b/packages/blockchain/src/index.ts index a1fb9b7fba..042e7febc4 100644 --- a/packages/blockchain/src/index.ts +++ b/packages/blockchain/src/index.ts @@ -641,7 +641,36 @@ export default class Blockchain implements BlockchainInterface { /** * Returns the specified iterator head. * - * @param name - Optional name of the state root head (default: 'vm') + * This function replaces the old `getHead()` method. Note that + * the function deviates from the old behavior and returns the + * genesis hash instead of the current head block if an iterator + * has not been run. This matches the behavior of the `iterator()` + * method. + * + * @param name - Optional name of the iterator head (default: 'vm') + */ + async getIteratorHead(name = 'vm'): Promise { + return await this.initAndLock(async () => { + // if the head is not found return the genesis hash + const hash = this._heads[name] || this._genesis + if (!hash) { + throw new Error('No head found.') + } + + const block = await this._getBlock(hash) + return block + }) + } + + /** + * Returns the specified iterator head. + * + * @param name - Optional name of the iterator head (default: 'vm') + * + * @deprecated use `getIteratorHead()` instead. Note that `getIteratorHead()` + * doesn't return the `headHeader` but the genesis hash as an initial + * iterator head value (now matching the behavior of the `iterator()` + * method on a first run) */ async getHead(name = 'vm'): Promise { return await this.initAndLock(async () => { @@ -1160,6 +1189,18 @@ export default class Blockchain implements BlockchainInterface { * @param tag - The tag to save the headHash to * @param headHash - The head hash to save */ + async setIteratorHead(tag: string, headHash: Buffer) { + return await this.setHead(tag, headHash) + } + + /** + * Set header hash of a certain `tag`. + * When calling the iterator, the iterator will start running the first child block after the header hash currenntly stored. + * @param tag - The tag to save the headHash to + * @param headHash - The head hash to save + * + * @deprecated use `setIteratorHead()` instead + */ async setHead(tag: string, headHash: Buffer) { await this.initAndLock(async () => { this._heads[tag] = headHash diff --git a/packages/blockchain/test/index.spec.ts b/packages/blockchain/test/index.spec.ts index 9590a5f049..d8f5793ea8 100644 --- a/packages/blockchain/test/index.spec.ts +++ b/packages/blockchain/test/index.spec.ts @@ -24,8 +24,18 @@ tape('blockchain test', (t) => { const blockchain = new Blockchain({ common }) const head = await blockchain.getHead() + const iteratorHead = await blockchain.getIteratorHead() - st.equals(head.hash().toString('hex'), common.genesis().hash.slice(2), 'correct genesis hash') + st.equals( + head.hash().toString('hex'), + common.genesis().hash.slice(2), + 'correct genesis hash (getHead())' + ) + st.equals( + iteratorHead.hash().toString('hex'), + common.genesis().hash.slice(2), + 'correct genesis hash (getIteratorHead())' + ) st.end() }) @@ -430,14 +440,14 @@ tape('blockchain test', (t) => { // Note: if st.end() is not called (Promise did not throw), then this test fails, as it does not end. }) - t.test('should test setHead method', async (st) => { + t.test('should test setHead (@deprecated)/setIteratorHead method', async (st) => { const { blockchain, blocks, error } = await generateBlockchain(25) st.error(error, 'no error') const headBlockIndex = 5 const headHash = blocks[headBlockIndex].hash() - await blockchain.setHead('myHead', headHash) + await blockchain.setIteratorHead('myHead', headHash) const currentHeadBlock = await blockchain.getHead('myHead') st.ok(headHash.equals(currentHeadBlock.hash()), 'head hash equals the provided head hash') diff --git a/packages/client/lib/sync/execution/vmexecution.ts b/packages/client/lib/sync/execution/vmexecution.ts index ec2773bb6f..6a362292d9 100644 --- a/packages/client/lib/sync/execution/vmexecution.ts +++ b/packages/client/lib/sync/execution/vmexecution.ts @@ -66,12 +66,11 @@ export class VMExecution extends Execution { let numExecuted: number | undefined const blockchain = this.vm.blockchain - let startHeadBlock = await this.vm.blockchain.getHead() + let startHeadBlock = await this.vm.blockchain.getIteratorHead() let canonicalHead = await this.vm.blockchain.getLatestBlock() let headBlock: Block | undefined let parentState: Buffer | undefined - while ( (numExecuted === undefined || numExecuted === this.NUM_BLOCKS_PER_ITERATION) && !startHeadBlock.hash().equals(canonicalHead.hash()) && diff --git a/packages/client/test/integration/util.ts b/packages/client/test/integration/util.ts index ad7eda74b6..411e40d162 100644 --- a/packages/client/test/integration/util.ts +++ b/packages/client/test/integration/util.ts @@ -38,9 +38,11 @@ export async function setup( ...serviceOpts, lightserv: true, }) - service.synchronizer.execution.syncing = false } await service.open() + if ('execution' in service.synchronizer) { + service.synchronizer.execution.syncing = false + } await service.start() return [server, service] diff --git a/packages/client/test/sync/execution/vmexecution.spec.ts b/packages/client/test/sync/execution/vmexecution.spec.ts index 34d125ae4d..f1f89448f9 100644 --- a/packages/client/test/sync/execution/vmexecution.spec.ts +++ b/packages/client/test/sync/execution/vmexecution.spec.ts @@ -17,13 +17,13 @@ tape('[VMExecution]', async (t) => { config, chain, }) - t.equals(exec.vm, vm, 'should use provided') + t.equals(exec.vm, vm, 'should use vm provided') t.end() }) async function testSetup(blockchain: Blockchain, common?: Common) { - const vm = new VM() - const config = new Config({ common, vm, loglevel: 'error', transports: [] }) + const config = new Config({ common, loglevel: 'error', transports: [] }) + const chain = new Chain({ config, blockchain }) const exec = new VMExecution({ config, From 6fe4a1ddab7c78a5551dce139de4b3e988f2cd7b Mon Sep 17 00:00:00 2001 From: holgerd77 Date: Thu, 28 Jan 2021 12:35:54 +0100 Subject: [PATCH 12/12] common: made Chain interface consensus type specific config parameters optional --- packages/common/src/types.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/common/src/types.ts b/packages/common/src/types.ts index 017df165a5..1fe5ffa98e 100644 --- a/packages/common/src/types.ts +++ b/packages/common/src/types.ts @@ -27,10 +27,11 @@ export interface Chain { consensus?: { type: string algorithm: string - clique: { + clique?: { period: number epoch: number } + ethash?: any } }