From 38eb6417f4fc935ca3245f35e447ccb4d25d8483 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Mon, 19 Jul 2021 15:24:05 +0800 Subject: [PATCH 01/32] all: work for eth1/2 transtition --- accounts/abi/bind/backends/simulated.go | 2 +- cmd/geth/config.go | 13 +- cmd/utils/flags.go | 17 +- consensus/beacon/consensus.go | 414 ++++++++++++++++++++ consensus/clique/clique_test.go | 6 +- consensus/clique/snapshot_test.go | 2 +- core/bench_test.go | 4 +- core/block_validator_test.go | 175 ++++++++- core/blockchain.go | 488 ++++++++++++++++------- core/blockchain_repair_test.go | 6 +- core/blockchain_sethead_test.go | 2 +- core/blockchain_snapshot_test.go | 24 +- core/blockchain_test.go | 494 ++++++++++++++++++++---- core/chain_makers.go | 6 + core/chain_makers_test.go | 2 +- core/dao_test.go | 12 +- core/forkchoice.go | 123 ++++++ core/genesis_test.go | 2 +- core/headerchain.go | 251 ++++++------ core/headerchain_test.go | 25 +- core/merger.go | 134 +++++++ core/rawdb/accessors_metadata.go | 13 + core/rawdb/database.go | 2 +- core/rawdb/schema.go | 3 + core/state_processor_test.go | 6 +- eth/api_backend.go | 2 +- eth/backend.go | 23 +- eth/catalyst/api.go | 180 +++++++-- eth/catalyst/api_test.go | 233 +++++++---- eth/catalyst/api_types.go | 12 +- eth/catalyst/gen_blockparams.go | 16 +- eth/catalyst/gen_ed.go | 34 +- eth/catalyst/sync.go | 74 ++++ eth/downloader/downloader.go | 3 + eth/ethconfig/config.go | 55 +-- eth/gasprice/gasprice_test.go | 2 +- eth/handler.go | 67 +++- eth/handler_eth.go | 12 + eth/handler_eth_test.go | 6 +- eth/handler_test.go | 3 +- eth/protocols/eth/handler_test.go | 2 +- eth/state_accessor.go | 6 +- eth/sync.go | 5 +- eth/tracers/api_test.go | 2 +- les/client.go | 8 +- les/client_handler.go | 14 +- les/fetcher.go | 24 +- les/test_helper.go | 3 +- light/lightchain.go | 53 ++- light/lightchain_test.go | 6 +- light/odr_test.go | 4 +- light/trie_test.go | 2 +- light/txpool_test.go | 4 +- miner/miner.go | 4 +- miner/miner_test.go | 4 +- miner/stress/beacon/main.go | 489 +++++++++++++++++++++++ miner/worker.go | 12 +- miner/worker_test.go | 6 +- tests/block_test_util.go | 2 +- tests/fuzzers/les/les-fuzzer.go | 2 +- 60 files changed, 2960 insertions(+), 640 deletions(-) create mode 100644 consensus/beacon/consensus.go create mode 100644 core/forkchoice.go create mode 100644 core/merger.go create mode 100644 eth/catalyst/sync.go create mode 100644 miner/stress/beacon/main.go diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 6854c9624e35..1f8a663bfe40 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -78,7 +78,7 @@ type SimulatedBackend struct { func NewSimulatedBackendWithDatabase(database ethdb.Database, alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend { genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc} genesis.MustCommit(database) - blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}, nil, nil, core.NewMerger(database)) backend := &SimulatedBackend{ database: database, diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 08b9a1154f89..487dd53d6251 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -32,7 +32,6 @@ import ( "github.com/ethereum/go-ethereum/accounts/scwallet" "github.com/ethereum/go-ethereum/accounts/usbwallet" "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/eth/catalyst" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/log" @@ -159,17 +158,7 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { if ctx.GlobalIsSet(utils.OverrideArrowGlacierFlag.Name) { cfg.Eth.OverrideArrowGlacier = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideArrowGlacierFlag.Name)) } - backend, eth := utils.RegisterEthService(stack, &cfg.Eth) - - // Configure catalyst. - if ctx.GlobalBool(utils.CatalystFlag.Name) { - if eth == nil { - utils.Fatalf("Catalyst does not work in light client mode.") - } - if err := catalyst.Register(stack, eth); err != nil { - utils.Fatalf("%v", err) - } - } + backend, _ := utils.RegisterEthService(stack, &cfg.Eth, ctx.GlobalBool(utils.CatalystFlag.Name)) // Configure GraphQL if requested if ctx.GlobalIsSet(utils.GraphQLEnabledFlag.Name) { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index b3d9f395b064..3403bc231372 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -45,6 +45,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/catalyst" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/gasprice" @@ -1182,7 +1183,7 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) { cfg.NetRestrict = list } - if ctx.GlobalBool(DeveloperFlag.Name) || ctx.GlobalBool(CatalystFlag.Name) { + if ctx.GlobalBool(DeveloperFlag.Name) { // --dev mode can't use p2p networking. cfg.MaxPeers = 0 cfg.ListenAddr = "" @@ -1683,13 +1684,18 @@ func SetDNSDiscoveryDefaults(cfg *ethconfig.Config, genesis common.Hash) { // RegisterEthService adds an Ethereum client to the stack. // The second return value is the full node instance, which may be nil if the // node is running as a light client. -func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) (ethapi.Backend, *eth.Ethereum) { +func RegisterEthService(stack *node.Node, cfg *ethconfig.Config, isCatalyst bool) (ethapi.Backend, *eth.Ethereum) { if cfg.SyncMode == downloader.LightSync { backend, err := les.New(stack, cfg) if err != nil { Fatalf("Failed to register the Ethereum service: %v", err) } stack.RegisterAPIs(tracers.APIs(backend.ApiBackend)) + if isCatalyst { + if err := catalyst.RegisterLight(stack, backend); err != nil { + Fatalf("Failed to register the catalyst service: %v", err) + } + } return backend.ApiBackend, nil } backend, err := eth.New(stack, cfg) @@ -1702,6 +1708,11 @@ func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) (ethapi.Backend Fatalf("Failed to create the LES server: %v", err) } } + if isCatalyst { + if err := catalyst.Register(stack, backend); err != nil { + Fatalf("Failed to register the catalyst service: %v", err) + } + } stack.RegisterAPIs(tracers.APIs(backend.APIBackend)) return backend.APIBackend, backend } @@ -1891,7 +1902,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai // TODO(rjl493456442) disable snapshot generation/wiping if the chain is read only. // Disable transaction indexing/unindexing by default. - chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil, nil) + chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil, nil, core.NewMerger(chainDb)) if err != nil { Fatalf("Can't create BlockChain: %v", err) } diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go new file mode 100644 index 000000000000..878cb60fa2c2 --- /dev/null +++ b/consensus/beacon/consensus.go @@ -0,0 +1,414 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package beacon + +import ( + "errors" + "fmt" + "math/big" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/trie" +) + +// Proof-of-stake protocol constants. +var ( + beaconDifficulty = common.Big1 // The default block difficulty in the beacon consensus + beaconNonce = types.EncodeNonce(0) // The default block nonce in the beacon consensus +) + +// Various error messages to mark blocks invalid. These should be private to +// prevent engine specific errors from being referenced in the remainder of the +// codebase, inherently breaking if the engine is swapped out. Please put common +// error types into the consensus package. +var ( + errTooManyUncles = errors.New("too many uncles") + errInvalidDifficulty = errors.New("invalid difficulty") + errInvalidMixDigest = errors.New("invalid mix digest") + errInvalidNonce = errors.New("invalid nonce") + errInvalidUncleHash = errors.New("invalid uncle hash") +) + +// Beacon is a consensus engine combines the ethereum 1 consensus and proof-of-stake +// algorithm. There is a special flag inside to decide whether to use legacy consensus +// rules or new rules. The transition rule is described in the eth1/2 merge spec. +// https://hackmd.io/@n0ble/ethereum_consensus_upgrade_mainnet_perspective#Transition-process +// +// The beacon here is a half-functional consensus engine with partial functions which +// is only used for necessary consensus checks. The legacy consensus engine can be any +// engine implements the consensus interface(except the beacon itself). +type Beacon struct { + ethone consensus.Engine // Classic consensus engine used in the eth1, ethash or clique + + // transitioned is the flag whether the transition has been triggered. + // It's triggered by receiving the first "NewHead" message from the + // external consensus engine. + transitioned bool + lock sync.RWMutex +} + +// New creates a consensus engine with the given embedded ethereum 1 engine. +func New(ethone consensus.Engine, transitioned bool) *Beacon { + if _, ok := ethone.(*Beacon); ok { + panic("nested consensus engine") + } + return &Beacon{ + ethone: ethone, + transitioned: transitioned, + } +} + +// Author implements consensus.Engine, returning the header's coinbase as the +// verified author of the block. +func (beacon *Beacon) Author(header *types.Header) (common.Address, error) { + if !beacon.IsPostMergeHeader(header) { + return beacon.ethone.Author(header) + } + return header.Coinbase, nil +} + +// VerifyHeader checks whether a header conforms to the consensus rules of the +// stock Ethereum consensus engine. +func (beacon *Beacon) VerifyHeader(chain consensus.ChainHeaderReader, header *types.Header, seal bool) error { + if !beacon.IsPostMergeHeader(header) { + return beacon.ethone.VerifyHeader(chain, header, seal) + } + // Short circuit if the header is known, or its parent not + number := header.Number.Uint64() + if chain.GetHeader(header.Hash(), number) != nil { + return nil + } + parent := chain.GetHeader(header.ParentHash, number-1) + if parent == nil { + return consensus.ErrUnknownAncestor + } + // Sanity checks passed, do a proper verification + return beacon.verifyHeader(chain, header, parent) +} + +// VerifyHeaders is similar to VerifyHeader, but verifies a batch of headers +// concurrently. The method returns a quit channel to abort the operations and +// a results channel to retrieve the async verifications. +func (beacon *Beacon) VerifyHeaders(chain consensus.ChainHeaderReader, headers []*types.Header, seals []bool) (chan<- struct{}, <-chan error) { + if !beacon.IsPostMergeHeader(headers[len(headers)-1]) { + return beacon.ethone.VerifyHeaders(chain, headers, seals) + } + var ( + preHeaders []*types.Header + postHeaders []*types.Header + preSeals []bool + ) + for index, header := range headers { + if beacon.IsPostMergeHeader(header) { + preHeaders = headers[:index] + postHeaders = headers[index:] + preSeals = seals[:index] + break + } + } + // All the headers have passed the transition point, use new rules. + if len(preHeaders) == 0 { + return beacon.verifyHeaders(chain, headers, nil) + } + // The transition point exists in the middle, separate the headers + // into two batches and apply different verification rules for them. + var ( + abort = make(chan struct{}) + results = make(chan error, len(headers)) + ) + go func() { + var ( + old, new, out = 0, len(preHeaders), 0 + errors = make([]error, len(headers)) + done = make([]bool, len(headers)) + oldDone, oldResult = beacon.ethone.VerifyHeaders(chain, preHeaders, preSeals) + newDone, newResult = beacon.verifyHeaders(chain, postHeaders, preHeaders[len(preHeaders)-1]) + ) + for { + for ; done[out]; out++ { + results <- errors[out] + if out == len(headers)-1 { + return + } + } + select { + case err := <-oldResult: + errors[old], done[old] = err, true + old++ + case err := <-newResult: + errors[new], done[new] = err, true + new++ + case <-abort: + close(oldDone) + close(newDone) + return + } + } + }() + return abort, results +} + +// VerifyUncles verifies that the given block's uncles conform to the consensus +// rules of the stock Ethereum consensus engine. +func (beacon *Beacon) VerifyUncles(chain consensus.ChainReader, block *types.Block) error { + if !beacon.IsPostMergeHeader(block.Header()) { + return beacon.ethone.VerifyUncles(chain, block) + } + // Verify that there is no uncle block. It's explicitly disabled in the beacon + if len(block.Uncles()) > 0 { + return errTooManyUncles + } + return nil +} + +// verifyHeader checks whether a header conforms to the consensus rules of the +// stock Ethereum consensus engine. The difference between the beacon and ethash is +// (a) the difficulty, mixhash, nonce, extradata and unclehash are expected +// to be the desired constants +// (b) the timestamp is not verified anymore +func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, parent *types.Header) error { + // Ensure that the header's extra-data section is of a reasonable size + if len(header.Extra) != 0 { + return fmt.Errorf("non-empty extra-data(%d)", len(header.Extra)) + } + // Verify the block's difficulty to ensure it's the default constant + if beaconDifficulty.Cmp(header.Difficulty) != 0 { + return fmt.Errorf("invalid difficulty: have %v, want %v", header.Difficulty, beaconDifficulty) + } + // Verify that the gas limit is <= 2^63-1 + cap := uint64(0x7fffffffffffffff) + if header.GasLimit > cap { + return fmt.Errorf("invalid gasLimit: have %v, max %v", header.GasLimit, cap) + } + // Verify that the gasUsed is <= gasLimit + if header.GasUsed > header.GasLimit { + return fmt.Errorf("invalid gasUsed: have %d, gasLimit %d", header.GasUsed, header.GasLimit) + } + // Verify that the gas limit remains within allowed bounds + diff := int64(parent.GasLimit) - int64(header.GasLimit) + if diff < 0 { + diff *= -1 + } + limit := parent.GasLimit / params.GasLimitBoundDivisor + if uint64(diff) >= limit || header.GasLimit < params.MinGasLimit { + return fmt.Errorf("invalid gas limit: have %d, want %d += %d", header.GasLimit, parent.GasLimit, limit) + } + // Verify that the block number is parent's +1 + if diff := new(big.Int).Sub(header.Number, parent.Number); diff.Cmp(big.NewInt(1)) != 0 { + return consensus.ErrInvalidNumber + } + // Verify the seal parts. Ensure the mixhash, nonce and uncle hash are the expected value. + if header.MixDigest != (common.Hash{}) { + return errInvalidMixDigest + } + if header.Nonce != beaconNonce { + return errInvalidNonce + } + if header.UncleHash != types.EmptyUncleHash { + return errInvalidUncleHash + } + return nil +} + +// verifyHeaders is similar to verifyHeader, but verifies a batch of headers +// concurrently. The method returns a quit channel to abort the operations and +// a results channel to retrieve the async verifications. An additional parent +// header will be passed if the relevant header is not in the database yet. +func (beacon *Beacon) verifyHeaders(chain consensus.ChainHeaderReader, headers []*types.Header, ancestor *types.Header) (chan<- struct{}, <-chan error) { + var ( + abort = make(chan struct{}) + results = make(chan error, len(headers)) + ) + go func() { + for i, header := range headers { + var parent *types.Header + if i == 0 { + if ancestor != nil { + parent = ancestor + } else { + parent = chain.GetHeader(headers[0].ParentHash, headers[0].Number.Uint64()-1) + } + } else if headers[i-1].Hash() == headers[i].ParentHash { + parent = headers[i-1] + } + if parent == nil { + select { + case <-abort: + return + case results <- consensus.ErrUnknownAncestor: + } + continue + } + err := beacon.verifyHeader(chain, header, parent) + select { + case <-abort: + return + case results <- err: + } + } + }() + return abort, results +} + +// Prepare implements consensus.Engine, initializing the difficulty field of a +// header to conform to the beacon protocol. The changes are done inline. +func (beacon *Beacon) Prepare(chain consensus.ChainHeaderReader, header *types.Header) error { + beacon.lock.RLock() + defer beacon.lock.RUnlock() + + // Transition isn't triggered yet, use the legacy rules for preparation. + if !beacon.transitioned { + return beacon.ethone.Prepare(chain, header) + } + parent := chain.GetHeader(header.ParentHash, header.Number.Uint64()-1) + if parent == nil { + return consensus.ErrUnknownAncestor + } + header.Difficulty = beaconDifficulty + return nil +} + +// Finalize implements consensus.Engine, setting the final state on the header +func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, uncles []*types.Header) { + beacon.lock.RLock() + defer beacon.lock.RUnlock() + + // Finalize is different with Prepare, it can be used in both block generation + // and verification. So determine the consensus rules by header type. + if !beacon.IsPostMergeHeader(header) { + beacon.ethone.Finalize(chain, header, state, txs, uncles) + return + } + // The block reward is no longer handled here. It's done by the + // external consensus engine. + header.Root = state.IntermediateRoot(chain.Config().IsEIP158(header.Number)) +} + +// FinalizeAndAssemble implements consensus.Engine, setting the final state and +// assembling the block. +func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, uncles []*types.Header, receipts []*types.Receipt) (*types.Block, error) { + beacon.lock.RLock() + defer beacon.lock.RUnlock() + + // FinalizeAndAssemble is different with Prepare, it can be used in both block + // generation and verification. So determine the consensus rules by header type. + if !beacon.IsPostMergeHeader(header) { + return beacon.ethone.FinalizeAndAssemble(chain, header, state, txs, uncles, receipts) + } + // Finalize and assemble the block + beacon.Finalize(chain, header, state, txs, uncles) + return types.NewBlock(header, txs, uncles, receipts, trie.NewStackTrie(nil)), nil +} + +// Seal generates a new sealing request for the given input block and pushes +// the result into the given channel. +// +// Note, the method returns immediately and will send the result async. More +// than one result may also be returned depending on the consensus algorithm. +func (beacon *Beacon) Seal(chain consensus.ChainHeaderReader, block *types.Block, results chan<- *types.Block, stop <-chan struct{}) error { + beacon.lock.RLock() + defer beacon.lock.RUnlock() + + if !beacon.IsPostMergeHeader(block.Header()) { + return beacon.ethone.Seal(chain, block, results, stop) + } + // The seal verification is done by the external consensus engine, + // return directly without pushing any block back. In another word + // beacon won't return any result by `results` channel which may + // blocks the receiver logic forever. + return nil +} + +// SealHash returns the hash of a block prior to it being sealed. +func (beacon *Beacon) SealHash(header *types.Header) common.Hash { + return beacon.ethone.SealHash(header) +} + +// CalcDifficulty is the difficulty adjustment algorithm. It returns +// the difficulty that a new block should have when created at time +// given the parent block's time and difficulty. +func (beacon *Beacon) CalcDifficulty(chain consensus.ChainHeaderReader, time uint64, parent *types.Header) *big.Int { + beacon.lock.RLock() + defer beacon.lock.RUnlock() + + // Transition isn't triggered yet, use the legacy rules for calculation + if !beacon.transitioned { + return beacon.ethone.CalcDifficulty(chain, time, parent) + } + return beaconDifficulty +} + +// APIs implements consensus.Engine, returning the user facing RPC APIs. +func (beacon *Beacon) APIs(chain consensus.ChainHeaderReader) []rpc.API { + return beacon.ethone.APIs(chain) +} + +// Close shutdowns the consensus engine +func (beacon *Beacon) Close() error { + return beacon.ethone.Close() +} + +// IsPostMergeHeader reports the header belongs to the PoS-stage with some special fields. +// This function is not suitable for a part of APIs like Prepare or CalcDifficulty because +// the header difficulty is not set yet. +func (beacon *Beacon) IsPostMergeHeader(header *types.Header) bool { + // These fields can be used to filter out ethash block + if header.Difficulty.Cmp(beaconDifficulty) != 0 { + return false + } + if header.MixDigest != (common.Hash{}) { + return false + } + if header.Nonce != beaconNonce { + return false + } + // Extra field can be used to filter out clique block + if len(header.Extra) != 0 { + return false + } + return true +} + +// MarkTransitioned sets the transitioned flag. +func (beacon *Beacon) MarkTransitioned() { + beacon.lock.Lock() + defer beacon.lock.Unlock() + + beacon.transitioned = true +} + +// InnerEngine returns the embedded eth1 consensus engine. +func (beacon *Beacon) InnerEngine() consensus.Engine { + return beacon.ethone +} + +// SetThreads updates the mining threads. Delegate the call +// to the eth1 engine if it's threaded. +func (beacon *Beacon) SetThreads(threads int) { + type threaded interface { + SetThreads(threads int) + } + if th, ok := beacon.ethone.(threaded); ok { + th.SetThreads(threads) + } +} diff --git a/consensus/clique/clique_test.go b/consensus/clique/clique_test.go index 1bd32acd3746..354da78a87dd 100644 --- a/consensus/clique/clique_test.go +++ b/consensus/clique/clique_test.go @@ -55,7 +55,7 @@ func TestReimportMirroredState(t *testing.T) { genesis := genspec.MustCommit(db) // Generate a batch of blocks, each properly signed - chain, _ := core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil) + chain, _ := core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) defer chain.Stop() blocks, _ := core.GenerateChain(params.AllCliqueProtocolChanges, genesis, engine, db, 3, func(i int, block *core.BlockGen) { @@ -89,7 +89,7 @@ func TestReimportMirroredState(t *testing.T) { db = rawdb.NewMemoryDatabase() genspec.MustCommit(db) - chain, _ = core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil) + chain, _ = core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) defer chain.Stop() if _, err := chain.InsertChain(blocks[:2]); err != nil { @@ -102,7 +102,7 @@ func TestReimportMirroredState(t *testing.T) { // Simulate a crash by creating a new chain on top of the database, without // flushing the dirty states out. Insert the last block, triggering a sidechain // reimport. - chain, _ = core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil) + chain, _ = core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) defer chain.Stop() if _, err := chain.InsertChain(blocks[2:]); err != nil { diff --git a/consensus/clique/snapshot_test.go b/consensus/clique/snapshot_test.go index 094868ca744d..1168c434439c 100644 --- a/consensus/clique/snapshot_test.go +++ b/consensus/clique/snapshot_test.go @@ -450,7 +450,7 @@ func TestClique(t *testing.T) { batches[len(batches)-1] = append(batches[len(batches)-1], block) } // Pass all the headers through clique and ensure tallying succeeds - chain, err := core.NewBlockChain(db, nil, &config, engine, vm.Config{}, nil, nil) + chain, err := core.NewBlockChain(db, nil, &config, engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Errorf("test %d: failed to create test chain: %v", i, err) continue diff --git a/core/bench_test.go b/core/bench_test.go index 959979763d66..201dfbc9bdf8 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -201,7 +201,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { // Time the insertion of the new chain. // State and blocks are stored in the same DB. - chainman, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + chainman, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) defer chainman.Stop() b.ReportAllocs() b.ResetTimer() @@ -316,7 +316,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) { if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) } - chain, err := NewBlockChain(db, &cacheConfig, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) + chain, err := NewBlockChain(db, &cacheConfig, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { b.Fatalf("error creating chain: %v", err) } diff --git a/core/block_validator_test.go b/core/block_validator_test.go index 86f9835a0150..80168f6c0fe2 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -17,14 +17,21 @@ package core import ( + "encoding/json" + "math/big" "runtime" "testing" "time" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/consensus/beacon" + "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" ) @@ -42,7 +49,7 @@ func TestHeaderVerification(t *testing.T) { headers[i] = block.Header() } // Run the header checker for blocks one-by-one, checking for both valid and invalid nonces - chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) + chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) defer chain.Stop() for i := 0; i < len(blocks); i++ { @@ -76,6 +83,166 @@ func TestHeaderVerification(t *testing.T) { } } +func TestHeaderVerificationForMergingClique(t *testing.T) { testHeaderVerificationForMerging(t, true) } +func TestHeaderVerificationForMergingEthash(t *testing.T) { testHeaderVerificationForMerging(t, false) } + +// Tests the verification for eth1/2 merging, including pre-merge and post-merge +func testHeaderVerificationForMerging(t *testing.T, isClique bool) { + var ( + testdb = rawdb.NewMemoryDatabase() + preBlocks []*types.Block + postBlocks []*types.Block + runEngine consensus.Engine + chainConfig *params.ChainConfig + ) + if isClique { + var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + addr = crypto.PubkeyToAddress(key.PublicKey) + engine = clique.New(params.AllCliqueProtocolChanges.Clique, testdb) + ) + genspec := &Genesis{ + ExtraData: make([]byte, 32+common.AddressLength+crypto.SignatureLength), + Alloc: map[common.Address]GenesisAccount{ + addr: {Balance: big.NewInt(1)}, + }, + BaseFee: big.NewInt(params.InitialBaseFee), + } + copy(genspec.ExtraData[32:], addr[:]) + genesis := genspec.MustCommit(testdb) + + genEngine := beacon.New(engine, false) + preBlocks, _ = GenerateChain(params.AllCliqueProtocolChanges, genesis, genEngine, testdb, 8, nil) + for i, block := range preBlocks { + header := block.Header() + if i > 0 { + header.ParentHash = preBlocks[i-1].Hash() + } + header.Extra = make([]byte, 32+crypto.SignatureLength) + header.Difficulty = big.NewInt(2) + + sig, _ := crypto.Sign(genEngine.SealHash(header).Bytes(), key) + copy(header.Extra[len(header.Extra)-crypto.SignatureLength:], sig) + preBlocks[i] = block.WithSeal(header) + } + genEngine.MarkTransitioned() + postBlocks, _ = GenerateChain(params.AllCliqueProtocolChanges, preBlocks[len(preBlocks)-1], genEngine, testdb, 8, nil) + chainConfig = params.AllCliqueProtocolChanges + runEngine = beacon.New(engine, false) + } else { + gspec := &Genesis{Config: params.TestChainConfig} + genesis := gspec.MustCommit(testdb) + engine := beacon.New(ethash.NewFaker(), false) + + preBlocks, _ = GenerateChain(params.TestChainConfig, genesis, engine, testdb, 8, nil) + engine.MarkTransitioned() + postBlocks, _ = GenerateChain(params.TestChainConfig, preBlocks[len(preBlocks)-1], engine, testdb, 8, nil) + + chainConfig = params.TestChainConfig + runEngine = beacon.New(ethash.NewFaker(), false) + } + + preHeaders := make([]*types.Header, len(preBlocks)) + for i, block := range preBlocks { + preHeaders[i] = block.Header() + + blob, _ := json.Marshal(block.Header()) + t.Logf("Log header before the merging %d: %v", block.NumberU64(), string(blob)) + } + postHeaders := make([]*types.Header, len(postBlocks)) + for i, block := range postBlocks { + postHeaders[i] = block.Header() + + blob, _ := json.Marshal(block.Header()) + t.Logf("Log header after the merging %d: %v", block.NumberU64(), string(blob)) + } + + // Run the header checker for blocks one-by-one, checking for both valid and invalid nonces + merger := NewMerger(rawdb.NewMemoryDatabase()) + merger.SubscribeLeavePoW(func() { + runEngine.(*beacon.Beacon).MarkTransitioned() + }) + chain, _ := NewBlockChain(testdb, nil, chainConfig, runEngine, vm.Config{}, nil, nil, merger) + defer chain.Stop() + + // Verify the blocks before the merging + for i := 0; i < len(preBlocks); i++ { + _, results := runEngine.VerifyHeaders(chain, []*types.Header{preHeaders[i]}, []bool{true}) + // Wait for the verification result + select { + case result := <-results: + if result != nil { + t.Errorf("test %d: verification failed %v", i, result) + } + case <-time.After(time.Second): + t.Fatalf("test %d: verification timeout", i) + } + // Make sure no more data is returned + select { + case result := <-results: + t.Fatalf("test %d: unexpected result returned: %v", i, result) + case <-time.After(25 * time.Millisecond): + } + chain.InsertChain(preBlocks[i : i+1]) + } + + // Make the transition + merger.LeavePoW() + merger.EnterPoS() + + // Verify the blocks after the merging + for i := 0; i < len(postBlocks); i++ { + _, results := runEngine.VerifyHeaders(chain, []*types.Header{postHeaders[i]}, []bool{true}) + // Wait for the verification result + select { + case result := <-results: + if result != nil { + t.Errorf("test %d: verification failed %v", i, result) + } + case <-time.After(time.Second): + t.Fatalf("test %d: verification timeout", i) + } + // Make sure no more data is returned + select { + case result := <-results: + t.Fatalf("test %d: unexpected result returned: %v", i, result) + case <-time.After(25 * time.Millisecond): + } + chain.InsertChain(postBlocks[i : i+1]) + } + + // Verify the blocks with pre-merge blocks and post-merge blocks + var ( + headers []*types.Header + seals []bool + ) + for _, block := range preBlocks { + headers = append(headers, block.Header()) + seals = append(seals, true) + } + for _, block := range postBlocks { + headers = append(headers, block.Header()) + seals = append(seals, true) + } + _, results := runEngine.VerifyHeaders(chain, headers, seals) + for i := 0; i < len(headers); i++ { + select { + case result := <-results: + if result != nil { + t.Errorf("test %d: verification failed %v", i, result) + } + case <-time.After(time.Second): + t.Fatalf("test %d: verification timeout", i) + } + } + // Make sure no more data is returned + select { + case result := <-results: + t.Fatalf("unexpected result returned: %v", result) + case <-time.After(25 * time.Millisecond): + } +} + // Tests that concurrent header verification works, for both good and bad blocks. func TestHeaderConcurrentVerification2(t *testing.T) { testHeaderConcurrentVerification(t, 2) } func TestHeaderConcurrentVerification8(t *testing.T) { testHeaderConcurrentVerification(t, 8) } @@ -106,11 +273,11 @@ func testHeaderConcurrentVerification(t *testing.T, threads int) { var results <-chan error if valid { - chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) + chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) _, results = chain.engine.VerifyHeaders(chain, headers, seals) chain.Stop() } else { - chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeFailer(uint64(len(headers)-1)), vm.Config{}, nil, nil) + chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeFailer(uint64(len(headers)-1)), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) _, results = chain.engine.VerifyHeaders(chain, headers, seals) chain.Stop() } @@ -173,7 +340,7 @@ func testHeaderConcurrentAbortion(t *testing.T, threads int) { defer runtime.GOMAXPROCS(old) // Start the verifications and immediately abort - chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeDelayer(time.Millisecond), vm.Config{}, nil, nil) + chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeDelayer(time.Millisecond), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) defer chain.Stop() abort, results := chain.engine.VerifyHeaders(chain, headers, seals) diff --git a/core/blockchain.go b/core/blockchain.go index ff372870dd75..a2553737a560 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -22,7 +22,6 @@ import ( "fmt" "io" "math/big" - mrand "math/rand" "sort" "sync" "sync/atomic" @@ -208,15 +207,17 @@ type BlockChain struct { validator Validator // Block and state validator interface prefetcher Prefetcher processor Processor // Block transaction processor interface + forker *ForkChoice vmConfig vm.Config - shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block. + shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block. + terminateInsert func(common.Hash, uint64) bool // Testing hook used to terminate ancient receipt chain insertion. } // NewBlockChain returns a fully initialised block chain using information -// available in the database. It initialises the default Ethereum Validator and -// Processor. -func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(block *types.Block) bool, txLookupLimit *uint64) (*BlockChain, error) { +// available in the database. It initialises the default Ethereum Validator +// and Processor. +func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(header *types.Header) bool, txLookupLimit *uint64, merger *Merger) (*BlockChain, error) { if cacheConfig == nil { cacheConfig = defaultCacheConfig } @@ -237,18 +238,20 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par Journal: cacheConfig.TrieCleanJournal, Preimages: cacheConfig.Preimages, }), - quit: make(chan struct{}), - chainmu: syncx.NewClosableMutex(), - shouldPreserve: shouldPreserve, - bodyCache: bodyCache, - bodyRLPCache: bodyRLPCache, - receiptsCache: receiptsCache, - blockCache: blockCache, - txLookupCache: txLookupCache, - futureBlocks: futureBlocks, - engine: engine, - vmConfig: vmConfig, - } + quit: make(chan struct{}), + bodyCache: bodyCache, + bodyRLPCache: bodyRLPCache, + receiptsCache: receiptsCache, + blockCache: blockCache, + txLookupCache: txLookupCache, + futureBlocks: futureBlocks, + engine: engine, + vmConfig: vmConfig, + } + bc.forker = NewForkChoice(bc, merger.LeftPoW(), shouldPreserve) + merger.SubscribeLeavePoW(func() { + bc.forker.MarkTransitioned() + }) bc.validator = NewBlockValidator(chainConfig, bc, engine) bc.prefetcher = newStatePrefetcher(chainConfig, bc, engine) bc.processor = NewStateProcessor(chainConfig, bc, engine) @@ -382,7 +385,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par // Start future block processor. bc.wg.Add(1) - go bc.futureBlocksLoop() + go bc.update() // Start tx indexer/unindexer. if txLookupLimit != nil { @@ -748,6 +751,8 @@ func (bc *BlockChain) writeHeadBlock(block *types.Block) { rawdb.WriteHeadBlockHash(batch, block.Hash()) // If the block is better than our head or is on a different chain, force update heads + // TODO(rjl493456442) What if the (header head/fast block head) is lower than the chain + // head? Is it possible in practise? if updateHeads { rawdb.WriteHeadHeaderHash(batch, block.Hash()) rawdb.WriteHeadFastBlockHash(batch, block.Hash()) @@ -928,13 +933,14 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ // Rewind may have occurred, skip in that case. if bc.CurrentHeader().Number.Cmp(head.Number()) >= 0 { - currentFastBlock, td := bc.CurrentFastBlock(), bc.GetTd(head.Hash(), head.NumberU64()) - if bc.GetTd(currentFastBlock.Hash(), currentFastBlock.NumberU64()).Cmp(td) < 0 { - rawdb.WriteHeadFastBlockHash(bc.db, head.Hash()) - bc.currentFastBlock.Store(head) - headFastBlockGauge.Update(int64(head.NumberU64())) - return true + reorg, err := bc.forker.Reorg(bc.CurrentFastBlock().Header(), head.Header()) + if err != nil || !reorg { + return false } + rawdb.WriteHeadFastBlockHash(bc.db, head.Hash()) + bc.currentFastBlock.Store(head) + headFastBlockGauge.Update(int64(head.NumberU64())) + return true } return false } @@ -1181,30 +1187,18 @@ func (bc *BlockChain) writeKnownBlock(block *types.Block) error { return nil } -// WriteBlockWithState writes the block and all associated state to the database. -func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.Receipt, logs []*types.Log, state *state.StateDB, emitHeadEvent bool) (status WriteStatus, err error) { - if !bc.chainmu.TryLock() { - return NonStatTy, errInsertionInterrupted - } - defer bc.chainmu.Unlock() - return bc.writeBlockWithState(block, receipts, logs, state, emitHeadEvent) -} - -// writeBlockWithState writes the block and all associated state to the database, -// but is expects the chain mutex to be held. -func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.Receipt, logs []*types.Log, state *state.StateDB, emitHeadEvent bool) (status WriteStatus, err error) { - if bc.insertStopped() { - return NonStatTy, errInsertionInterrupted - } +// writeBlockWithState writes block, metadata and corresponding state data to the +// database. +func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.Receipt, logs []*types.Log, state *state.StateDB) error { + bc.wg.Add(1) + defer bc.wg.Done() // Calculate the total difficulty of the block ptd := bc.GetTd(block.ParentHash(), block.NumberU64()-1) if ptd == nil { - return NonStatTy, consensus.ErrUnknownAncestor + return consensus.ErrUnknownAncestor } // Make sure no inconsistent state is leaked during insertion - currentBlock := bc.CurrentBlock() - localTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64()) externTd := new(big.Int).Add(block.Difficulty(), ptd) // Irrelevant of the canonical status, write the block itself to the database. @@ -1222,15 +1216,13 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. // Commit all cached state changes into underlying memory database. root, err := state.Commit(bc.chainConfig.IsEIP158(block.Number())) if err != nil { - return NonStatTy, err + return err } triedb := bc.stateCache.TrieDB() // If we're running an archive node, always flush if bc.cacheConfig.TrieDirtyDisabled { - if err := triedb.Commit(root, false, nil); err != nil { - return NonStatTy, err - } + return triedb.Commit(root, false, nil) } else { // Full but not archive node, do proper garbage collection triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive @@ -1278,23 +1270,30 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. } } } - // If the total difficulty is higher than our known, add it to the canonical chain - // Second clause in the if statement reduces the vulnerability to selfish mining. - // Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf - reorg := externTd.Cmp(localTd) > 0 - currentBlock = bc.CurrentBlock() - if !reorg && externTd.Cmp(localTd) == 0 { - // Split same-difficulty blocks by number, then preferentially select - // the block generated by the local miner as the canonical block. - if block.NumberU64() < currentBlock.NumberU64() { - reorg = true - } else if block.NumberU64() == currentBlock.NumberU64() { - var currentPreserve, blockPreserve bool - if bc.shouldPreserve != nil { - currentPreserve, blockPreserve = bc.shouldPreserve(currentBlock), bc.shouldPreserve(block) - } - reorg = !currentPreserve && (blockPreserve || mrand.Float64() < 0.5) - } + return nil +} + +// WriteBlockWithState writes the block and all associated state to the database. +func (bc *BlockChain) WriteBlockAndSetHead(block *types.Block, receipts []*types.Receipt, logs []*types.Log, state *state.StateDB, emitHeadEvent bool) (status WriteStatus, err error) { + if !bc.chainmu.TryLock() { + return NonStatTy, errChainStopped + } + defer bc.chainmu.Unlock() + + return bc.writeBlockAndSetHead(block, receipts, logs, state, emitHeadEvent) +} + +// writeBlockAndSetHead writes the block and all associated state to the database, +// and also it applies the given block as the new chain head. This function expects +// the chain mutex to be held. +func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types.Receipt, logs []*types.Log, state *state.StateDB, emitHeadEvent bool) (status WriteStatus, err error) { + if err := bc.writeBlockWithState(block, receipts, logs, state); err != nil { + return NonStatTy, err + } + currentBlock := bc.CurrentBlock() + reorg, err := bc.forker.Reorg(currentBlock.Header(), block.Header()) + if err != nil { + return NonStatTy, err } if reorg { // Reorganise the chain if the parent is not the head block @@ -1320,7 +1319,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. } // In theory we should fire a ChainHeadEvent when we inject // a canonical block, but sometimes we can insert a batch of - // canonicial blocks. Avoid firing too much ChainHeadEvents, + // canonicial blocks. Avoid firing too many ChainHeadEvents, // we will fire an accumulated ChainHeadEvent and disable fire // event here. if emitHeadEvent { @@ -1335,6 +1334,9 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. // addFutureBlock checks if the block is within the max allowed window to get // accepted for future processing, and returns an error if the block is too far // ahead and was not added. +// +// TODO after the transition, the future block shouldn't be kept. Because +// it's not checked in the Geth side anymore. func (bc *BlockChain) addFutureBlock(block *types.Block) error { max := uint64(time.Now().Unix() + maxTimeFutureBlocks) if block.Time() > max { @@ -1347,15 +1349,12 @@ func (bc *BlockChain) addFutureBlock(block *types.Block) error { // InsertChain attempts to insert the given batch of blocks in to the canonical // chain or, otherwise, create a fork. If an error is returned it will return // the index number of the failing block as well an error describing what went -// wrong. -// -// After insertion is done, all accumulated events will be fired. +// wrong. After insertion is done, all accumulated events will be fired. func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { // Sanity check that we have something meaningful to import if len(chain) == 0 { return 0, nil } - bc.blockProcFeed.Send(true) defer bc.blockProcFeed.Send(false) @@ -1374,8 +1373,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { prev.Hash().Bytes()[:4], i, block.NumberU64(), block.Hash().Bytes()[:4], block.ParentHash().Bytes()[:4]) } } - - // Pre-check passed, start the full block imports. + // Pre-checks passed, start the full block imports if !bc.chainmu.TryLock() { return 0, errChainStopped } @@ -1383,19 +1381,6 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { return bc.insertChain(chain, true) } -// InsertChainWithoutSealVerification works exactly the same -// except for seal verification, seal verification is omitted -func (bc *BlockChain) InsertChainWithoutSealVerification(block *types.Block) (int, error) { - bc.blockProcFeed.Send(true) - defer bc.blockProcFeed.Send(false) - - if !bc.chainmu.TryLock() { - return 0, errChainStopped - } - defer bc.chainmu.Unlock() - return bc.insertChain(types.Blocks([]*types.Block{block}), false) -} - // insertChain is the internal implementation of InsertChain, which assumes that // 1) chains are contiguous, and 2) The chain mutex is held. // @@ -1446,14 +1431,23 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er // from the canonical chain, which has not been verified. // Skip all known blocks that are behind us. var ( - current = bc.CurrentBlock() - localTd = bc.GetTd(current.Hash(), current.NumberU64()) - externTd = bc.GetTd(block.ParentHash(), block.NumberU64()-1) // The first block can't be nil + reorg bool + current = bc.CurrentBlock() ) for block != nil && bc.skipBlock(err, it) { - externTd = new(big.Int).Add(externTd, block.Difficulty()) - if localTd.Cmp(externTd) < 0 { - break + reorg, err = bc.forker.Reorg(current.Header(), block.Header()) + if err != nil { + return it.index, err + } + if reorg { + // Switch to import mode if the forker says the reorg is necessary + // and also the block is not on the canonical chain. + // In eth2 the forker always returns True for reorg desision(blindly trust + // the external consensus engine), but in order to prevent the unnecessary + // reorgs when importing known blocks, the special case is handled here. + if bc.GetCanonicalHash(block.NumberU64()) != block.Hash() || block.NumberU64() > current.NumberU64() { + break + } } log.Debug("Ignoring already known block", "number", block.Number(), "hash", block.Hash()) stats.ignored++ @@ -1644,7 +1638,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er // Write the block to the chain and get the status. substart = time.Now() - status, err := bc.writeBlockWithState(block, receipts, logs, statedb, false) + status, err := bc.writeBlockAndSetHead(block, receipts, logs, statedb, false) atomic.StoreUint32(&followupInterrupt, 1) if err != nil { return it.index, err @@ -1717,8 +1711,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er // switch over to the new chain if the TD exceeded the current chain. func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (int, error) { var ( - externTd *big.Int - current = bc.CurrentBlock() + externTd *big.Int + lastBlock = block + current = bc.CurrentBlock() ) // The first sidechain block error is already verified to be ErrPrunedAncestor. // Since we don't import them here, we expect ErrUnknownAncestor for the remaining @@ -1769,6 +1764,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()), "root", block.Root()) } + lastBlock = block } // At this point, we've written all sidechain blocks to database. Loop ended // either on some other error or all were processed. If there was some other @@ -1776,8 +1772,12 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i // // If the externTd was larger than our local TD, we now need to reimport the previous // blocks to regenerate the required state - localTd := bc.GetTd(current.Hash(), current.NumberU64()) - if localTd.Cmp(externTd) > 0 { + reorg, err := bc.forker.Reorg(current.Header(), lastBlock.Header()) + if err != nil { + return it.index, err + } + if !reorg { + localTd := bc.GetTd(current.Hash(), current.NumberU64()) log.Info("Sidechain written to disk", "start", it.first().NumberU64(), "end", it.previous().Number, "sidetd", externTd, "localtd", localTd) return it.index, err } @@ -1832,9 +1832,92 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i return 0, nil } +// recoverAncestors finds the closest ancestor with available state and re-execute +// all the ancestor blocks since that. +func (bc *BlockChain) recoverAncestors(block *types.Block) error { + // Gather all the sidechain hashes (full blocks may be memory heavy) + var ( + hashes []common.Hash + numbers []uint64 + parent = block + ) + for parent != nil && !bc.HasState(parent.Root()) { + hashes = append(hashes, parent.Hash()) + numbers = append(numbers, parent.NumberU64()) + parent = bc.GetBlock(parent.ParentHash(), parent.NumberU64()-1) + + // If the chain is terminating, stop iteration + if bc.insertStopped() { + log.Debug("Abort during blocks iteration") + return errInsertionInterrupted + } + } + if parent == nil { + return errors.New("missing parent") + } + // Import all the pruned blocks to make the state available + for i := len(hashes) - 1; i >= 0; i-- { + // If the chain is terminating, stop processing blocks + if bc.insertStopped() { + log.Debug("Abort during blocks processing") + return errInsertionInterrupted + } + var b *types.Block + if i == 0 { + b = block + } else { + b = bc.GetBlock(hashes[i], numbers[i]) + } + if err := bc.insertBlock(b); err != nil { + return err + } + } + return nil +} + +// collectLogs collects the logs that were generated or removed during +// the processing of the block that corresponds with the given hash. +// These logs are later announced as deleted or reborn. +func (bc *BlockChain) collectLogs(hash common.Hash, removed bool) []*types.Log { + number := bc.hc.GetBlockNumber(hash) + if number == nil { + return nil + } + receipts := rawdb.ReadReceipts(bc.db, hash, *number, bc.chainConfig) + + var logs []*types.Log + for _, receipt := range receipts { + for _, log := range receipt.Logs { + l := *log + if removed { + l.Removed = true + } + logs = append(logs, &l) + } + } + return logs +} + +// mergeLogs returns a merged log slice with specified sort order. +func (bc *BlockChain) mergeLogs(logs [][]*types.Log, reverse bool) []*types.Log { + var ret []*types.Log + if reverse { + for i := len(logs) - 1; i >= 0; i-- { + ret = append(ret, logs[i]...) + } + } else { + for i := 0; i < len(logs); i++ { + ret = append(ret, logs[i]...) + } + } + return ret +} + // reorg takes two blocks, an old chain and a new chain and will reconstruct the // blocks and inserts them to be part of the new canonical chain and accumulates // potential missing transactions and post an event about them. +// Note the new head block won't be processed here, callers need to handle it +// externally. func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { var ( newChain types.Blocks @@ -1846,49 +1929,6 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { deletedLogs [][]*types.Log rebirthLogs [][]*types.Log - - // collectLogs collects the logs that were generated or removed during - // the processing of the block that corresponds with the given hash. - // These logs are later announced as deleted or reborn - collectLogs = func(hash common.Hash, removed bool) { - number := bc.hc.GetBlockNumber(hash) - if number == nil { - return - } - receipts := rawdb.ReadReceipts(bc.db, hash, *number, bc.chainConfig) - - var logs []*types.Log - for _, receipt := range receipts { - for _, log := range receipt.Logs { - l := *log - if removed { - l.Removed = true - } - logs = append(logs, &l) - } - } - if len(logs) > 0 { - if removed { - deletedLogs = append(deletedLogs, logs) - } else { - rebirthLogs = append(rebirthLogs, logs) - } - } - } - // mergeLogs returns a merged log slice with specified sort order. - mergeLogs = func(logs [][]*types.Log, reverse bool) []*types.Log { - var ret []*types.Log - if reverse { - for i := len(logs) - 1; i >= 0; i-- { - ret = append(ret, logs[i]...) - } - } else { - for i := 0; i < len(logs); i++ { - ret = append(ret, logs[i]...) - } - } - return ret - } ) // Reduce the longer chain to the same number as the shorter one if oldBlock.NumberU64() > newBlock.NumberU64() { @@ -1896,7 +1936,12 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { for ; oldBlock != nil && oldBlock.NumberU64() != newBlock.NumberU64(); oldBlock = bc.GetBlock(oldBlock.ParentHash(), oldBlock.NumberU64()-1) { oldChain = append(oldChain, oldBlock) deletedTxs = append(deletedTxs, oldBlock.Transactions()...) - collectLogs(oldBlock.Hash(), true) + + // Collect deleted logs for notification + logs := bc.collectLogs(oldBlock.Hash(), true) + if len(logs) > 0 { + deletedLogs = append(deletedLogs, logs) + } } } else { // New chain is longer, stash all blocks away for subsequent insertion @@ -1921,8 +1966,12 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // Remove an old block as well as stash away a new block oldChain = append(oldChain, oldBlock) deletedTxs = append(deletedTxs, oldBlock.Transactions()...) - collectLogs(oldBlock.Hash(), true) + // Collect deleted logs for notification + logs := bc.collectLogs(oldBlock.Hash(), true) + if len(logs) > 0 { + deletedLogs = append(deletedLogs, logs) + } newChain = append(newChain, newBlock) // Step back with both chains @@ -1948,8 +1997,15 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { blockReorgAddMeter.Mark(int64(len(newChain))) blockReorgDropMeter.Mark(int64(len(oldChain))) blockReorgMeter.Mark(1) + } else if len(newChain) > 0 { + // Special case happens in the post merge stage that current head is + // the ancestor of new head while these two blocks are not consecutive + log.Info("Extend chain", "add", len(newChain), "number", newChain[0].NumberU64(), "hash", newChain[0].Hash()) + blockReorgAddMeter.Mark(int64(len(newChain))) } else { - log.Error("Impossible reorg, please file an issue", "oldnum", oldBlock.Number(), "oldhash", oldBlock.Hash(), "newnum", newBlock.Number(), "newhash", newBlock.Hash()) + // len(newChain) == 0 && len(oldChain) > 0 + // rewind the canonical chain to a lower point. + log.Error("Impossible reorg, please file an issue", "oldnum", oldBlock.Number(), "oldhash", oldBlock.Hash(), "oldblocks", len(oldChain), "newnum", newBlock.Number(), "newhash", newBlock.Hash(), "newblocks", len(newChain)) } // Insert the new chain(except the head block(reverse order)), // taking care of the proper incremental order. @@ -1958,8 +2014,10 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { bc.writeHeadBlock(newChain[i]) // Collect reborn logs due to chain reorg - collectLogs(newChain[i].Hash(), false) - + logs := bc.collectLogs(newChain[i].Hash(), false) + if len(logs) > 0 { + rebirthLogs = append(rebirthLogs, logs) + } // Collect the new added transactions. addedTxs = append(addedTxs, newChain[i].Transactions()...) } @@ -1986,10 +2044,10 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // ever happens if we're reorging empty blocks, which will only happen on idle // networks where performance is not an issue either way. if len(deletedLogs) > 0 { - bc.rmLogsFeed.Send(RemovedLogsEvent{mergeLogs(deletedLogs, true)}) + bc.rmLogsFeed.Send(RemovedLogsEvent{bc.mergeLogs(deletedLogs, true)}) } if len(rebirthLogs) > 0 { - bc.logsFeed.Send(mergeLogs(rebirthLogs, false)) + bc.logsFeed.Send(bc.mergeLogs(rebirthLogs, false)) } if len(oldChain) > 0 { for i := len(oldChain) - 1; i >= 0; i-- { @@ -1999,10 +2057,144 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { return nil } -// futureBlocksLoop processes the 'future block' queue. -func (bc *BlockChain) futureBlocksLoop() { - defer bc.wg.Done() +// InsertBlock accepts a single block and the given consensus engine as the +// parameters. It will firstly execute the block, run the necessary verfication +// upon it and then persist the block and the associate state into the database. +// The key difference between the InsertChain is it won't do the canonical chain +// udpating. It relays on the additional SetChainHead call to finalize the entire +// procedure. +func (bc *BlockChain) InsertBlock(block *types.Block) error { + if !bc.chainmu.TryLock() { + return errChainStopped + } + defer bc.chainmu.Unlock() + + return bc.insertBlock(block) +} + +// insertBlock is the inner version of InsertBlock without holding the lock. +func (bc *BlockChain) insertBlock(block *types.Block) error { + // If the chain is terminating, don't even bother starting up + if bc.insertStopped() { + return errInsertionInterrupted + } + // Start a parallel signature recovery (signer will fluke on fork transition, minimal perf loss) + senderCacher.recoverFromBlocks(types.MakeSigner(bc.chainConfig, block.Number()), []*types.Block{block}) + + // Verify the header with the given consensus engine. + if err := bc.engine.VerifyHeader(bc, block.Header(), true); err != nil { + return err + } + // Verify the block body then. + err := bc.validator.ValidateBody(block) + switch { + // The block is already known, return without any error + // but print a warning log. It's not expected behavior. + case errors.Is(err, ErrKnownBlock): + log.Warn("Received known block", "hash", block.Hash(), "number", block.NumberU64()) + return nil + + // The parent is pruned, try to recover the parent state + case errors.Is(err, consensus.ErrPrunedAncestor): + log.Debug("Pruned ancestor", "number", block.Number(), "hash", block.Hash()) + return bc.recoverAncestors(block) + + // Some other error occurred(include the future block), abort + case err != nil: + return err + } + // If the header is a banned one, straight out abort + if BadHashes[block.Hash()] { + bc.reportBlock(block, nil, ErrBlacklistedHash) + return ErrBlacklistedHash + } + // Retrieve the parent block and it's state to execute on top + start := time.Now() + parent := bc.GetHeader(block.ParentHash(), block.NumberU64()-1) + statedb, err := state.New(parent.Root, bc.stateCache, bc.snaps) + if err != nil { + return err + } + // Enable prefetching to pull in trie node paths while processing transactions + statedb.StartPrefetcher("chain") + defer statedb.StopPrefetcher() + + // Process block using the parent state as reference point + substart := time.Now() + receipts, logs, usedGas, err := bc.processor.Process(block, statedb, bc.vmConfig) + if err != nil { + bc.reportBlock(block, receipts, err) + return err + } + // Update the metrics touched during block processing + accountReadTimer.Update(statedb.AccountReads) // Account reads are complete, we can mark them + storageReadTimer.Update(statedb.StorageReads) // Storage reads are complete, we can mark them + accountUpdateTimer.Update(statedb.AccountUpdates) // Account updates are complete, we can mark them + storageUpdateTimer.Update(statedb.StorageUpdates) // Storage updates are complete, we can mark them + snapshotAccountReadTimer.Update(statedb.SnapshotAccountReads) // Account reads are complete, we can mark them + snapshotStorageReadTimer.Update(statedb.SnapshotStorageReads) // Storage reads are complete, we can mark them + triehash := statedb.AccountHashes + statedb.StorageHashes // Save to not double count in validation + trieproc := statedb.SnapshotAccountReads + statedb.AccountReads + statedb.AccountUpdates + trieproc += statedb.SnapshotStorageReads + statedb.StorageReads + statedb.StorageUpdates + blockExecutionTimer.Update(time.Since(substart) - trieproc - triehash) + + // Validate the state using the default validator + substart = time.Now() + if err := bc.validator.ValidateState(block, statedb, receipts, usedGas); err != nil { + bc.reportBlock(block, receipts, err) + return err + } + // Update the metrics touched during block validation + accountHashTimer.Update(statedb.AccountHashes) // Account hashes are complete, we can mark them + storageHashTimer.Update(statedb.StorageHashes) // Storage hashes are complete, we can mark them + blockValidationTimer.Update(time.Since(substart) - (statedb.AccountHashes + statedb.StorageHashes - triehash)) + + // Write the block to the chain and get the status. + substart = time.Now() + if err := bc.writeBlockWithState(block, receipts, logs, statedb); err != nil { + return err + } + // Update the metrics touched during block commit + accountCommitTimer.Update(statedb.AccountCommits) // Account commits are complete, we can mark them + storageCommitTimer.Update(statedb.StorageCommits) // Storage commits are complete, we can mark them + snapshotCommitTimer.Update(statedb.SnapshotCommits) // Snapshot commits are complete, we can mark them + + blockWriteTimer.Update(time.Since(substart) - statedb.AccountCommits - statedb.StorageCommits - statedb.SnapshotCommits) + blockInsertTimer.UpdateSince(start) + log.Info("Inserted block", "number", block.Number(), "hash", block.Hash(), "txs", len(block.Transactions()), "elasped", common.PrettyDuration(time.Since(start))) + return nil +} + +// SetChainHead rewinds the chain to set the new head block as the specified +// block. It's possible that after the reorg the relevant state of head +// is missing. It can be fixed by inserting a new block which triggers +// the re-execution. +func (bc *BlockChain) SetChainHead(newBlock *types.Block) error { + if !bc.chainmu.TryLock() { + return errChainStopped + } + defer bc.chainmu.Unlock() + + // Run the reorg if necessary and set the given block as new head. + if newBlock.ParentHash() != bc.CurrentBlock().Hash() { + if err := bc.reorg(bc.CurrentBlock(), newBlock); err != nil { + return err + } + } + bc.writeHeadBlock(newBlock) + + // Emit events + logs := bc.collectLogs(newBlock.Hash(), false) + bc.chainFeed.Send(ChainEvent{Block: newBlock, Hash: newBlock.Hash(), Logs: logs}) + if len(logs) > 0 { + bc.logsFeed.Send(logs) + } + bc.chainHeadFeed.Send(ChainHeadEvent{Block: newBlock}) + log.Info("Set the chain head", "number", newBlock.Number(), "hash", newBlock.Hash()) + return nil +} +func (bc *BlockChain) update() { futureTimer := time.NewTicker(5 * time.Second) defer futureTimer.Stop() for { @@ -2185,6 +2377,6 @@ func (bc *BlockChain) InsertHeaderChain(chain []*types.Header, checkFreq int) (i return 0, errChainStopped } defer bc.chainmu.Unlock() - _, err := bc.hc.InsertHeaderChain(chain, start) + _, err := bc.hc.InsertHeaderChain(chain, start, bc.forker) return 0, err } diff --git a/core/blockchain_repair_test.go b/core/blockchain_repair_test.go index f4f762078732..29a361534d6d 100644 --- a/core/blockchain_repair_test.go +++ b/core/blockchain_repair_test.go @@ -1783,7 +1783,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { config.SnapshotLimit = 256 config.SnapshotWait = true } - chain, err := NewBlockChain(db, config, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(db, config, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("Failed to create chain: %v", err) } @@ -1829,14 +1829,14 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { // Pull the plug on the database, simulating a hard crash db.Close() - // Start a new blockchain back up and see where the repait leads us + // Start a new blockchain back up and see where the repair leads us db, err = rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false) if err != nil { t.Fatalf("Failed to reopen persistent database: %v", err) } defer db.Close() - chain, err = NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) + chain, err = NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } diff --git a/core/blockchain_sethead_test.go b/core/blockchain_sethead_test.go index 27b6be6e1363..85a5b891f195 100644 --- a/core/blockchain_sethead_test.go +++ b/core/blockchain_sethead_test.go @@ -1982,7 +1982,7 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) { config.SnapshotLimit = 256 config.SnapshotWait = true } - chain, err := NewBlockChain(db, config, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(db, config, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("Failed to create chain: %v", err) } diff --git a/core/blockchain_snapshot_test.go b/core/blockchain_snapshot_test.go index a8044ecb4d81..4fb14349f65d 100644 --- a/core/blockchain_snapshot_test.go +++ b/core/blockchain_snapshot_test.go @@ -80,7 +80,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo // will happen during the block insertion. cacheConfig = defaultCacheConfig ) - chain, err := NewBlockChain(db, cacheConfig, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(db, cacheConfig, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("Failed to create chain: %v", err) } @@ -223,7 +223,7 @@ func (snaptest *snapshotTest) test(t *testing.T) { // Restart the chain normally chain.Stop() - newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -259,13 +259,13 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) { // the crash, we do restart twice here: one after the crash and one // after the normal stop. It's used to ensure the broken snapshot // can be detected all the time. - newchain, err := NewBlockChain(newdb, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + newchain, err := NewBlockChain(newdb, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } newchain.Stop() - newchain, err = NewBlockChain(newdb, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + newchain, err = NewBlockChain(newdb, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -301,7 +301,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) { TrieTimeLimit: 5 * time.Minute, SnapshotLimit: 0, } - newchain, err := NewBlockChain(snaptest.db, cacheConfig, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + newchain, err := NewBlockChain(snaptest.db, cacheConfig, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -309,7 +309,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) { newchain.Stop() // Restart the chain with enabling the snapshot - newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -337,7 +337,7 @@ func (snaptest *setHeadSnapshotTest) test(t *testing.T) { chain.SetHead(snaptest.setHead) chain.Stop() - newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -368,7 +368,7 @@ func (snaptest *restartCrashSnapshotTest) test(t *testing.T) { // and state committed. chain.Stop() - newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -385,7 +385,7 @@ func (snaptest *restartCrashSnapshotTest) test(t *testing.T) { // journal and latest state will be committed // Restart the chain after the crash - newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -420,7 +420,7 @@ func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) { TrieTimeLimit: 5 * time.Minute, SnapshotLimit: 0, } - newchain, err := NewBlockChain(snaptest.db, config, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + newchain, err := NewBlockChain(snaptest.db, config, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -436,13 +436,13 @@ func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) { SnapshotLimit: 256, SnapshotWait: false, // Don't wait rebuild } - newchain, err = NewBlockChain(snaptest.db, config, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + newchain, err = NewBlockChain(snaptest.db, config, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } // Simulate the blockchain crash. - newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 80d07eb30ab0..a0a65a634b96 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -28,7 +28,9 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" @@ -56,7 +58,7 @@ func newCanonical(engine consensus.Engine, n int, full bool) (ethdb.Database, *B ) // Initialize a fresh chain with only a genesis block - blockchain, _ := NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) // Create and inject the requested chain if n == 0 { return db, blockchain, nil @@ -210,6 +212,58 @@ func TestLastBlock(t *testing.T) { } } +// Test inserts the blocks/headers after the fork choice rule is changed. +// The chain is reorged to whatever specified. +func testInsertAfterMerge(t *testing.T, blockchain *BlockChain, i, n int, full bool) { + // Copy old chain up to #i into a new db + db, blockchain2, err := newCanonical(ethash.NewFaker(), i, full) + if err != nil { + t.Fatal("could not make new canonical in testFork", err) + } + defer blockchain2.Stop() + + // Assert the chains have the same header/block at #i + var hash1, hash2 common.Hash + if full { + hash1 = blockchain.GetBlockByNumber(uint64(i)).Hash() + hash2 = blockchain2.GetBlockByNumber(uint64(i)).Hash() + } else { + hash1 = blockchain.GetHeaderByNumber(uint64(i)).Hash() + hash2 = blockchain2.GetHeaderByNumber(uint64(i)).Hash() + } + if hash1 != hash2 { + t.Errorf("chain content mismatch at %d: have hash %v, want hash %v", i, hash2, hash1) + } + + // Trigger the transition explicitly + blockchain2.forker.MarkTransitioned() + + // Extend the newly created chain + if full { + blockChainB := makeBlockChain(blockchain2.CurrentBlock(), n, ethash.NewFaker(), db, forkSeed) + if _, err := blockchain2.InsertChain(blockChainB); err != nil { + t.Fatalf("failed to insert forking chain: %v", err) + } + if blockchain2.CurrentBlock().NumberU64() != blockChainB[len(blockChainB)-1].NumberU64() { + t.Fatalf("failed to reorg to the given chain") + } + if blockchain2.CurrentBlock().Hash() != blockChainB[len(blockChainB)-1].Hash() { + t.Fatalf("failed to reorg to the given chain") + } + } else { + headerChainB := makeHeaderChain(blockchain2.CurrentHeader(), n, ethash.NewFaker(), db, forkSeed) + if _, err := blockchain2.InsertHeaderChain(headerChainB, 1); err != nil { + t.Fatalf("failed to insert forking chain: %v", err) + } + if blockchain2.CurrentHeader().Number.Uint64() != headerChainB[len(headerChainB)-1].Number.Uint64() { + t.Fatalf("failed to reorg to the given chain") + } + if blockchain2.CurrentHeader().Hash() != headerChainB[len(headerChainB)-1].Hash() { + t.Fatalf("failed to reorg to the given chain") + } + } +} + // Tests that given a starting canonical chain of a given size, it can be extended // with various length chains. func TestExtendCanonicalHeaders(t *testing.T) { testExtendCanonical(t, false) } @@ -238,6 +292,25 @@ func testExtendCanonical(t *testing.T, full bool) { testFork(t, processor, length, 10, full, better) } +// Tests that given a starting canonical chain of a given size, it can be extended +// with various length chains. +func TestExtendCanonicalHeadersAfterMerge(t *testing.T) { testExtendCanonicalAfterMerge(t, false) } +func TestExtendCanonicalBlocksAfterMerge(t *testing.T) { testExtendCanonicalAfterMerge(t, true) } + +func testExtendCanonicalAfterMerge(t *testing.T, full bool) { + length := 5 + + // Make first chain starting from genesis + _, processor, err := newCanonical(ethash.NewFaker(), length, full) + if err != nil { + t.Fatalf("failed to make new canonical chain: %v", err) + } + defer processor.Stop() + + testInsertAfterMerge(t, processor, length, 1, full) + testInsertAfterMerge(t, processor, length, 10, full) +} + // Tests that given a starting canonical chain of a given size, creating shorter // forks do not take canonical ownership. func TestShorterForkHeaders(t *testing.T) { testShorterFork(t, false) } @@ -268,6 +341,29 @@ func testShorterFork(t *testing.T, full bool) { testFork(t, processor, 5, 4, full, worse) } +// Tests that given a starting canonical chain of a given size, creating shorter +// forks do not take canonical ownership. +func TestShorterForkHeadersAfterMerge(t *testing.T) { testShorterForkAfterMerge(t, false) } +func TestShorterForkBlocksAfterMerge(t *testing.T) { testShorterForkAfterMerge(t, true) } + +func testShorterForkAfterMerge(t *testing.T, full bool) { + length := 10 + + // Make first chain starting from genesis + _, processor, err := newCanonical(ethash.NewFaker(), length, full) + if err != nil { + t.Fatalf("failed to make new canonical chain: %v", err) + } + defer processor.Stop() + + testInsertAfterMerge(t, processor, 0, 3, full) + testInsertAfterMerge(t, processor, 0, 7, full) + testInsertAfterMerge(t, processor, 1, 1, full) + testInsertAfterMerge(t, processor, 1, 7, full) + testInsertAfterMerge(t, processor, 5, 3, full) + testInsertAfterMerge(t, processor, 5, 4, full) +} + // Tests that given a starting canonical chain of a given size, creating longer // forks do take canonical ownership. func TestLongerForkHeaders(t *testing.T) { testLongerFork(t, false) } @@ -283,19 +379,35 @@ func testLongerFork(t *testing.T, full bool) { } defer processor.Stop() - // Define the difficulty comparator - better := func(td1, td2 *big.Int) { - if td2.Cmp(td1) <= 0 { - t.Errorf("total difficulty mismatch: have %v, expected more than %v", td2, td1) - } + testInsertAfterMerge(t, processor, 0, 11, full) + testInsertAfterMerge(t, processor, 0, 15, full) + testInsertAfterMerge(t, processor, 1, 10, full) + testInsertAfterMerge(t, processor, 1, 12, full) + testInsertAfterMerge(t, processor, 5, 6, full) + testInsertAfterMerge(t, processor, 5, 8, full) +} + +// Tests that given a starting canonical chain of a given size, creating longer +// forks do take canonical ownership. +func TestLongerForkHeadersAfterMerge(t *testing.T) { testLongerForkAfterMerge(t, false) } +func TestLongerForkBlocksAfterMerge(t *testing.T) { testLongerForkAfterMerge(t, true) } + +func testLongerForkAfterMerge(t *testing.T, full bool) { + length := 10 + + // Make first chain starting from genesis + _, processor, err := newCanonical(ethash.NewFaker(), length, full) + if err != nil { + t.Fatalf("failed to make new canonical chain: %v", err) } - // Sum of numbers must be greater than `length` for this to be a longer fork - testFork(t, processor, 0, 11, full, better) - testFork(t, processor, 0, 15, full, better) - testFork(t, processor, 1, 10, full, better) - testFork(t, processor, 1, 12, full, better) - testFork(t, processor, 5, 6, full, better) - testFork(t, processor, 5, 8, full, better) + defer processor.Stop() + + testInsertAfterMerge(t, processor, 0, 11, full) + testInsertAfterMerge(t, processor, 0, 15, full) + testInsertAfterMerge(t, processor, 1, 10, full) + testInsertAfterMerge(t, processor, 1, 12, full) + testInsertAfterMerge(t, processor, 5, 6, full) + testInsertAfterMerge(t, processor, 5, 8, full) } // Tests that given a starting canonical chain of a given size, creating equal @@ -328,6 +440,29 @@ func testEqualFork(t *testing.T, full bool) { testFork(t, processor, 9, 1, full, equal) } +// Tests that given a starting canonical chain of a given size, creating equal +// forks do take canonical ownership. +func TestEqualForkHeadersAfterMerge(t *testing.T) { testEqualForkAfterMerge(t, false) } +func TestEqualForkBlocksAfterMerge(t *testing.T) { testEqualForkAfterMerge(t, true) } + +func testEqualForkAfterMerge(t *testing.T, full bool) { + length := 10 + + // Make first chain starting from genesis + _, processor, err := newCanonical(ethash.NewFaker(), length, full) + if err != nil { + t.Fatalf("failed to make new canonical chain: %v", err) + } + defer processor.Stop() + + testInsertAfterMerge(t, processor, 0, 10, full) + testInsertAfterMerge(t, processor, 1, 9, full) + testInsertAfterMerge(t, processor, 2, 8, full) + testInsertAfterMerge(t, processor, 5, 5, full) + testInsertAfterMerge(t, processor, 6, 4, full) + testInsertAfterMerge(t, processor, 9, 1, full) +} + // Tests that chains missing links do not get accepted by the processor. func TestBrokenHeaderChain(t *testing.T) { testBrokenChain(t, false) } func TestBrokenBlockChain(t *testing.T) { testBrokenChain(t, true) } @@ -522,7 +657,7 @@ func testReorgBadHashes(t *testing.T, full bool) { blockchain.Stop() // Create a new BlockChain and check that it rolled back the state. - ncm, err := NewBlockChain(blockchain.db, nil, blockchain.chainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) + ncm, err := NewBlockChain(blockchain.db, nil, blockchain.chainConfig, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("failed to create new chain manager: %v", err) } @@ -635,7 +770,7 @@ func TestFastVsFullChains(t *testing.T) { // Import the chain as an archive node for the comparison baseline archiveDb := rawdb.NewMemoryDatabase() gspec.MustCommit(archiveDb) - archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) defer archive.Stop() if n, err := archive.InsertChain(blocks); err != nil { @@ -644,7 +779,7 @@ func TestFastVsFullChains(t *testing.T) { // Fast import the chain as a non-archive node to test fastDb := rawdb.NewMemoryDatabase() gspec.MustCommit(fastDb) - fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) defer fast.Stop() headers := make([]*types.Header, len(blocks)) @@ -668,7 +803,7 @@ func TestFastVsFullChains(t *testing.T) { t.Fatalf("failed to create temp freezer db: %v", err) } gspec.MustCommit(ancientDb) - ancient, _ := NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + ancient, _ := NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) defer ancient.Stop() if n, err := ancient.InsertHeaderChain(headers, 1); err != nil { @@ -790,7 +925,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { archiveCaching := *defaultCacheConfig archiveCaching.TrieDirtyDisabled = true - archive, _ := NewBlockChain(archiveDb, &archiveCaching, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + archive, _ := NewBlockChain(archiveDb, &archiveCaching, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if n, err := archive.InsertChain(blocks); err != nil { t.Fatalf("failed to process block %d: %v", n, err) } @@ -803,7 +938,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Import the chain as a non-archive node and ensure all pointers are updated fastDb, delfn := makeDb() defer delfn() - fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) defer fast.Stop() headers := make([]*types.Header, len(blocks)) @@ -823,7 +958,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Import the chain as a ancient-first node and ensure all pointers are updated ancientDb, delfn := makeDb() defer delfn() - ancient, _ := NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + ancient, _ := NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) defer ancient.Stop() if n, err := ancient.InsertHeaderChain(headers, 1); err != nil { @@ -842,7 +977,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Import the chain as a light node and ensure all pointers are updated lightDb, delfn := makeDb() defer delfn() - light, _ := NewBlockChain(lightDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + light, _ := NewBlockChain(lightDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if n, err := light.InsertHeaderChain(headers, 1); err != nil { t.Fatalf("failed to insert header %d: %v", n, err) } @@ -911,7 +1046,7 @@ func TestChainTxReorgs(t *testing.T) { } }) // Import the chain. This runs all block validation rules. - blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if i, err := blockchain.InsertChain(chain); err != nil { t.Fatalf("failed to insert original chain[%d]: %v", i, err) } @@ -981,7 +1116,7 @@ func TestLogReorgs(t *testing.T) { signer = types.LatestSigner(gspec.Config) ) - blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) defer blockchain.Stop() rmLogsCh := make(chan RemovedLogsEvent) @@ -1034,7 +1169,7 @@ func TestLogRebirth(t *testing.T) { genesis = gspec.MustCommit(db) signer = types.LatestSigner(gspec.Config) engine = ethash.NewFaker() - blockchain, _ = NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}, nil, nil) + blockchain, _ = NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) ) defer blockchain.Stop() @@ -1097,7 +1232,7 @@ func TestSideLogRebirth(t *testing.T) { gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} genesis = gspec.MustCommit(db) signer = types.LatestSigner(gspec.Config) - blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) ) defer blockchain.Stop() @@ -1172,7 +1307,7 @@ func TestReorgSideEvent(t *testing.T) { signer = types.LatestSigner(gspec.Config) ) - blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) defer blockchain.Stop() chain, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 3, func(i int, gen *BlockGen) {}) @@ -1304,7 +1439,7 @@ func TestEIP155Transition(t *testing.T) { genesis = gspec.MustCommit(db) ) - blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) defer blockchain.Stop() blocks, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 4, func(i int, block *BlockGen) { @@ -1412,7 +1547,7 @@ func TestEIP161AccountRemoval(t *testing.T) { } genesis = gspec.MustCommit(db) ) - blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) defer blockchain.Stop() blocks, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 3, func(i int, block *BlockGen) { @@ -1487,7 +1622,7 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { diskdb := rawdb.NewMemoryDatabase() (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -1531,7 +1666,7 @@ func TestTrieForkGC(t *testing.T) { diskdb := rawdb.NewMemoryDatabase() (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -1570,7 +1705,7 @@ func TestLargeReorgTrieGC(t *testing.T) { diskdb := rawdb.NewMemoryDatabase() (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -1631,7 +1766,7 @@ func TestBlockchainRecovery(t *testing.T) { t.Fatalf("failed to create temp freezer db: %v", err) } gspec.MustCommit(ancientDb) - ancient, _ := NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + ancient, _ := NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) headers := make([]*types.Header, len(blocks)) for i, block := range blocks { @@ -1651,7 +1786,7 @@ func TestBlockchainRecovery(t *testing.T) { rawdb.WriteHeadFastBlockHash(ancientDb, midBlock.Hash()) // Reopen broken blockchain again - ancient, _ = NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + ancient, _ = NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) defer ancient.Stop() if num := ancient.CurrentBlock().NumberU64(); num != 0 { t.Errorf("head block mismatch: have #%v, want #%v", num, 0) @@ -1703,7 +1838,7 @@ func TestInsertReceiptChainRollback(t *testing.T) { } gspec := Genesis{Config: params.AllEthashProtocolChanges} gspec.MustCommit(ancientDb) - ancientChain, _ := NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + ancientChain, _ := NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) defer ancientChain.Stop() // Import the canonical header chain. @@ -1764,7 +1899,7 @@ func TestLowDiffLongChain(t *testing.T) { diskdb := rawdb.NewMemoryDatabase() (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -1800,21 +1935,51 @@ func TestLowDiffLongChain(t *testing.T) { // - C is canon chain, containing blocks [G..Cn..Cm] // - A common ancestor is placed at prune-point + blocksBetweenCommonAncestorAndPruneblock // - The sidechain S is prepended with numCanonBlocksInSidechain blocks from the canon chain -func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommonAncestorAndPruneblock int) { - +// +// The mergePoint can be these values: +// -1: the transition won't happen +// 0: the transition happens since genesis +// 1: the transition happens after some chain segments +func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommonAncestorAndPruneblock int, mergePoint int) { // Generate a canonical chain to act as the main dataset - engine := ethash.NewFaker() - db := rawdb.NewMemoryDatabase() - genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) + var ( + genEngine = beacon.New(ethash.NewFaker(), false) + runEngine = beacon.New(ethash.NewFaker(), false) + db = rawdb.NewMemoryDatabase() + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + addr = crypto.PubkeyToAddress(key.PublicKey) + nonce = uint64(0) + + gspec = &Genesis{ + Config: params.TestChainConfig, + Alloc: GenesisAlloc{addr: {Balance: big.NewInt(math.MaxInt64)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + signer = types.LatestSigner(gspec.Config) + genesis, _ = gspec.Commit(db) + ) // Generate and import the canonical chain - blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*TriesInMemory, nil) diskdb := rawdb.NewMemoryDatabase() - (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) + gspec.MustCommit(diskdb) + chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, runEngine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } + // Activate the transition since genesis if required + if mergePoint == 0 { + genEngine.MarkTransitioned() + runEngine.MarkTransitioned() + chain.forker.MarkTransitioned() + } + blocks, _ := GenerateChain(params.TestChainConfig, genesis, genEngine, db, 2*TriesInMemory, func(i int, gen *BlockGen) { + tx, err := types.SignTx(types.NewTransaction(nonce, common.HexToAddress("deadbeef"), big.NewInt(100), 21000, big.NewInt(int64(i+1)*params.GWei), nil), signer, key) + if err != nil { + t.Fatalf("failed to create tx: %v", err) + } + gen.AddTx(tx) + nonce++ + }) if n, err := chain.InsertChain(blocks); err != nil { t.Fatalf("block %d: failed to insert into chain: %v", n, err) } @@ -1831,6 +1996,14 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon if !chain.HasBlockAndState(firstNonPrunedBlock.Hash(), firstNonPrunedBlock.NumberU64()) { t.Errorf("Block %d pruned", firstNonPrunedBlock.NumberU64()) } + + // Activate the transition in the middle of the chain + if mergePoint == 1 { + genEngine.MarkTransitioned() + runEngine.MarkTransitioned() + chain.forker.MarkTransitioned() + } + // Generate the sidechain // First block should be a known block, block after should be a pruned block. So // canon(pruned), side, side... @@ -1838,7 +2011,7 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon // Generate fork chain, make it longer than canon parentIndex := lastPrunedIndex + blocksBetweenCommonAncestorAndPruneblock parent := blocks[parentIndex] - fork, _ := GenerateChain(params.TestChainConfig, parent, engine, db, 2*TriesInMemory, func(i int, b *BlockGen) { + fork, _ := GenerateChain(params.TestChainConfig, parent, genEngine, db, 2*TriesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) }) // Prepend the parent(s) @@ -1847,9 +2020,9 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon sidechain = append(sidechain, blocks[parentIndex+1-i]) } sidechain = append(sidechain, fork...) - _, err = chain.InsertChain(sidechain) + n, err := chain.InsertChain(sidechain) if err != nil { - t.Errorf("Got error, %v", err) + t.Errorf("Got error, %v number %d - %d", err, sidechain[n].NumberU64(), n) } head := chain.CurrentBlock() if got := fork[len(fork)-1].Hash(); got != head.Hash() { @@ -1870,11 +2043,28 @@ func TestPrunedImportSide(t *testing.T) { //glogger := log.NewGlogHandler(log.StreamHandler(os.Stdout, log.TerminalFormat(false))) //glogger.Verbosity(3) //log.Root().SetHandler(log.Handler(glogger)) - testSideImport(t, 3, 3) - testSideImport(t, 3, -3) - testSideImport(t, 10, 0) - testSideImport(t, 1, 10) - testSideImport(t, 1, -10) + testSideImport(t, 3, 3, -1) + testSideImport(t, 3, -3, -1) + testSideImport(t, 10, 0, -1) + testSideImport(t, 1, 10, -1) + testSideImport(t, 1, -10, -1) +} + +func TestPrunedImportSideWithMerging(t *testing.T) { + //glogger := log.NewGlogHandler(log.StreamHandler(os.Stdout, log.TerminalFormat(false))) + //glogger.Verbosity(3) + //log.Root().SetHandler(log.Handler(glogger)) + testSideImport(t, 3, 3, 0) + testSideImport(t, 3, -3, 0) + testSideImport(t, 10, 0, 0) + testSideImport(t, 1, 10, 0) + testSideImport(t, 1, -10, 0) + + testSideImport(t, 3, 3, 1) + testSideImport(t, 3, -3, 1) + testSideImport(t, 10, 0, 1) + testSideImport(t, 1, 10, 1) + testSideImport(t, 1, -10, 1) } func TestInsertKnownHeaders(t *testing.T) { testInsertKnownChainData(t, "headers") } @@ -1908,7 +2098,7 @@ func testInsertKnownChainData(t *testing.T, typ string) { (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(chaindb) defer os.RemoveAll(dir) - chain, err := NewBlockChain(chaindb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(chaindb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2002,6 +2192,177 @@ func testInsertKnownChainData(t *testing.T, typ string) { asserter(t, blocks2[len(blocks2)-1]) } +func TestInsertKnownHeadersWithMerging(t *testing.T) { + testInsertKnownChainDataWithMerging(t, "headers", 0) +} +func TestInsertKnownReceiptChainWithMerging(t *testing.T) { + testInsertKnownChainDataWithMerging(t, "receipts", 0) +} +func TestInsertKnownBlocksWithMerging(t *testing.T) { + testInsertKnownChainDataWithMerging(t, "blocks", 0) +} +func TestInsertKnownHeadersAfterMerging(t *testing.T) { + testInsertKnownChainDataWithMerging(t, "headers", 1) +} +func TestInsertKnownReceiptChainAfterMerging(t *testing.T) { + testInsertKnownChainDataWithMerging(t, "receipts", 1) +} +func TestInsertKnownBlocksAfterMerging(t *testing.T) { + testInsertKnownChainDataWithMerging(t, "blocks", 1) +} + +// mergeHeight can be assigned in these values: +// 0: means the merging is applied since genesis +// 1: means the merging is applied after the first segment +func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight int) { + var ( + db = rawdb.NewMemoryDatabase() + genesis = (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) + runEngine = beacon.New(ethash.NewFaker(), false) + genEngine = beacon.New(ethash.NewFaker(), false) + ) + applyMerge := func(engine *beacon.Beacon, forker *ForkChoice) { + if engine != nil { + engine.MarkTransitioned() + } + if forker != nil { + forker.MarkTransitioned() + } + } + + // Apply merging since genesis + if mergeHeight == 0 { + applyMerge(genEngine, nil) + } + blocks, receipts := GenerateChain(params.TestChainConfig, genesis, genEngine, db, 32, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) + + // Apply merging after the first segment + if mergeHeight == 1 { + applyMerge(genEngine, nil) + } + // Longer chain and shorter chain + blocks2, receipts2 := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], genEngine, db, 65, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) + blocks3, receipts3 := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], genEngine, db, 64, func(i int, b *BlockGen) { + b.SetCoinbase(common.Address{1}) + b.OffsetTime(-9) // Time shifted, difficulty shouldn't be changed + }) + + // Import the shared chain and the original canonical one + dir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to create temp freezer dir: %v", err) + } + defer os.Remove(dir) + chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), dir, "", false) + if err != nil { + t.Fatalf("failed to create temp freezer db: %v", err) + } + (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(chaindb) + defer os.RemoveAll(dir) + + chain, err := NewBlockChain(chaindb, nil, params.TestChainConfig, runEngine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + if err != nil { + t.Fatalf("failed to create tester chain: %v", err) + } + var ( + inserter func(blocks []*types.Block, receipts []types.Receipts) error + asserter func(t *testing.T, block *types.Block) + ) + if typ == "headers" { + inserter = func(blocks []*types.Block, receipts []types.Receipts) error { + headers := make([]*types.Header, 0, len(blocks)) + for _, block := range blocks { + headers = append(headers, block.Header()) + } + _, err := chain.InsertHeaderChain(headers, 1) + return err + } + asserter = func(t *testing.T, block *types.Block) { + if chain.CurrentHeader().Hash() != block.Hash() { + t.Fatalf("current head header mismatch, have %v, want %v", chain.CurrentHeader().Hash().Hex(), block.Hash().Hex()) + } + } + } else if typ == "receipts" { + inserter = func(blocks []*types.Block, receipts []types.Receipts) error { + headers := make([]*types.Header, 0, len(blocks)) + for _, block := range blocks { + headers = append(headers, block.Header()) + } + _, err := chain.InsertHeaderChain(headers, 1) + if err != nil { + return err + } + _, err = chain.InsertReceiptChain(blocks, receipts, 0) + return err + } + asserter = func(t *testing.T, block *types.Block) { + if chain.CurrentFastBlock().Hash() != block.Hash() { + t.Fatalf("current head fast block mismatch, have %v, want %v", chain.CurrentFastBlock().Hash().Hex(), block.Hash().Hex()) + } + } + } else { + inserter = func(blocks []*types.Block, receipts []types.Receipts) error { + _, err := chain.InsertChain(blocks) + return err + } + asserter = func(t *testing.T, block *types.Block) { + if chain.CurrentBlock().Hash() != block.Hash() { + t.Fatalf("current head block mismatch, have %v, want %v", chain.CurrentBlock().Hash().Hex(), block.Hash().Hex()) + } + } + } + + // Apply merging since genesis if required + if mergeHeight == 0 { + applyMerge(runEngine, chain.forker) + } + if err := inserter(blocks, receipts); err != nil { + t.Fatalf("failed to insert chain data: %v", err) + } + + // Reimport the chain data again. All the imported + // chain data are regarded "known" data. + if err := inserter(blocks, receipts); err != nil { + t.Fatalf("failed to insert chain data: %v", err) + } + asserter(t, blocks[len(blocks)-1]) + + // Import a long canonical chain with some known data as prefix. + rollback := blocks[len(blocks)/2].NumberU64() + chain.SetHead(rollback - 1) + if err := inserter(blocks, receipts); err != nil { + t.Fatalf("failed to insert chain data: %v", err) + } + asserter(t, blocks[len(blocks)-1]) + + // Apply merging after the first segment + if mergeHeight == 1 { + applyMerge(runEngine, chain.forker) + } + + // Import a longer chain with some known data as prefix. + if err := inserter(append(blocks, blocks2...), append(receipts, receipts2...)); err != nil { + t.Fatalf("failed to insert chain data: %v", err) + } + asserter(t, blocks2[len(blocks2)-1]) + + // Import a shorter chain with some known data as prefix. + // The reorg is expected since the fork choice rule is + // already changed. + if err := inserter(append(blocks, blocks3...), append(receipts, receipts3...)); err != nil { + t.Fatalf("failed to insert chain data: %v", err) + } + // The head shouldn't change. + asserter(t, blocks3[len(blocks3)-1]) + + // Reimport the longer chain again, the reorg is still expected + chain.SetHead(rollback - 1) + if err := inserter(append(blocks, blocks2...), append(receipts, receipts2...)); err != nil { + t.Fatalf("failed to insert chain data: %v", err) + } + asserter(t, blocks2[len(blocks2)-1]) +} + // getLongAndShortChains returns two chains: A is longer, B is heavier. func getLongAndShortChains() (bc *BlockChain, longChain []*types.Block, heavyChain []*types.Block, err error) { // Generate a canonical chain to act as the main dataset @@ -2017,7 +2378,7 @@ func getLongAndShortChains() (bc *BlockChain, longChain []*types.Block, heavyCha diskdb := rawdb.NewMemoryDatabase() (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { return nil, nil, nil, fmt.Errorf("failed to create tester chain: %v", err) } @@ -2210,7 +2571,7 @@ func TestTransactionIndices(t *testing.T) { // Import all blocks into ancient db l := uint64(0) - chain, err := NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l) + chain, err := NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2235,7 +2596,7 @@ func TestTransactionIndices(t *testing.T) { t.Fatalf("failed to create temp freezer db: %v", err) } gspec.MustCommit(ancientDb) - chain, err = NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l) + chain, err = NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2259,7 +2620,7 @@ func TestTransactionIndices(t *testing.T) { limit = []uint64{0, 64 /* drop stale */, 32 /* shorten history */, 64 /* extend history */, 0 /* restore all */} tails := []uint64{0, 67 /* 130 - 64 + 1 */, 100 /* 131 - 32 + 1 */, 69 /* 132 - 64 + 1 */, 0} for i, l := range limit { - chain, err = NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l) + chain, err = NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2337,7 +2698,7 @@ func TestSkipStaleTxIndicesInFastSync(t *testing.T) { // Import all blocks into ancient db, only HEAD-32 indices are kept. l := uint64(32) - chain, err := NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l) + chain, err := NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2401,7 +2762,7 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in diskdb := rawdb.NewMemoryDatabase() gspec.MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { b.Fatalf("failed to create tester chain: %v", err) } @@ -2482,8 +2843,9 @@ func TestSideImportPrunedBlocks(t *testing.T) { // Generate and import the canonical chain blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*TriesInMemory, nil) diskdb := rawdb.NewMemoryDatabase() + (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2577,7 +2939,7 @@ func TestDeleteCreateRevert(t *testing.T) { diskdb := rawdb.NewMemoryDatabase() gspec.MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2691,7 +3053,7 @@ func TestDeleteRecreateSlots(t *testing.T) { chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{ Debug: true, Tracer: vm.NewJSONLogger(nil, os.Stdout), - }, nil, nil) + }, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2771,7 +3133,7 @@ func TestDeleteRecreateAccount(t *testing.T) { chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{ Debug: true, Tracer: vm.NewJSONLogger(nil, os.Stdout), - }, nil, nil) + }, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2944,7 +3306,7 @@ func TestDeleteRecreateSlotsAcrossManyBlocks(t *testing.T) { chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{ //Debug: true, //Tracer: vm.NewJSONLogger(nil, os.Stdout), - }, nil, nil) + }, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3078,7 +3440,7 @@ func TestInitThenFailCreateContract(t *testing.T) { chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{ //Debug: true, //Tracer: vm.NewJSONLogger(nil, os.Stdout), - }, nil, nil) + }, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3165,7 +3527,7 @@ func TestEIP2718Transition(t *testing.T) { diskdb := rawdb.NewMemoryDatabase() gspec.MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, gspec.Config, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(diskdb, nil, gspec.Config, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3260,7 +3622,7 @@ func TestEIP1559Transition(t *testing.T) { diskdb := rawdb.NewMemoryDatabase() gspec.MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, gspec.Config, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(diskdb, nil, gspec.Config, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } diff --git a/core/chain_makers.go b/core/chain_makers.go index b113c0d1be9d..5316f0d84e01 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -205,6 +205,12 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine} b.header = makeHeader(chainreader, parent, statedb, b.engine) + // Set the difficulty for clique block. The chain maker doesn't have access + // to a chain, so the difficulty will be lets unset (nil). Set it here to the + // correct value. + if b.header.Difficulty == nil { + b.header.Difficulty = big.NewInt(2) + } // Mutate the state and block according to any hard-fork specs if daoBlock := config.DAOForkBlock; daoBlock != nil { limit := new(big.Int).Add(daoBlock, params.DAOForkExtraRange) diff --git a/core/chain_makers_test.go b/core/chain_makers_test.go index 85a029f7c757..79c8314d61a8 100644 --- a/core/chain_makers_test.go +++ b/core/chain_makers_test.go @@ -79,7 +79,7 @@ func ExampleGenerateChain() { }) // Import the chain. This runs all block validation rules. - blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) defer blockchain.Stop() if i, err := blockchain.InsertChain(chain); err != nil { diff --git a/core/dao_test.go b/core/dao_test.go index c9c765a3832a..5b5cf18579b5 100644 --- a/core/dao_test.go +++ b/core/dao_test.go @@ -45,7 +45,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { proConf.DAOForkBlock = forkBlock proConf.DAOForkSupport = true - proBc, _ := NewBlockChain(proDb, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil, nil) + proBc, _ := NewBlockChain(proDb, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) defer proBc.Stop() conDb := rawdb.NewMemoryDatabase() @@ -55,7 +55,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { conConf.DAOForkBlock = forkBlock conConf.DAOForkSupport = false - conBc, _ := NewBlockChain(conDb, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil, nil) + conBc, _ := NewBlockChain(conDb, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) defer conBc.Stop() if _, err := proBc.InsertChain(prefix); err != nil { @@ -69,7 +69,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Create a pro-fork block, and try to feed into the no-fork chain db = rawdb.NewMemoryDatabase() gspec.MustCommit(db) - bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil, nil) + bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) defer bc.Stop() blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64())) @@ -94,7 +94,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Create a no-fork block, and try to feed into the pro-fork chain db = rawdb.NewMemoryDatabase() gspec.MustCommit(db) - bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil, nil) + bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) defer bc.Stop() blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64())) @@ -120,7 +120,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Verify that contra-forkers accept pro-fork extra-datas after forking finishes db = rawdb.NewMemoryDatabase() gspec.MustCommit(db) - bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil, nil) + bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) defer bc.Stop() blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64())) @@ -140,7 +140,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Verify that pro-forkers accept contra-fork extra-datas after forking finishes db = rawdb.NewMemoryDatabase() gspec.MustCommit(db) - bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil, nil) + bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) defer bc.Stop() blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64())) diff --git a/core/forkchoice.go b/core/forkchoice.go new file mode 100644 index 000000000000..9bc1f79facef --- /dev/null +++ b/core/forkchoice.go @@ -0,0 +1,123 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package core + +import ( + crand "crypto/rand" + "errors" + "math/big" + mrand "math/rand" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" +) + +// ChainReader defines a small collection of methods needed to access the local +// blockchain during header verification. It's implemented by both blockchain +// and lightchain. +type ChainReader interface { + // GetTd returns the total difficulty of a local block. + GetTd(common.Hash, uint64) *big.Int +} + +// ForkChoice is the fork chooser based on the highest total difficulty of the +// chain(the fork choice used in the eth1) and the external fork choice (the fork +// choice used in the eth2). This main goal of this ForkChoice is not only for +// offering fork choice during the eth1/2 merge phase, but also keep the compatibility +// for all other proof-of-work networks. +type ForkChoice struct { + chain ChainReader + rand *mrand.Rand + + // transitioned is the flag whether the chain has started(or finished) + // the transition. It's triggered by receiving the first "NewHead" message + // from the external consensus engine. + transitioned bool + lock sync.RWMutex + + // preserve is a helper function used in td fork choice. + // Miners will prefer to choose the local mined block if the + // local td is equal to the extern one. It can nil for light + // client + preserve func(header *types.Header) bool +} + +func NewForkChoice(chainReader ChainReader, transitioned bool, preserve func(header *types.Header) bool) *ForkChoice { + // Seed a fast but crypto originating random generator + seed, err := crand.Int(crand.Reader, big.NewInt(math.MaxInt64)) + if err != nil { + log.Crit("Failed to initialize random seed", "err", err) + } + return &ForkChoice{ + chain: chainReader, + rand: mrand.New(mrand.NewSource(seed.Int64())), + transitioned: transitioned, + preserve: preserve, + } +} + +// Reorg returns the result whether the reorg should be applied +// based on the given external header and local canonical chain. +// In the td mode, the new head is chosen if the corresponding +// total difficulty is higher. In the extern mode, the trusted +// header is always selected as the head. +func (f *ForkChoice) Reorg(current *types.Header, header *types.Header) (bool, error) { + f.lock.RLock() + defer f.lock.RUnlock() + + // Accept the new header as the chain head if the transition + // is already triggered. We assume all the headers after the + // transition come from the trusted consensus layer. + if f.transitioned { + return true, nil + } + var ( + localTD = f.chain.GetTd(current.Hash(), current.Number.Uint64()) + externTd = f.chain.GetTd(header.Hash(), header.Number.Uint64()) + ) + if localTD == nil || externTd == nil { + return false, errors.New("missing td") + } + // If the total difficulty is higher than our known, add it to the canonical chain + // Second clause in the if statement reduces the vulnerability to selfish mining. + // Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf + reorg := externTd.Cmp(localTD) > 0 + if !reorg && externTd.Cmp(localTD) == 0 { + number, headNumber := header.Number.Uint64(), current.Number.Uint64() + if number < headNumber { + reorg = true + } else if number == headNumber { + var currentPreserve, externPreserve bool + if f.preserve != nil { + currentPreserve, externPreserve = f.preserve(current), f.preserve(header) + } + reorg = !currentPreserve && (externPreserve || f.rand.Float64() < 0.5) + } + } + return reorg, nil +} + +// MarkTransitioned marks the transition has started. +func (f *ForkChoice) MarkTransitioned() { + f.lock.Lock() + defer f.lock.Unlock() + + f.transitioned = true +} diff --git a/core/genesis_test.go b/core/genesis_test.go index 056c1c195f6b..aa02279dc3f7 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -136,7 +136,7 @@ func TestSetupGenesis(t *testing.T) { // Advance to block #4, past the homestead transition block of customg. genesis := oldcustomg.MustCommit(db) - bc, _ := NewBlockChain(db, nil, oldcustomg.Config, ethash.NewFullFaker(), vm.Config{}, nil, nil) + bc, _ := NewBlockChain(db, nil, oldcustomg.Config, ethash.NewFullFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) defer bc.Stop() blocks, _ := GenerateChain(oldcustomg.Config, genesis, ethash.NewFaker(), db, 4, nil) diff --git a/core/headerchain.go b/core/headerchain.go index 9f2b708d0a31..6975c9b2a032 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -49,15 +49,14 @@ const ( // HeaderChain is responsible for maintaining the header chain including the // header query and updating. // -// The components maintained by headerchain includes: (1) total difficult +// The components maintained by headerchain includes: (1) total difficulty // (2) header (3) block hash -> number mapping (4) canonical number -> hash mapping // and (5) head header flag. // // It is not thread safe either, the encapsulating chain structures should do // the necessary mutex locking/unlocking. type HeaderChain struct { - config *params.ChainConfig - + config *params.ChainConfig chainDb ethdb.Database genesisHeader *types.Header @@ -86,7 +85,6 @@ func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine c if err != nil { return nil, err } - hc := &HeaderChain{ config: config, chainDb: chainDb, @@ -97,12 +95,10 @@ func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine c rand: mrand.New(mrand.NewSource(seed.Int64())), engine: engine, } - hc.genesisHeader = hc.GetHeaderByNumber(0) if hc.genesisHeader == nil { return nil, ErrNoGenesis } - hc.currentHeader.Store(hc.genesisHeader) if head := rawdb.ReadHeadBlockHash(chainDb); head != (common.Hash{}) { if chead := hc.GetHeaderByHash(head); chead != nil { @@ -111,7 +107,6 @@ func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine c } hc.currentHeaderHash = hc.CurrentHeader().Hash() headHeaderGauge.Update(hc.CurrentHeader().Number.Int64()) - return hc, nil } @@ -137,35 +132,87 @@ type headerWriteResult struct { lastHeader *types.Header } -// WriteHeaders writes a chain of headers into the local chain, given that the parents -// are already known. If the total difficulty of the newly inserted chain becomes -// greater than the current known TD, the canonical chain is reorged. -// -// Note: This method is not concurrent-safe with inserting blocks simultaneously -// into the chain, as side effects caused by reorganisations cannot be emulated -// without the real blocks. Hence, writing headers directly should only be done -// in two scenarios: pure-header mode of operation (light clients), or properly -// separated header/block phases (non-archive clients). -func (hc *HeaderChain) writeHeaders(headers []*types.Header) (result *headerWriteResult, err error) { +// Reorg reorgs the local canonical chain into the specified chain. The reorg +// can be classified into two cases: (a) extend the local chain (b) switch the +// head to the given header. +func (hc *HeaderChain) Reorg(headers []*types.Header) error { + // Short circuit if nothing to reorg. + if len(headers) == 0 { + return nil + } + // If the parent of the (first) block is already the canon header, + // we don't have to go backwards to delete canon blocks, but simply + // pile them onto the existing chain. Otherwise, do the necessary + // reorgs. + var ( + first = headers[0] + last = headers[len(headers)-1] + batch = hc.chainDb.NewBatch() + ) + if first.ParentHash != hc.currentHeaderHash { + // Delete any canonical number assignments above the new head + for i := last.Number.Uint64() + 1; ; i++ { + hash := rawdb.ReadCanonicalHash(hc.chainDb, i) + if hash == (common.Hash{}) { + break + } + rawdb.DeleteCanonicalHash(batch, i) + } + // Overwrite any stale canonical number assignments, going + // backwards from the first header in this import until the + // cross link between two chains. + var ( + header = first + headNumber = header.Number.Uint64() + headHash = header.Hash() + ) + for rawdb.ReadCanonicalHash(hc.chainDb, headNumber) != headHash { + rawdb.WriteCanonicalHash(batch, headHash, headNumber) + if headNumber == 0 { + break // It shouldn't be reached + } + headHash, headNumber = header.ParentHash, header.Number.Uint64()-1 + header = hc.GetHeader(headHash, headNumber) + if header == nil { + return fmt.Errorf("missing parent %d %x", headNumber, headHash) + } + } + } + // Extend the canonical chain with the new headers + for i := 0; i < len(headers); i++ { + hash := headers[i].Hash() + num := headers[i].Number.Uint64() + rawdb.WriteCanonicalHash(batch, hash, num) + rawdb.WriteHeadHeaderHash(batch, hash) + } + if err := batch.Write(); err != nil { + return err + } + // Last step update all in-memory head header markers + hc.currentHeaderHash = last.Hash() + hc.currentHeader.Store(types.CopyHeader(last)) + headHeaderGauge.Update(last.Number.Int64()) + return nil +} + +// WriteHeaders writes a chain of headers into the local chain, given that the +// parents are already known. The chain head header won't be updated in this +// function, the addtional setChainHead is expected in order to finish the entire +// procedure. +func (hc *HeaderChain) WriteHeaders(headers []*types.Header) (int, error) { if len(headers) == 0 { - return &headerWriteResult{}, nil + return 0, nil } ptd := hc.GetTd(headers[0].ParentHash, headers[0].Number.Uint64()-1) if ptd == nil { - return &headerWriteResult{}, consensus.ErrUnknownAncestor + return 0, consensus.ErrUnknownAncestor } var ( - lastNumber = headers[0].Number.Uint64() - 1 // Last successfully imported number - lastHash = headers[0].ParentHash // Last imported header hash - newTD = new(big.Int).Set(ptd) // Total difficulty of inserted chain - - lastHeader *types.Header - inserted []numberHash // Ephemeral lookup of number/hash for the chain - firstInserted = -1 // Index of the first non-ignored header + newTD = new(big.Int).Set(ptd) // Total difficulty of inserted chain + inserted []numberHash // Ephemeral lookup of number/hash for the chain + parentKnown = true // Set to true to force hc.HasHeader check the first iteration + batch = hc.chainDb.NewBatch() ) - - batch := hc.chainDb.NewBatch() - parentKnown := true // Set to true to force hc.HasHeader check the first iteration for i, header := range headers { var hash common.Hash // The headers have already been validated at this point, so we already @@ -191,113 +238,64 @@ func (hc *HeaderChain) writeHeaders(headers []*types.Header) (result *headerWrit inserted = append(inserted, numberHash{number, hash}) hc.headerCache.Add(hash, header) hc.numberCache.Add(hash, number) - if firstInserted < 0 { - firstInserted = i - } } parentKnown = alreadyKnown - lastHeader, lastHash, lastNumber = header, hash, number } - // Skip the slow disk write of all headers if interrupted. if hc.procInterrupt() { log.Debug("Premature abort during headers import") - return &headerWriteResult{}, errors.New("aborted") + return 0, errors.New("aborted") } // Commit to disk! if err := batch.Write(); err != nil { log.Crit("Failed to write headers", "error", err) } - batch.Reset() + return len(inserted), nil +} +// writeHeadersAndSetHead writes a batch of block headers and applies the last +// header as the chain head if the fork choicer says it's ok to update the chain. +// Note: This method is not concurrent-safe with inserting blocks simultaneously +// into the chain, as side effects caused by reorganisations cannot be emulated +// without the real blocks. Hence, writing headers directly should only be done +// in two scenarios: pure-header mode of operation (light clients), or properly +// separated header/block phases (non-archive clients). +func (hc *HeaderChain) writeHeadersAndSetHead(headers []*types.Header, forker *ForkChoice) (*headerWriteResult, error) { + inserted, err := hc.WriteHeaders(headers) + if err != nil { + return nil, err + } var ( - head = hc.CurrentHeader().Number.Uint64() - localTD = hc.GetTd(hc.currentHeaderHash, head) - status = SideStatTy + lastHeader = headers[len(headers)-1] + lastHash = headers[len(headers)-1].Hash() + result = &headerWriteResult{ + status: NonStatTy, + ignored: len(headers) - inserted, + imported: inserted, + lastHash: lastHash, + lastHeader: lastHeader, + } ) - // If the total difficulty is higher than our known, add it to the canonical chain - // Second clause in the if statement reduces the vulnerability to selfish mining. - // Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf - reorg := newTD.Cmp(localTD) > 0 - if !reorg && newTD.Cmp(localTD) == 0 { - if lastNumber < head { - reorg = true - } else if lastNumber == head { - reorg = mrand.Float64() < 0.5 + // Ask the fork choicer if the reorg is necessary + if reorg, err := forker.Reorg(hc.CurrentHeader(), lastHeader); err != nil { + return nil, err + } else if !reorg { + if inserted != 0 { + result.status = SideStatTy } + return result, nil } - // If the parent of the (first) block is already the canon header, - // we don't have to go backwards to delete canon blocks, but - // simply pile them onto the existing chain - chainAlreadyCanon := headers[0].ParentHash == hc.currentHeaderHash - if reorg { - // If the header can be added into canonical chain, adjust the - // header chain markers(canonical indexes and head header flag). - // - // Note all markers should be written atomically. - markerBatch := batch // we can reuse the batch to keep allocs down - if !chainAlreadyCanon { - // Delete any canonical number assignments above the new head - for i := lastNumber + 1; ; i++ { - hash := rawdb.ReadCanonicalHash(hc.chainDb, i) - if hash == (common.Hash{}) { - break - } - rawdb.DeleteCanonicalHash(markerBatch, i) - } - // Overwrite any stale canonical number assignments, going - // backwards from the first header in this import - var ( - headHash = headers[0].ParentHash // inserted[0].parent? - headNumber = headers[0].Number.Uint64() - 1 // inserted[0].num-1 ? - headHeader = hc.GetHeader(headHash, headNumber) - ) - for rawdb.ReadCanonicalHash(hc.chainDb, headNumber) != headHash { - rawdb.WriteCanonicalHash(markerBatch, headHash, headNumber) - headHash = headHeader.ParentHash - headNumber = headHeader.Number.Uint64() - 1 - headHeader = hc.GetHeader(headHash, headNumber) - } - // If some of the older headers were already known, but obtained canon-status - // during this import batch, then we need to write that now - // Further down, we continue writing the staus for the ones that - // were not already known - for i := 0; i < firstInserted; i++ { - hash := headers[i].Hash() - num := headers[i].Number.Uint64() - rawdb.WriteCanonicalHash(markerBatch, hash, num) - rawdb.WriteHeadHeaderHash(markerBatch, hash) - } - } - // Extend the canonical chain with the new headers - for _, hn := range inserted { - rawdb.WriteCanonicalHash(markerBatch, hn.hash, hn.number) - rawdb.WriteHeadHeaderHash(markerBatch, hn.hash) - } - if err := markerBatch.Write(); err != nil { - log.Crit("Failed to write header markers into disk", "err", err) - } - markerBatch.Reset() - // Last step update all in-memory head header markers - hc.currentHeaderHash = lastHash - hc.currentHeader.Store(types.CopyHeader(lastHeader)) - headHeaderGauge.Update(lastHeader.Number.Int64()) - - // Chain status is canonical since this insert was a reorg. - // Note that all inserts which have higher TD than existing are 'reorg'. - status = CanonStatTy - } - - if len(inserted) == 0 { - status = NonStatTy - } - return &headerWriteResult{ - status: status, - ignored: len(headers) - len(inserted), - imported: len(inserted), - lastHash: lastHash, - lastHeader: lastHeader, - }, nil + // Special case, all the inserted headers are already on the canonical + // header chain, skip the reorg operation. + if hc.GetCanonicalHash(lastHeader.Number.Uint64()) == lastHash && lastHeader.Number.Uint64() <= hc.CurrentHeader().Number.Uint64() { + return result, nil + } + // Apply the reorg operation + if err := hc.Reorg(headers); err != nil { + return nil, err + } + result.status = CanonStatTy + return result, nil } func (hc *HeaderChain) ValidateHeaderChain(chain []*types.Header, checkFreq int) (int, error) { @@ -357,7 +355,7 @@ func (hc *HeaderChain) ValidateHeaderChain(chain []*types.Header, checkFreq int) return 0, nil } -// InsertHeaderChain inserts the given headers. +// InsertHeaderChain inserts the given headers and does the reorganisations. // // The validity of the headers is NOT CHECKED by this method, i.e. they need to be // validated by ValidateHeaderChain before calling InsertHeaderChain. @@ -367,20 +365,19 @@ func (hc *HeaderChain) ValidateHeaderChain(chain []*types.Header, checkFreq int) // // The returned 'write status' says if the inserted headers are part of the canonical chain // or a side chain. -func (hc *HeaderChain) InsertHeaderChain(chain []*types.Header, start time.Time) (WriteStatus, error) { +func (hc *HeaderChain) InsertHeaderChain(chain []*types.Header, start time.Time, forker *ForkChoice) (WriteStatus, error) { if hc.procInterrupt() { return 0, errors.New("aborted") } - res, err := hc.writeHeaders(chain) - + res, err := hc.writeHeadersAndSetHead(chain, forker) + if err != nil { + return 0, err + } // Report some public statistics so the user has a clue what's going on context := []interface{}{ "count", res.imported, "elapsed", common.PrettyDuration(time.Since(start)), } - if err != nil { - context = append(context, "err", err) - } if last := res.lastHeader; last != nil { context = append(context, "number", last.Number, "hash", res.lastHash) if timestamp := time.Unix(int64(last.Time), 0); time.Since(timestamp) > time.Minute { diff --git a/core/headerchain_test.go b/core/headerchain_test.go index f3e40b6213f3..5371ff98642d 100644 --- a/core/headerchain_test.go +++ b/core/headerchain_test.go @@ -51,10 +51,10 @@ func verifyUnbrokenCanonchain(hc *HeaderChain) error { return nil } -func testInsert(t *testing.T, hc *HeaderChain, chain []*types.Header, wantStatus WriteStatus, wantErr error) { +func testInsert(t *testing.T, hc *HeaderChain, chain []*types.Header, wantStatus WriteStatus, wantErr error, forker *ForkChoice) { t.Helper() - status, err := hc.InsertHeaderChain(chain, time.Now()) + status, err := hc.InsertHeaderChain(chain, time.Now(), forker) if status != wantStatus { t.Errorf("wrong write status from InsertHeaderChain: got %v, want %v", status, wantStatus) } @@ -80,37 +80,38 @@ func TestHeaderInsertion(t *testing.T) { } // chain A: G->A1->A2...A128 chainA := makeHeaderChain(genesis.Header(), 128, ethash.NewFaker(), db, 10) - // chain B: G->A1->B2...B128 + // chain B: G->A1->B1...B128 chainB := makeHeaderChain(chainA[0], 128, ethash.NewFaker(), db, 10) log.Root().SetHandler(log.StdoutHandler) + forker := NewForkChoice(hc, false, nil) // Inserting 64 headers on an empty chain, expecting // 1 callbacks, 1 canon-status, 0 sidestatus, - testInsert(t, hc, chainA[:64], CanonStatTy, nil) + testInsert(t, hc, chainA[:64], CanonStatTy, nil, forker) // Inserting 64 identical headers, expecting // 0 callbacks, 0 canon-status, 0 sidestatus, - testInsert(t, hc, chainA[:64], NonStatTy, nil) + testInsert(t, hc, chainA[:64], NonStatTy, nil, forker) // Inserting the same some old, some new headers // 1 callbacks, 1 canon, 0 side - testInsert(t, hc, chainA[32:96], CanonStatTy, nil) + testInsert(t, hc, chainA[32:96], CanonStatTy, nil, forker) // Inserting side blocks, but not overtaking the canon chain - testInsert(t, hc, chainB[0:32], SideStatTy, nil) + testInsert(t, hc, chainB[0:32], SideStatTy, nil, forker) // Inserting more side blocks, but we don't have the parent - testInsert(t, hc, chainB[34:36], NonStatTy, consensus.ErrUnknownAncestor) + testInsert(t, hc, chainB[34:36], NonStatTy, consensus.ErrUnknownAncestor, forker) // Inserting more sideblocks, overtaking the canon chain - testInsert(t, hc, chainB[32:97], CanonStatTy, nil) + testInsert(t, hc, chainB[32:97], CanonStatTy, nil, forker) // Inserting more A-headers, taking back the canonicality - testInsert(t, hc, chainA[90:100], CanonStatTy, nil) + testInsert(t, hc, chainA[90:100], CanonStatTy, nil, forker) // And B becomes canon again - testInsert(t, hc, chainB[97:107], CanonStatTy, nil) + testInsert(t, hc, chainB[97:107], CanonStatTy, nil, forker) // And B becomes even longer - testInsert(t, hc, chainB[107:128], CanonStatTy, nil) + testInsert(t, hc, chainB[107:128], CanonStatTy, nil, forker) } diff --git a/core/merger.go b/core/merger.go new file mode 100644 index 000000000000..c6e2f0eceb89 --- /dev/null +++ b/core/merger.go @@ -0,0 +1,134 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package core + +import ( + "sync" + + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" +) + +// transitionStatus describes the status of eth1/2 transition. This switch +// between modes is a one-way action which is triggered by corresponding +// consensus-layer message. +type transitionStatus struct { + LeftPoW bool // The flag is set when the first NewHead message received + EnteredPoS bool // The flag is set when the first FinalisedBlock message received +} + +// Merger is an internal help structure used to track the eth1/2 transition status. +// It's a common structure can be used in both full node and light client. +type Merger struct { + db ethdb.KeyValueStore + status transitionStatus + leavePoWCalls []func() + enterPoSCalls []func() + lock sync.Mutex +} + +func NewMerger(db ethdb.KeyValueStore) *Merger { + var status transitionStatus + blob := rawdb.ReadTransitionStatus(db) + if len(blob) != 0 { + if err := rlp.DecodeBytes(blob, &status); err != nil { + log.Crit("Failed to decode the transition status", "err", err) + } + } + return &Merger{ + db: db, + status: status, + } +} + +// SubscribeLeavePoW registers callback so that if the chain leaves +// from the PoW stage and enters to 'transition' stage it can be invoked. +func (m *Merger) SubscribeLeavePoW(callback func()) { + m.lock.Lock() + defer m.lock.Unlock() + + m.leavePoWCalls = append(m.leavePoWCalls, callback) +} + +// SubscribeEnterPoS registers callback so that if the chain leaves +// from the 'transition' stage and enters the PoS stage it can be invoked. +func (m *Merger) SubscribeEnterPoS(callback func()) { + m.lock.Lock() + defer m.lock.Unlock() + + m.enterPoSCalls = append(m.enterPoSCalls, callback) +} + +// LeavePoW is called whenever the first NewHead message received +// from the consensus-layer. +func (m *Merger) LeavePoW() { + m.lock.Lock() + defer m.lock.Unlock() + + if m.status.LeftPoW { + return + } + m.status = transitionStatus{LeftPoW: true} + blob, err := rlp.EncodeToBytes(m.status) + if err != nil { + log.Crit("Failed to encode the transition status", "err", err) + } + rawdb.WriteTransitionStatus(m.db, blob) + for _, call := range m.leavePoWCalls { + call() + } + log.Info("Left PoW stage") +} + +// EnterPoS is called whenever the first FinalisedBlock message received +// from the consensus-layer. +func (m *Merger) EnterPoS() { + m.lock.Lock() + defer m.lock.Unlock() + + if m.status.EnteredPoS { + return + } + m.status = transitionStatus{LeftPoW: true, EnteredPoS: true} + blob, err := rlp.EncodeToBytes(m.status) + if err != nil { + log.Crit("Failed to encode the transition status", "err", err) + } + rawdb.WriteTransitionStatus(m.db, blob) + for _, call := range m.enterPoSCalls { + call() + } + log.Info("Entered PoS stage") +} + +// LeftPoW reports whether the chain has left the PoW stage. +func (m *Merger) LeftPoW() bool { + m.lock.Lock() + defer m.lock.Unlock() + + return m.status.LeftPoW +} + +// EnteredPoS reports whether the chain has entered the PoS stage. +func (m *Merger) EnteredPoS() bool { + m.lock.Lock() + defer m.lock.Unlock() + + return m.status.EnteredPoS +} diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go index 079e335fa6fb..4c72ca714838 100644 --- a/core/rawdb/accessors_metadata.go +++ b/core/rawdb/accessors_metadata.go @@ -138,3 +138,16 @@ func PopUncleanShutdownMarker(db ethdb.KeyValueStore) { log.Warn("Failed to clear unclean-shutdown marker", "err", err) } } + +// ReadTransitionStatus retrieves the eth2 transition status from the database +func ReadTransitionStatus(db ethdb.KeyValueReader) []byte { + data, _ := db.Get(transitionStatusKey) + return data +} + +// WriteTransitionStatus stores the eth2 transition status to the database +func WriteTransitionStatus(db ethdb.KeyValueWriter, data []byte) { + if err := db.Put(transitionStatusKey, data); err != nil { + log.Crit("Failed to store the eth2 transition status", "err", err) + } +} diff --git a/core/rawdb/database.go b/core/rawdb/database.go index f7c7f5d3363f..a6a6f790b40f 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -395,7 +395,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { databaseVersionKey, headHeaderKey, headBlockKey, headFastBlockKey, lastPivotKey, fastTrieProgressKey, snapshotDisabledKey, snapshotRootKey, snapshotJournalKey, snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey, - uncleanShutdownKey, badBlockKey, + uncleanShutdownKey, badBlockKey, transitionStatusKey, } { if bytes.Equal(key, meta) { metadata.Add(size) diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 2505ce90b947..9fd6d39b0023 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -75,6 +75,9 @@ var ( // uncleanShutdownKey tracks the list of local crashes uncleanShutdownKey = []byte("unclean-shutdown") // config prefix for the db + // transitionStatusKey tracks the eth2 transition status. + transitionStatusKey = []byte("eth2-transition") + // Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes). headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header headerTDSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + headerTDSuffix -> td diff --git a/core/state_processor_test.go b/core/state_processor_test.go index 13a9eb810df6..ee493416f456 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -85,7 +85,7 @@ func TestStateProcessorErrors(t *testing.T) { }, } genesis = gspec.MustCommit(db) - blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) ) defer blockchain.Stop() bigNumber := new(big.Int).SetBytes(common.FromHex("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")) @@ -220,7 +220,7 @@ func TestStateProcessorErrors(t *testing.T) { }, } genesis = gspec.MustCommit(db) - blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) ) defer blockchain.Stop() for i, tt := range []struct { @@ -260,7 +260,7 @@ func TestStateProcessorErrors(t *testing.T) { }, } genesis = gspec.MustCommit(db) - blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(nil)) ) defer blockchain.Stop() for i, tt := range []struct { diff --git a/eth/api_backend.go b/eth/api_backend.go index 01e68f6781dd..60bb92d718fe 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -353,7 +353,7 @@ func (b *EthAPIBackend) StartMining(threads int) error { } func (b *EthAPIBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, checkLive bool) (*state.StateDB, error) { - return b.eth.stateAtBlock(block, reexec, base, checkLive) + return b.eth.StateAtBlock(block, reexec, base, checkLive) } func (b *EthAPIBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, error) { diff --git a/eth/backend.go b/eth/backend.go index ae4e6e85d7fd..ff98191fabe8 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -71,6 +71,7 @@ type Ethereum struct { handler *handler ethDialCandidates enode.Iterator snapDialCandidates enode.Iterator + merger *core.Merger // DB interfaces chainDb ethdb.Database // Block chain database @@ -140,12 +141,14 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb, stack.ResolvePath(config.TrieCleanCacheJournal)); err != nil { log.Error("Failed to recover state", "error", err) } + merger := core.NewMerger(chainDb) eth := &Ethereum{ config: config, + merger: merger, chainDb: chainDb, eventMux: stack.EventMux(), accountManager: stack.AccountManager(), - engine: ethconfig.CreateConsensusEngine(stack, chainConfig, ðashConfig, config.Miner.Notify, config.Miner.Noverify, chainDb), + engine: ethconfig.CreateConsensusEngine(stack, chainConfig, ðashConfig, config.Miner.Notify, config.Miner.Noverify, chainDb, merger), closeBloomHandler: make(chan struct{}), networkID: config.NetworkId, gasPrice: config.Miner.GasPrice, @@ -188,7 +191,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { Preimages: config.Preimages, } ) - eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, chainConfig, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit) + eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, chainConfig, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit, merger) if err != nil { return nil, err } @@ -215,6 +218,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { Database: chainDb, Chain: eth.blockchain, TxPool: eth.txPool, + Merger: merger, Network: config.NetworkId, Sync: config.SyncMode, BloomCache: uint64(cacheLimit), @@ -225,7 +229,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { return nil, err } - eth.miner = miner.New(eth, &config.Miner, chainConfig, eth.EventMux(), eth.engine, eth.isLocalBlock) + eth.miner = miner.New(eth, &config.Miner, chainConfig, eth.EventMux(), eth.engine, eth.isLocalBlock, merger) eth.miner.SetExtra(makeExtraData(config.Miner.ExtraData)) eth.APIBackend = &EthAPIBackend{stack.Config().ExtRPCEnabled(), stack.Config().AllowUnprotectedTxs, eth, nil} @@ -256,6 +260,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { stack.RegisterAPIs(eth.APIs()) stack.RegisterProtocols(eth.Protocols()) stack.RegisterLifecycle(eth) + // Check for unclean shutdown if uncleanShutdowns, discards, err := rawdb.PushUncleanShutdownMarker(chainDb); err != nil { log.Error("Could not update unclean-shutdown-marker list", "error", err) @@ -378,10 +383,10 @@ func (s *Ethereum) Etherbase() (eb common.Address, err error) { // // We regard two types of accounts as local miner account: etherbase // and accounts specified via `txpool.locals` flag. -func (s *Ethereum) isLocalBlock(block *types.Block) bool { - author, err := s.engine.Author(block.Header()) +func (s *Ethereum) isLocalBlock(header *types.Header) bool { + author, err := s.engine.Author(header) if err != nil { - log.Warn("Failed to retrieve block author", "number", block.NumberU64(), "hash", block.Hash(), "err", err) + log.Warn("Failed to retrieve block author", "number", header.Number.Uint64(), "hash", header.Hash(), "err", err) return false } // Check whether the given address is etherbase. @@ -404,7 +409,7 @@ func (s *Ethereum) isLocalBlock(block *types.Block) bool { // shouldPreserve checks whether we should preserve the given block // during the chain reorg depending on whether the author of block // is a local account. -func (s *Ethereum) shouldPreserve(block *types.Block) bool { +func (s *Ethereum) shouldPreserve(header *types.Header) bool { // The reason we need to disable the self-reorg preserving for clique // is it can be probable to introduce a deadlock. // @@ -424,7 +429,7 @@ func (s *Ethereum) shouldPreserve(block *types.Block) bool { if _, ok := s.engine.(*clique.Clique); ok { return false } - return s.isLocalBlock(block) + return s.isLocalBlock(header) } // SetEtherbase sets the mining reward address. @@ -508,8 +513,10 @@ func (s *Ethereum) ChainDb() ethdb.Database { return s.chainDb } func (s *Ethereum) IsListening() bool { return true } // Always listening func (s *Ethereum) Downloader() *downloader.Downloader { return s.handler.downloader } func (s *Ethereum) Synced() bool { return atomic.LoadUint32(&s.handler.acceptTxs) == 1 } +func (s *Ethereum) SetSynced() { atomic.StoreUint32(&s.handler.acceptTxs, 1) } func (s *Ethereum) ArchiveMode() bool { return s.config.NoPruning } func (s *Ethereum) BloomIndexer() *core.ChainIndexer { return s.bloomIndexer } +func (s *Ethereum) Merger() *core.Merger { return s.merger } // Protocols returns all the currently configured // network protocols to start. diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 3913da757222..e6b40416e3fc 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -24,11 +24,14 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/consensus/misc" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/les" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" chainParams "github.com/ethereum/go-ethereum/params" @@ -36,31 +39,64 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -// Register adds catalyst APIs to the node. +// Register adds catalyst APIs to the full node. func Register(stack *node.Node, backend *eth.Ethereum) error { - chainconfig := backend.BlockChain().Config() - if chainconfig.TerminalTotalDifficulty == nil { - return errors.New("catalyst started without valid total difficulty") - } + log.Warn("Catalyst mode enabled", "protocol", "eth") + stack.RegisterAPIs([]rpc.API{ + { + Namespace: "consensus", + Version: "1.0", + Service: NewConsensusAPI(backend, nil), + Public: true, + }, + }) + return nil +} - log.Warn("Catalyst mode enabled") +// RegisterLight adds catalyst APIs to the light client. +func RegisterLight(stack *node.Node, backend *les.LightEthereum) error { + log.Warn("Catalyst mode enabled", "protocol", "les") stack.RegisterAPIs([]rpc.API{ { Namespace: "consensus", Version: "1.0", - Service: newConsensusAPI(backend), + Service: NewConsensusAPI(nil, backend), Public: true, }, }) return nil } -type consensusAPI struct { - eth *eth.Ethereum +type ConsensusAPI struct { + light bool + eth *eth.Ethereum + les *les.LightEthereum + engine consensus.Engine // engine is the post-merge consensus engine, only for block creation + syncer *syncer // syncer is responsible for triggering chain sync } -func newConsensusAPI(eth *eth.Ethereum) *consensusAPI { - return &consensusAPI{eth: eth} +func NewConsensusAPI(eth *eth.Ethereum, les *les.LightEthereum) *ConsensusAPI { + var engine consensus.Engine + if eth == nil { + if b, ok := les.Engine().(*beacon.Beacon); ok { + engine = beacon.New(b.InnerEngine(), true) + } else { + engine = beacon.New(les.Engine(), true) + } + } else { + if b, ok := eth.Engine().(*beacon.Beacon); ok { + engine = beacon.New(b.InnerEngine(), true) + } else { + engine = beacon.New(eth.Engine(), true) + } + } + return &ConsensusAPI{ + light: eth == nil, + eth: eth, + les: les, + engine: engine, + syncer: newSyncer(), + } } // blockExecutionEnv gathers all the data required to execute @@ -89,8 +125,24 @@ func (env *blockExecutionEnv) commitTransaction(tx *types.Transaction, coinbase return nil } -func (api *consensusAPI) makeEnv(parent *types.Block, header *types.Header) (*blockExecutionEnv, error) { - state, err := api.eth.BlockChain().StateAt(parent.Root()) +func (api *ConsensusAPI) makeEnv(parent *types.Block, header *types.Header) (*blockExecutionEnv, error) { + // The parent state might be missing. It can be the special scenario + // that consensus layer tries to build a new block based on the very + // old side chain block and the relevant state is already pruned. So + // try to retrieve the live state from the chain, if it's not existent, + // do the necessary recovery work. + var ( + err error + state *state.StateDB + ) + if api.eth.BlockChain().HasState(parent.Root()) { + state, err = api.eth.BlockChain().StateAt(parent.Root()) + } else { + // The maximum acceptable reorg depth can be limited by the + // finalised block somehow. TODO(rjl493456442) fix the hard- + // coded number here later. + state, err = api.eth.StateAtBlock(parent, 1000, nil, false) + } if err != nil { return nil, err } @@ -105,7 +157,10 @@ func (api *consensusAPI) makeEnv(parent *types.Block, header *types.Header) (*bl // AssembleBlock creates a new block, inserts it into the chain, and returns the "execution // data" required for eth2 clients to process the new block. -func (api *consensusAPI) AssembleBlock(params assembleBlockParams) (*executableData, error) { +func (api *ConsensusAPI) AssembleBlock(params AssembleBlockParams) (*ExecutableData, error) { + if api.light { + return nil, errors.New("not supported") + } log.Info("Producing block", "parentHash", params.ParentHash) bc := api.eth.BlockChain() @@ -115,8 +170,6 @@ func (api *consensusAPI) AssembleBlock(params assembleBlockParams) (*executableD return nil, fmt.Errorf("cannot assemble block with unknown parent %s", params.ParentHash) } - pool := api.eth.TxPool() - if parent.Time() >= params.Timestamp { return nil, fmt.Errorf("child timestamp lower than parent's: %d >= %d", parent.Time(), params.Timestamp) } @@ -125,9 +178,7 @@ func (api *consensusAPI) AssembleBlock(params assembleBlockParams) (*executableD log.Info("Producing block too far in the future", "wait", common.PrettyDuration(wait)) time.Sleep(wait) } - - pending := pool.Pending(true) - + pending := api.eth.TxPool().Pending(true) coinbase, err := api.eth.Etherbase() if err != nil { return nil, err @@ -144,16 +195,14 @@ func (api *consensusAPI) AssembleBlock(params assembleBlockParams) (*executableD if config := api.eth.BlockChain().Config(); config.IsLondon(header.Number) { header.BaseFee = misc.CalcBaseFee(config, parent.Header()) } - err = api.eth.Engine().Prepare(bc, header) + err = api.engine.Prepare(bc, header) if err != nil { return nil, err } - env, err := api.makeEnv(parent, header) if err != nil { return nil, err } - var ( signer = types.MakeSigner(bc.Config(), header.Number) txHeap = types.NewTransactionsByPriceAndNonce(signer, pending, nil) @@ -204,13 +253,12 @@ func (api *consensusAPI) AssembleBlock(params assembleBlockParams) (*executableD txHeap.Shift() } } - // Create the block. - block, err := api.eth.Engine().FinalizeAndAssemble(bc, header, env.state, transactions, nil /* uncles */, env.receipts) + block, err := api.engine.FinalizeAndAssemble(bc, header, env.state, transactions, nil /* uncles */, env.receipts) if err != nil { return nil, err } - return &executableData{ + return &ExecutableData{ BlockHash: block.Hash(), ParentHash: block.ParentHash(), Miner: block.Coinbase(), @@ -245,12 +293,11 @@ func decodeTransactions(enc [][]byte) ([]*types.Transaction, error) { return txs, nil } -func insertBlockParamsToBlock(config *chainParams.ChainConfig, parent *types.Header, params executableData) (*types.Block, error) { +func InsertBlockParamsToBlock(config *chainParams.ChainConfig, parent *types.Header, params ExecutableData) (*types.Block, error) { txs, err := decodeTransactions(params.Transactions) if err != nil { return nil, err } - number := big.NewInt(0) number.SetUint64(params.Number) header := &types.Header{ @@ -271,27 +318,42 @@ func insertBlockParamsToBlock(config *chainParams.ChainConfig, parent *types.Hea header.BaseFee = misc.CalcBaseFee(config, parent) } block := types.NewBlockWithHeader(header).WithBody(txs, nil /* uncles */) + if block.Hash() != params.BlockHash { + return nil, fmt.Errorf("blockhash mismatch, want %x, got %x", params.BlockHash, block.Hash()) + } return block, nil } // NewBlock creates an Eth1 block, inserts it in the chain, and either returns true, // or false + an error. This is a bit redundant for go, but simplifies things on the // eth2 side. -func (api *consensusAPI) NewBlock(params executableData) (*newBlockResponse, error) { +func (api *ConsensusAPI) NewBlock(params ExecutableData) (*NewBlockResponse, error) { + if api.light { + parent := api.les.BlockChain().GetHeaderByHash(params.ParentHash) + if parent == nil { + return &NewBlockResponse{false}, fmt.Errorf("could not find parent %x", params.ParentHash) + } + block, err := InsertBlockParamsToBlock(api.les.BlockChain().Config(), parent, params) + if err != nil { + return nil, err + } + err = api.les.BlockChain().InsertHeader(block.Header()) + return &NewBlockResponse{err == nil}, err + } parent := api.eth.BlockChain().GetBlockByHash(params.ParentHash) if parent == nil { - return &newBlockResponse{false}, fmt.Errorf("could not find parent %x", params.ParentHash) + return &NewBlockResponse{false}, fmt.Errorf("could not find parent %x", params.ParentHash) } - block, err := insertBlockParamsToBlock(api.eth.BlockChain().Config(), parent.Header(), params) + block, err := InsertBlockParamsToBlock(api.eth.BlockChain().Config(), parent.Header(), params) if err != nil { return nil, err } - _, err = api.eth.BlockChain().InsertChainWithoutSealVerification(block) - return &newBlockResponse{err == nil}, err + err = api.eth.BlockChain().InsertBlock(block) + return &NewBlockResponse{err == nil}, err } // Used in tests to add a the list of transactions from a block to the tx pool. -func (api *consensusAPI) addBlockTxs(block *types.Block) error { +func (api *ConsensusAPI) addBlockTxs(block *types.Block) error { for _, tx := range block.Transactions() { api.eth.TxPool().AddLocal(tx) } @@ -300,11 +362,55 @@ func (api *consensusAPI) addBlockTxs(block *types.Block) error { // FinalizeBlock is called to mark a block as synchronized, so // that data that is no longer needed can be removed. -func (api *consensusAPI) FinalizeBlock(blockHash common.Hash) (*genericResponse, error) { - return &genericResponse{true}, nil +func (api *ConsensusAPI) FinalizeBlock(blockHash common.Hash) (*GenericResponse, error) { + // Finalize the transition if it's the first `FinalisedBlock` event. + merger := api.merger() + if !merger.EnteredPoS() { + merger.EnterPoS() + } + return &GenericResponse{true}, nil } // SetHead is called to perform a force choice. -func (api *consensusAPI) SetHead(newHead common.Hash) (*genericResponse, error) { - return &genericResponse{true}, nil +func (api *ConsensusAPI) SetHead(newHead common.Hash) (*GenericResponse, error) { + // Trigger the transition if it's the first `NewHead` event. + merger := api.merger() + if !merger.LeftPoW() { + merger.LeavePoW() + } + if api.light { + headHeader := api.les.BlockChain().CurrentHeader() + if headHeader.Hash() == newHead { + return &GenericResponse{true}, nil + } + newHeadHeader := api.les.BlockChain().GetHeaderByHash(newHead) + if newHeadHeader == nil { + return &GenericResponse{false}, nil + } + if err := api.les.BlockChain().SetChainHead(newHeadHeader); err != nil { + return &GenericResponse{false}, nil + } + return &GenericResponse{true}, nil + } + headBlock := api.eth.BlockChain().CurrentBlock() + if headBlock.Hash() == newHead { + return &GenericResponse{true}, nil + } + newHeadBlock := api.eth.BlockChain().GetBlockByHash(newHead) + if newHeadBlock == nil { + return &GenericResponse{false}, nil + } + if err := api.eth.BlockChain().SetChainHead(newHeadBlock); err != nil { + return &GenericResponse{false}, nil + } + api.eth.SetSynced() + return &GenericResponse{true}, nil +} + +// Helper function, return the merger instance. +func (api *ConsensusAPI) merger() *core.Merger { + if api.light { + return api.les.Merger() + } + return api.eth.Merger() } diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 11042086305d..a3d7c41e697e 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -19,7 +19,9 @@ package catalyst import ( "math/big" "testing" + "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" @@ -38,10 +40,10 @@ var ( // testAddr is the Ethereum address of the tester account. testAddr = crypto.PubkeyToAddress(testKey.PublicKey) - testBalance = big.NewInt(2e15) + testBalance = big.NewInt(2e18) ) -func generateTestChain() (*core.Genesis, []*types.Block) { +func generatePreMergeChain(n int) (*core.Genesis, []*types.Block) { db := rawdb.NewMemoryDatabase() config := params.AllEthashProtocolChanges genesis := &core.Genesis{ @@ -51,14 +53,17 @@ func generateTestChain() (*core.Genesis, []*types.Block) { Timestamp: 9000, BaseFee: big.NewInt(params.InitialBaseFee), } + testNonce := uint64(0) generate := func(i int, g *core.BlockGen) { g.OffsetTime(5) g.SetExtra([]byte("test")) + tx, _ := types.SignTx(types.NewTransaction(testNonce, common.HexToAddress("0x9a9070028361F7AAbeB3f2F2Dc07F82C4a98A02a"), big.NewInt(1), params.TxGas, big.NewInt(params.InitialBaseFee*2), nil), types.LatestSigner(config), testKey) + g.AddTx(tx) + testNonce++ } gblock := genesis.ToBlock(db) engine := ethash.NewFaker() - blocks, _ := core.GenerateChain(config, gblock, engine, db, 10, generate) - blocks = append([]*types.Block{gblock}, blocks...) + blocks, _ := core.GenerateChain(config, gblock, engine, db, n, generate) return genesis, blocks } @@ -110,118 +115,204 @@ func generateTestChainWithFork(n int, fork int) (*core.Genesis, []*types.Block, */ func TestEth2AssembleBlock(t *testing.T) { - genesis, blocks := generateTestChain() - n, ethservice := startEthService(t, genesis, blocks[1:9]) + genesis, blocks := generatePreMergeChain(10) + n, ethservice := startEthService(t, genesis, blocks) defer n.Close() - api := newConsensusAPI(ethservice) + api := NewConsensusAPI(ethservice, nil) signer := types.NewEIP155Signer(ethservice.BlockChain().Config().ChainID) - tx, err := types.SignTx(types.NewTransaction(0, blocks[8].Coinbase(), big.NewInt(1000), params.TxGas, big.NewInt(params.InitialBaseFee), nil), signer, testKey) + tx, err := types.SignTx(types.NewTransaction(uint64(10), blocks[9].Coinbase(), big.NewInt(1000), params.TxGas, big.NewInt(params.InitialBaseFee), nil), signer, testKey) if err != nil { t.Fatalf("error signing transaction, err=%v", err) } ethservice.TxPool().AddLocal(tx) - blockParams := assembleBlockParams{ - ParentHash: blocks[8].ParentHash(), - Timestamp: blocks[8].Time(), + blockParams := AssembleBlockParams{ + ParentHash: blocks[9].Hash(), + Timestamp: blocks[9].Time() + 5, } execData, err := api.AssembleBlock(blockParams) - if err != nil { t.Fatalf("error producing block, err=%v", err) } - if len(execData.Transactions) != 1 { t.Fatalf("invalid number of transactions %d != 1", len(execData.Transactions)) } } func TestEth2AssembleBlockWithAnotherBlocksTxs(t *testing.T) { - genesis, blocks := generateTestChain() - n, ethservice := startEthService(t, genesis, blocks[1:9]) + genesis, blocks := generatePreMergeChain(10) + n, ethservice := startEthService(t, genesis, blocks[:9]) defer n.Close() - api := newConsensusAPI(ethservice) + api := NewConsensusAPI(ethservice, nil) // Put the 10th block's tx in the pool and produce a new block api.addBlockTxs(blocks[9]) - blockParams := assembleBlockParams{ - ParentHash: blocks[9].ParentHash(), - Timestamp: blocks[9].Time(), + blockParams := AssembleBlockParams{ + ParentHash: blocks[8].Hash(), + Timestamp: blocks[8].Time() + 5, } execData, err := api.AssembleBlock(blockParams) if err != nil { t.Fatalf("error producing block, err=%v", err) } - if len(execData.Transactions) != blocks[9].Transactions().Len() { t.Fatalf("invalid number of transactions %d != 1", len(execData.Transactions)) } } -// TODO (MariusVanDerWijden) reenable once engine api is updated to the latest spec -/* +func checkLogEvents(t *testing.T, logsCh <-chan []*types.Log, rmLogsCh <-chan core.RemovedLogsEvent, wantNew, wantRemoved int) { + t.Helper() + + if len(logsCh) != wantNew { + t.Fatalf("wrong number of log events: got %d, want %d", len(logsCh), wantNew) + } + if len(rmLogsCh) != wantRemoved { + t.Fatalf("wrong number of removed log events: got %d, want %d", len(rmLogsCh), wantRemoved) + } + // Drain events. + for i := 0; i < len(logsCh); i++ { + <-logsCh + } + for i := 0; i < len(rmLogsCh); i++ { + <-rmLogsCh + } +} + func TestEth2NewBlock(t *testing.T) { - genesis, blocks, forkedBlocks := generateTestChainWithFork(10, 4) - n, ethservice := startEthService(t, genesis, blocks[1:5]) + genesis, preMergeBlocks := generatePreMergeChain(10) + n, ethservice := startEthService(t, genesis, preMergeBlocks) defer n.Close() - api := newConsensusAPI(ethservice) - for i := 5; i < 10; i++ { - p := executableData{ - ParentHash: ethservice.BlockChain().CurrentBlock().Hash(), - Miner: blocks[i].Coinbase(), - StateRoot: blocks[i].Root(), - GasLimit: blocks[i].GasLimit(), - GasUsed: blocks[i].GasUsed(), - Transactions: encodeTransactions(blocks[i].Transactions()), - ReceiptRoot: blocks[i].ReceiptHash(), - LogsBloom: blocks[i].Bloom().Bytes(), - BlockHash: blocks[i].Hash(), - Timestamp: blocks[i].Time(), - Number: uint64(i), - } - success, err := api.NewBlock(p) - if err != nil || !success.Valid { + var ( + api = NewConsensusAPI(ethservice, nil) + parent = preMergeBlocks[len(preMergeBlocks)-1] + + // This EVM code generates a log when the contract is created. + logCode = common.Hex2Bytes("60606040525b7f24ec1d3ff24c2f6ff210738839dbc339cd45a5294d85c79361016243157aae7b60405180905060405180910390a15b600a8060416000396000f360606040526008565b00") + ) + // The event channels. + newLogCh := make(chan []*types.Log, 10) + rmLogsCh := make(chan core.RemovedLogsEvent, 10) + ethservice.BlockChain().SubscribeLogsEvent(newLogCh) + ethservice.BlockChain().SubscribeRemovedLogsEvent(rmLogsCh) + + for i := 0; i < 10; i++ { + statedb, _ := ethservice.BlockChain().StateAt(parent.Root()) + nonce := statedb.GetNonce(testAddr) + tx, err := types.SignTx(types.NewContractCreation(nonce, new(big.Int), 1000000, big.NewInt(2*params.InitialBaseFee), logCode), types.LatestSigner(ethservice.BlockChain().Config()), testKey) + ethservice.TxPool().AddLocal(tx) + + execData, err := api.AssembleBlock(AssembleBlockParams{ + ParentHash: parent.Hash(), + Timestamp: parent.Time() + 5, + }) + if err != nil { + t.Fatalf("Failed to create the executable data %v", err) + } + block, err := InsertBlockParamsToBlock(ethservice.BlockChain().Config(), parent.Header(), *execData) + if err != nil { + t.Fatalf("Failed to convert executable data to block %v", err) + } + newResp, err := api.NewBlock(*execData) + if err != nil || !newResp.Valid { + t.Fatalf("Failed to insert block: %v", err) + } + if ethservice.BlockChain().CurrentBlock().NumberU64() != block.NumberU64()-1 { + t.Fatalf("Chain head shouldn't be updated") + } + checkLogEvents(t, newLogCh, rmLogsCh, 0, 0) + + setResp, err := api.SetHead(block.Hash()) + if err != nil || !setResp.Success { t.Fatalf("Failed to insert block: %v", err) } + if ethservice.BlockChain().CurrentBlock().NumberU64() != block.NumberU64() { + t.Fatalf("Chain head should be updated") + } + checkLogEvents(t, newLogCh, rmLogsCh, 1, 0) + + parent = block } - exp := ethservice.BlockChain().CurrentBlock().Hash() - - // Introduce the fork point. - lastBlockNum := blocks[4].Number() - lastBlock := blocks[4] - for i := 0; i < 4; i++ { - lastBlockNum.Add(lastBlockNum, big.NewInt(1)) - p := executableData{ - ParentHash: lastBlock.Hash(), - Miner: forkedBlocks[i].Coinbase(), - StateRoot: forkedBlocks[i].Root(), - Number: lastBlockNum.Uint64(), - GasLimit: forkedBlocks[i].GasLimit(), - GasUsed: forkedBlocks[i].GasUsed(), - Transactions: encodeTransactions(blocks[i].Transactions()), - ReceiptRoot: forkedBlocks[i].ReceiptHash(), - LogsBloom: forkedBlocks[i].Bloom().Bytes(), - BlockHash: forkedBlocks[i].Hash(), - Timestamp: forkedBlocks[i].Time(), - } - success, err := api.NewBlock(p) - if err != nil || !success.Valid { - t.Fatalf("Failed to insert forked block #%d: %v", i, err) - } - lastBlock, err = insertBlockParamsToBlock(ethservice.BlockChain().Config(), lastBlock.Header(), p) + // Introduce fork chain + var ( + head = ethservice.BlockChain().CurrentBlock().NumberU64() + ) + parent = preMergeBlocks[len(preMergeBlocks)-1] + for i := 0; i < 10; i++ { + execData, err := api.AssembleBlock(AssembleBlockParams{ + ParentHash: parent.Hash(), + Timestamp: parent.Time() + 6, + }) + if err != nil { + t.Fatalf("Failed to create the executable data %v", err) + } + block, err := InsertBlockParamsToBlock(ethservice.BlockChain().Config(), parent.Header(), *execData) if err != nil { - t.Fatal(err) + t.Fatalf("Failed to convert executable data to block %v", err) + } + newResp, err := api.NewBlock(*execData) + if err != nil || !newResp.Valid { + t.Fatalf("Failed to insert block: %v", err) + } + if ethservice.BlockChain().CurrentBlock().NumberU64() != head { + t.Fatalf("Chain head shouldn't be updated") } + + setResp, err := api.SetHead(block.Hash()) + if err != nil || !setResp.Success { + t.Fatalf("Failed to insert block: %v", err) + } + if ethservice.BlockChain().CurrentBlock().NumberU64() != block.NumberU64() { + t.Fatalf("Chain head should be updated") + } + parent, head = block, block.NumberU64() } +} + +func TestEth2DeepReorg(t *testing.T) { + genesis, preMergeBlocks := generatePreMergeChain(core.TriesInMemory * 2) + n, ethservice := startEthService(t, genesis, preMergeBlocks) + defer n.Close() - if ethservice.BlockChain().CurrentBlock().Hash() != exp { - t.Fatalf("Wrong head after inserting fork %x != %x", exp, ethservice.BlockChain().CurrentBlock().Hash()) + var ( + api = NewConsensusAPI(ethservice, nil) + parent = preMergeBlocks[len(preMergeBlocks)-core.TriesInMemory-1] + head = ethservice.BlockChain().CurrentBlock().NumberU64() + ) + if ethservice.BlockChain().HasBlockAndState(parent.Hash(), parent.NumberU64()) { + t.Errorf("Block %d not pruned", parent.NumberU64()) + } + for i := 0; i < 10; i++ { + execData, err := api.AssembleBlock(AssembleBlockParams{ + ParentHash: parent.Hash(), + Timestamp: parent.Time() + 5, + }) + if err != nil { + t.Fatalf("Failed to create the executable data %v", err) + } + block, err := InsertBlockParamsToBlock(ethservice.BlockChain().Config(), parent.Header(), *execData) + if err != nil { + t.Fatalf("Failed to convert executable data to block %v", err) + } + newResp, err := api.NewBlock(*execData) + if err != nil || !newResp.Valid { + t.Fatalf("Failed to insert block: %v", err) + } + if ethservice.BlockChain().CurrentBlock().NumberU64() != head { + t.Fatalf("Chain head shouldn't be updated") + } + setResp, err := api.SetHead(block.Hash()) + if err != nil || !setResp.Success { + t.Fatalf("Failed to insert block: %v", err) + } + if ethservice.BlockChain().CurrentBlock().NumberU64() != block.NumberU64() { + t.Fatalf("Chain head should be updated") + } + parent, head = block, block.NumberU64() } } -*/ // startEthService creates a full node instance for testing. func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block) (*node.Node, *eth.Ethereum) { @@ -232,7 +323,7 @@ func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block) t.Fatal("can't create node:", err) } - ethcfg := ðconfig.Config{Genesis: genesis, Ethash: ethash.Config{PowMode: ethash.ModeFake}} + ethcfg := ðconfig.Config{Genesis: genesis, Ethash: ethash.Config{PowMode: ethash.ModeFake}, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256} ethservice, err := eth.New(n, ethcfg) if err != nil { t.Fatal("can't create eth service:", err) diff --git a/eth/catalyst/api_types.go b/eth/catalyst/api_types.go index d5d351a99159..4f6c18ce4217 100644 --- a/eth/catalyst/api_types.go +++ b/eth/catalyst/api_types.go @@ -21,10 +21,10 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" ) -//go:generate go run github.com/fjl/gencodec -type assembleBlockParams -field-override assembleBlockParamsMarshaling -out gen_blockparams.go +//go:generate go run github.com/fjl/gencodec -type AssembleBlockParams -field-override assembleBlockParamsMarshaling -out gen_blockparams.go // Structure described at https://hackmd.io/T9x2mMA4S7us8tJwEB3FDQ -type assembleBlockParams struct { +type AssembleBlockParams struct { ParentHash common.Hash `json:"parentHash" gencodec:"required"` Timestamp uint64 `json:"timestamp" gencodec:"required"` } @@ -34,10 +34,10 @@ type assembleBlockParamsMarshaling struct { Timestamp hexutil.Uint64 } -//go:generate go run github.com/fjl/gencodec -type executableData -field-override executableDataMarshaling -out gen_ed.go +//go:generate go run github.com/fjl/gencodec -type ExecutableData -field-override executableDataMarshaling -out gen_ed.go // Structure described at https://notes.ethereum.org/@n0ble/rayonism-the-merge-spec#Parameters1 -type executableData struct { +type ExecutableData struct { BlockHash common.Hash `json:"blockHash" gencodec:"required"` ParentHash common.Hash `json:"parentHash" gencodec:"required"` Miner common.Address `json:"miner" gencodec:"required"` @@ -61,10 +61,10 @@ type executableDataMarshaling struct { Transactions []hexutil.Bytes } -type newBlockResponse struct { +type NewBlockResponse struct { Valid bool `json:"valid"` } -type genericResponse struct { +type GenericResponse struct { Success bool `json:"success"` } diff --git a/eth/catalyst/gen_blockparams.go b/eth/catalyst/gen_blockparams.go index a9a08ec3a809..4b4182629960 100644 --- a/eth/catalyst/gen_blockparams.go +++ b/eth/catalyst/gen_blockparams.go @@ -13,33 +13,33 @@ import ( var _ = (*assembleBlockParamsMarshaling)(nil) // MarshalJSON marshals as JSON. -func (a assembleBlockParams) MarshalJSON() ([]byte, error) { - type assembleBlockParams struct { +func (a AssembleBlockParams) MarshalJSON() ([]byte, error) { + type AssembleBlockParams struct { ParentHash common.Hash `json:"parentHash" gencodec:"required"` Timestamp hexutil.Uint64 `json:"timestamp" gencodec:"required"` } - var enc assembleBlockParams + var enc AssembleBlockParams enc.ParentHash = a.ParentHash enc.Timestamp = hexutil.Uint64(a.Timestamp) return json.Marshal(&enc) } // UnmarshalJSON unmarshals from JSON. -func (a *assembleBlockParams) UnmarshalJSON(input []byte) error { - type assembleBlockParams struct { +func (a *AssembleBlockParams) UnmarshalJSON(input []byte) error { + type AssembleBlockParams struct { ParentHash *common.Hash `json:"parentHash" gencodec:"required"` Timestamp *hexutil.Uint64 `json:"timestamp" gencodec:"required"` } - var dec assembleBlockParams + var dec AssembleBlockParams if err := json.Unmarshal(input, &dec); err != nil { return err } if dec.ParentHash == nil { - return errors.New("missing required field 'parentHash' for assembleBlockParams") + return errors.New("missing required field 'parentHash' for AssembleBlockParams") } a.ParentHash = *dec.ParentHash if dec.Timestamp == nil { - return errors.New("missing required field 'timestamp' for assembleBlockParams") + return errors.New("missing required field 'timestamp' for AssembleBlockParams") } a.Timestamp = uint64(*dec.Timestamp) return nil diff --git a/eth/catalyst/gen_ed.go b/eth/catalyst/gen_ed.go index 4c2e4c8ead5d..850d8617a5ae 100644 --- a/eth/catalyst/gen_ed.go +++ b/eth/catalyst/gen_ed.go @@ -13,8 +13,8 @@ import ( var _ = (*executableDataMarshaling)(nil) // MarshalJSON marshals as JSON. -func (e executableData) MarshalJSON() ([]byte, error) { - type executableData struct { +func (e ExecutableData) MarshalJSON() ([]byte, error) { + type ExecutableData struct { BlockHash common.Hash `json:"blockHash" gencodec:"required"` ParentHash common.Hash `json:"parentHash" gencodec:"required"` Miner common.Address `json:"miner" gencodec:"required"` @@ -27,7 +27,7 @@ func (e executableData) MarshalJSON() ([]byte, error) { LogsBloom hexutil.Bytes `json:"logsBloom" gencodec:"required"` Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` } - var enc executableData + var enc ExecutableData enc.BlockHash = e.BlockHash enc.ParentHash = e.ParentHash enc.Miner = e.Miner @@ -48,8 +48,8 @@ func (e executableData) MarshalJSON() ([]byte, error) { } // UnmarshalJSON unmarshals from JSON. -func (e *executableData) UnmarshalJSON(input []byte) error { - type executableData struct { +func (e *ExecutableData) UnmarshalJSON(input []byte) error { + type ExecutableData struct { BlockHash *common.Hash `json:"blockHash" gencodec:"required"` ParentHash *common.Hash `json:"parentHash" gencodec:"required"` Miner *common.Address `json:"miner" gencodec:"required"` @@ -62,52 +62,52 @@ func (e *executableData) UnmarshalJSON(input []byte) error { LogsBloom *hexutil.Bytes `json:"logsBloom" gencodec:"required"` Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` } - var dec executableData + var dec ExecutableData if err := json.Unmarshal(input, &dec); err != nil { return err } if dec.BlockHash == nil { - return errors.New("missing required field 'blockHash' for executableData") + return errors.New("missing required field 'blockHash' for ExecutableData") } e.BlockHash = *dec.BlockHash if dec.ParentHash == nil { - return errors.New("missing required field 'parentHash' for executableData") + return errors.New("missing required field 'parentHash' for ExecutableData") } e.ParentHash = *dec.ParentHash if dec.Miner == nil { - return errors.New("missing required field 'miner' for executableData") + return errors.New("missing required field 'miner' for ExecutableData") } e.Miner = *dec.Miner if dec.StateRoot == nil { - return errors.New("missing required field 'stateRoot' for executableData") + return errors.New("missing required field 'stateRoot' for ExecutableData") } e.StateRoot = *dec.StateRoot if dec.Number == nil { - return errors.New("missing required field 'number' for executableData") + return errors.New("missing required field 'number' for ExecutableData") } e.Number = uint64(*dec.Number) if dec.GasLimit == nil { - return errors.New("missing required field 'gasLimit' for executableData") + return errors.New("missing required field 'gasLimit' for ExecutableData") } e.GasLimit = uint64(*dec.GasLimit) if dec.GasUsed == nil { - return errors.New("missing required field 'gasUsed' for executableData") + return errors.New("missing required field 'gasUsed' for ExecutableData") } e.GasUsed = uint64(*dec.GasUsed) if dec.Timestamp == nil { - return errors.New("missing required field 'timestamp' for executableData") + return errors.New("missing required field 'timestamp' for ExecutableData") } e.Timestamp = uint64(*dec.Timestamp) if dec.ReceiptRoot == nil { - return errors.New("missing required field 'receiptsRoot' for executableData") + return errors.New("missing required field 'receiptsRoot' for ExecutableData") } e.ReceiptRoot = *dec.ReceiptRoot if dec.LogsBloom == nil { - return errors.New("missing required field 'logsBloom' for executableData") + return errors.New("missing required field 'logsBloom' for ExecutableData") } e.LogsBloom = *dec.LogsBloom if dec.Transactions == nil { - return errors.New("missing required field 'transactions' for executableData") + return errors.New("missing required field 'transactions' for ExecutableData") } e.Transactions = make([][]byte, len(dec.Transactions)) for k, v := range dec.Transactions { diff --git a/eth/catalyst/sync.go b/eth/catalyst/sync.go new file mode 100644 index 000000000000..61fd1ef9c166 --- /dev/null +++ b/eth/catalyst/sync.go @@ -0,0 +1,74 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package catalyst + +import ( + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" +) + +type syncer struct { + running bool + newBlocks map[common.Hash]*types.Block + lock sync.Mutex +} + +func newSyncer() *syncer { + return &syncer{ + newBlocks: make(map[common.Hash]*types.Block), + } +} + +// onNewBlock is the action for receiving new block event +func (s *syncer) onNewBlock(block *types.Block) { + s.lock.Lock() + defer s.lock.Unlock() + + if s.running { + return + } + s.newBlocks[block.Hash()] = block +} + +func (s *syncer) hasBlock(hash common.Hash) bool { + s.lock.Lock() + defer s.lock.Unlock() + + _, present := s.newBlocks[hash] + return present +} + +// onNewHead is the action for receiving new head event +func (s *syncer) onNewHead(head common.Hash) { + s.lock.Lock() + defer s.lock.Unlock() + + if s.running { + return + } + _, present := s.newBlocks[head] + if !present { + log.Error("Chain head is set with an unknown header") + return + } + s.running = true + + // todo call the SetHead function exposed by the downloader +} diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index a6bf87acbd22..9bfac76f2b23 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -1720,6 +1720,9 @@ func (d *Downloader) importBlockResults(results []*fetchResult) error { for i, result := range results { blocks[i] = types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles) } + // Downloaded blocks are always regarded as trusted after the + // transition. Because the downloaded chain is guided by the + // consensus-layer. if index, err := d.blockchain.InsertChain(blocks); err != nil { if index < len(results) { log.Debug("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "err", err) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 29b47af2598a..bebc78566bb2 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" @@ -207,32 +208,40 @@ type Config struct { } // CreateConsensusEngine creates a consensus engine for the given chain configuration. -func CreateConsensusEngine(stack *node.Node, chainConfig *params.ChainConfig, config *ethash.Config, notify []string, noverify bool, db ethdb.Database) consensus.Engine { +func CreateConsensusEngine(stack *node.Node, chainConfig *params.ChainConfig, config *ethash.Config, notify []string, noverify bool, db ethdb.Database, merger *core.Merger) consensus.Engine { // If proof-of-authority is requested, set it up + var engine consensus.Engine if chainConfig.Clique != nil { - return clique.New(chainConfig.Clique, db) + engine = clique.New(chainConfig.Clique, db) + } else { + switch config.PowMode { + case ethash.ModeFake: + log.Warn("Ethash used in fake mode") + case ethash.ModeTest: + log.Warn("Ethash used in test mode") + case ethash.ModeShared: + log.Warn("Ethash used in shared mode") + } + engine = ethash.New(ethash.Config{ + PowMode: config.PowMode, + CacheDir: stack.ResolvePath(config.CacheDir), + CachesInMem: config.CachesInMem, + CachesOnDisk: config.CachesOnDisk, + CachesLockMmap: config.CachesLockMmap, + DatasetDir: config.DatasetDir, + DatasetsInMem: config.DatasetsInMem, + DatasetsOnDisk: config.DatasetsOnDisk, + DatasetsLockMmap: config.DatasetsLockMmap, + NotifyFull: config.NotifyFull, + }, notify, noverify) + engine.(*ethash.Ethash).SetThreads(-1) // Disable CPU mining } - // Otherwise assume proof-of-work - switch config.PowMode { - case ethash.ModeFake: - log.Warn("Ethash used in fake mode") - case ethash.ModeTest: - log.Warn("Ethash used in test mode") - case ethash.ModeShared: - log.Warn("Ethash used in shared mode") + if merger == nil { + return engine } - engine := ethash.New(ethash.Config{ - PowMode: config.PowMode, - CacheDir: stack.ResolvePath(config.CacheDir), - CachesInMem: config.CachesInMem, - CachesOnDisk: config.CachesOnDisk, - CachesLockMmap: config.CachesLockMmap, - DatasetDir: config.DatasetDir, - DatasetsInMem: config.DatasetsInMem, - DatasetsOnDisk: config.DatasetsOnDisk, - DatasetsLockMmap: config.DatasetsLockMmap, - NotifyFull: config.NotifyFull, - }, notify, noverify) - engine.SetThreads(-1) // Disable CPU mining + engine = beacon.New(engine, merger.LeftPoW()) + merger.SubscribeLeavePoW(func() { + engine.(*beacon.Beacon).MarkTransitioned() + }) return engine } diff --git a/eth/gasprice/gasprice_test.go b/eth/gasprice/gasprice_test.go index 2d394200a0ec..0eaa41dd9d6b 100644 --- a/eth/gasprice/gasprice_test.go +++ b/eth/gasprice/gasprice_test.go @@ -144,7 +144,7 @@ func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBacke // Construct testing chain diskdb := rawdb.NewMemoryDatabase() gspec.Commit(diskdb) - chain, err := core.NewBlockChain(diskdb, &core.CacheConfig{TrieCleanNoPrefetch: true}, &config, engine, vm.Config{}, nil, nil) + chain, err := core.NewBlockChain(diskdb, &core.CacheConfig{TrieCleanNoPrefetch: true}, gspec.Config, engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("Failed to create local chain, %v", err) } diff --git a/eth/handler.go b/eth/handler.go index 41d89c5fb09d..1dcb2559b17d 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -25,6 +25,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/core/types" @@ -79,6 +80,7 @@ type handlerConfig struct { Database ethdb.Database // Database for direct sync insertions Chain *core.BlockChain // Blockchain to serve data from TxPool txPool // Transaction pool to propagate from + Merger *core.Merger // The manager for eth1/2 transition Network uint64 // Network identifier to adfvertise Sync downloader.SyncMode // Whether to fast or full sync BloomCache uint64 // Megabytes to alloc for fast sync bloom @@ -108,6 +110,7 @@ type handler struct { blockFetcher *fetcher.BlockFetcher txFetcher *fetcher.TxFetcher peers *peerSet + merger *core.Merger eventMux *event.TypeMux txsCh chan core.NewTxsEvent @@ -138,6 +141,7 @@ func newHandler(config *handlerConfig) (*handler, error) { txpool: config.TxPool, chain: config.Chain, peers: newPeerSet(), + merger: config.Merger, whitelist: config.Whitelist, quitSync: make(chan struct{}), } @@ -186,12 +190,41 @@ func newHandler(config *handlerConfig) (*handler, error) { // Construct the fetcher (short sync) validator := func(header *types.Header) error { + // All the block fetcher activities should be disabled + // after the transition. Print the warning log. + if h.merger.EnteredPoS() { + log.Warn("Unexpected validation activity", "hash", header.Hash(), "number", header.Number) + return errors.New("unexpected behavior after transition") + } + // Reject all the PoS style headers in the first place. No matter + // the chain has finished the transition or not, the PoS headers + // should only come from the trusted consensus layer instead of + // p2p network. + if beacon, ok := h.chain.Engine().(*beacon.Beacon); ok { + if beacon.IsPostMergeHeader(header) { + return errors.New("unexpected post-merge header") + } + } return h.chain.Engine().VerifyHeader(h.chain, header, true) } heighter := func() uint64 { return h.chain.CurrentBlock().NumberU64() } inserter := func(blocks types.Blocks) (int, error) { + // All the block fetcher activities should be disabled + // after the transition. Print the warning log. + if h.merger.EnteredPoS() { + var ctx []interface{} + ctx = append(ctx, "blocks", len(blocks)) + if len(blocks) > 0 { + ctx = append(ctx, "firsthash", blocks[0].Hash()) + ctx = append(ctx, "firstnumber", blocks[0].Number()) + ctx = append(ctx, "lasthash", blocks[len(blocks)-1].Hash()) + ctx = append(ctx, "lastnumber", blocks[len(blocks)-1].Number()) + } + log.Warn("Unexpected insertion activity", ctx...) + return 0, errors.New("unexpected behavior after transition") + } // If sync hasn't reached the checkpoint yet, deny importing weird blocks. // // Ideally we would also compare the head block's timestamp and similarly reject @@ -211,11 +244,26 @@ func newHandler(config *handlerConfig) (*handler, error) { log.Warn("Fast syncing, discarded propagated block", "number", blocks[0].Number(), "hash", blocks[0].Hash()) return 0, nil } - n, err := h.chain.InsertChain(blocks) - if err == nil { - atomic.StoreUint32(&h.acceptTxs, 1) // Mark initial sync done on any fetcher import + if h.merger.LeftPoW() { + // The blocks from the p2p network is regarded as untrusted + // after the transition. In theory block gossip should be disabled + // entirely whenever the transition is started. But in order to + // handle the transition boundary reorg in the consensus-layer, + // the legacy blocks are still accepted but the chain head won't + // be updated. + for i, block := range blocks { + if err := h.chain.InsertBlock(block); err != nil { + return i, err + } + } + return 0, nil + } else { + n, err := h.chain.InsertChain(blocks) + if err == nil { + atomic.StoreUint32(&h.acceptTxs, 1) // Mark initial sync done on any fetcher import + } + return n, err } - return n, err } h.blockFetcher = fetcher.NewBlockFetcher(false, nil, h.chain.GetBlockByHash, validator, h.BroadcastBlock, heighter, nil, inserter, h.removePeer) @@ -432,6 +480,17 @@ func (h *handler) Stop() { // BroadcastBlock will either propagate a block to a subset of its peers, or // will only announce its availability (depending what's requested). func (h *handler) BroadcastBlock(block *types.Block, propagate bool) { + // Disable the block propagation if the chain has already entered the PoS + // stage. The block propagation is delegated to the consensus layer. + if h.merger.EnteredPoS() { + return + } + // Disable the block propagation if it's the post-merge block. + if beacon, ok := h.chain.Engine().(*beacon.Beacon); ok { + if beacon.IsPostMergeHeader(block.Header()) { + return + } + } hash := block.Hash() peers := h.peers.peersWithoutBlock(hash) diff --git a/eth/handler_eth.go b/eth/handler_eth.go index 3ff9f2245be7..d6e8ac47eda1 100644 --- a/eth/handler_eth.go +++ b/eth/handler_eth.go @@ -180,6 +180,12 @@ func (h *ethHandler) handleBodies(peer *eth.Peer, txs [][]*types.Transaction, un // handleBlockAnnounces is invoked from a peer's message handler when it transmits a // batch of block announcements for the local node to process. func (h *ethHandler) handleBlockAnnounces(peer *eth.Peer, hashes []common.Hash, numbers []uint64) error { + // Drop all incoming block announces from the p2p network if + // the chain already entered the pos stage. TODO perhaps we + // should return the error here to disconnect the legacy node. + if h.merger.EnteredPoS() { + return nil + } // Schedule all the unknown hashes for retrieval var ( unknownHashes = make([]common.Hash, 0, len(hashes)) @@ -200,6 +206,12 @@ func (h *ethHandler) handleBlockAnnounces(peer *eth.Peer, hashes []common.Hash, // handleBlockBroadcast is invoked from a peer's message handler when it transmits a // block broadcast for the local node to process. func (h *ethHandler) handleBlockBroadcast(peer *eth.Peer, block *types.Block, td *big.Int) error { + // Drop all incoming block announces from the p2p network if + // the chain already entered the pos stage. TODO perhaps we + // should return the error here to disconnect the legacy node. + if h.merger.EnteredPoS() { + return nil + } // Schedule the block for import h.blockFetcher.Enqueue(peer.ID(), block) diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index b8db5039c93b..01e7762897e1 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -105,8 +105,8 @@ func testForkIDSplit(t *testing.T, protocol uint) { genesisNoFork = gspecNoFork.MustCommit(dbNoFork) genesisProFork = gspecProFork.MustCommit(dbProFork) - chainNoFork, _ = core.NewBlockChain(dbNoFork, nil, configNoFork, engine, vm.Config{}, nil, nil) - chainProFork, _ = core.NewBlockChain(dbProFork, nil, configProFork, engine, vm.Config{}, nil, nil) + chainNoFork, _ = core.NewBlockChain(dbNoFork, nil, configNoFork, engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) + chainProFork, _ = core.NewBlockChain(dbProFork, nil, configProFork, engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) blocksNoFork, _ = core.GenerateChain(configNoFork, genesisNoFork, engine, dbNoFork, 2, nil) blocksProFork, _ = core.GenerateChain(configProFork, genesisProFork, engine, dbProFork, 2, nil) @@ -115,6 +115,7 @@ func testForkIDSplit(t *testing.T, protocol uint) { Database: dbNoFork, Chain: chainNoFork, TxPool: newTestTxPool(), + Merger: core.NewMerger(rawdb.NewMemoryDatabase()), Network: 1, Sync: downloader.FullSync, BloomCache: 1, @@ -123,6 +124,7 @@ func testForkIDSplit(t *testing.T, protocol uint) { Database: dbProFork, Chain: chainProFork, TxPool: newTestTxPool(), + Merger: core.NewMerger(rawdb.NewMemoryDatabase()), Network: 1, Sync: downloader.FullSync, BloomCache: 1, diff --git a/eth/handler_test.go b/eth/handler_test.go index b2f00b7977aa..56313d7bb146 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -137,7 +137,7 @@ func newTestHandlerWithBlocks(blocks int) *testHandler { Alloc: core.GenesisAlloc{testAddr: {Balance: big.NewInt(1000000)}}, }).MustCommit(db) - chain, _ := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) + chain, _ := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) bs, _ := core.GenerateChain(params.TestChainConfig, chain.Genesis(), ethash.NewFaker(), db, blocks, nil) if _, err := chain.InsertChain(bs); err != nil { @@ -149,6 +149,7 @@ func newTestHandlerWithBlocks(blocks int) *testHandler { Database: db, Chain: chain, TxPool: txpool, + Merger: core.NewMerger(rawdb.NewMemoryDatabase()), Network: 1, Sync: downloader.FastSync, BloomCache: 1, diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index 66f013409694..0a4f1cbd84b1 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -69,7 +69,7 @@ func newTestBackendWithGenerator(blocks int, generator func(int, *core.BlockGen) Alloc: core.GenesisAlloc{testAddr: {Balance: big.NewInt(100_000_000_000_000_000)}}, }).MustCommit(db) - chain, _ := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) + chain, _ := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) bs, _ := core.GenerateChain(params.TestChainConfig, chain.Genesis(), ethash.NewFaker(), db, blocks, generator) if _, err := chain.InsertChain(bs); err != nil { diff --git a/eth/state_accessor.go b/eth/state_accessor.go index ca2002b60dee..aaf6f6c6054b 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -30,12 +30,12 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -// stateAtBlock retrieves the state database associated with a certain block. +// StateAtBlock retrieves the state database associated with a certain block. // If no state is locally available for the given block, a number of blocks // are attempted to be reexecuted to generate the desired state. The optional // base layer statedb can be passed then it's regarded as the statedb of the // parent block. -func (eth *Ethereum) stateAtBlock(block *types.Block, reexec uint64, base *state.StateDB, checkLive bool) (statedb *state.StateDB, err error) { +func (eth *Ethereum) StateAtBlock(block *types.Block, reexec uint64, base *state.StateDB, checkLive bool) (statedb *state.StateDB, err error) { var ( current *types.Block database state.Database @@ -152,7 +152,7 @@ func (eth *Ethereum) stateAtTransaction(block *types.Block, txIndex int, reexec } // Lookup the statedb of parent block from the live database, // otherwise regenerate it on the flight. - statedb, err := eth.stateAtBlock(parent, reexec, nil, true) + statedb, err := eth.StateAtBlock(parent, reexec, nil, true) if err != nil { return nil, vm.BlockContext{}, nil, err } diff --git a/eth/sync.go b/eth/sync.go index aaac6bef9054..8902ba3d041f 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -145,7 +145,10 @@ func (cs *chainSyncer) nextSyncOp() *chainSyncOp { if cs.doneCh != nil { return nil // Sync already running. } - + // Disable the td based sync trigger after the transition + if cs.handler.merger.LeftPoW() { + return nil + } // Ensure we're at minimum peer count. minPeers := defaultMinSyncPeers if cs.forced { diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index 9afd59d596bc..3f3a460e03b3 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -81,7 +81,7 @@ func newTestBackend(t *testing.T, n int, gspec *core.Genesis, generator func(i i SnapshotLimit: 0, TrieDirtyDisabled: true, // Archive mode } - chain, err := core.NewBlockChain(backend.chaindb, cacheConfig, backend.chainConfig, backend.engine, vm.Config{}, nil, nil) + chain, err := core.NewBlockChain(backend.chaindb, cacheConfig, backend.chainConfig, backend.engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } diff --git a/les/client.go b/les/client.go index 93319cb936b7..12f4174f86d1 100644 --- a/les/client.go +++ b/les/client.go @@ -63,6 +63,7 @@ type LightEthereum struct { serverPool *vfc.ServerPool serverPoolIterator enode.Iterator pruner *pruner + merger *core.Merger bloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests bloomIndexer *core.ChainIndexer // Bloom indexer operating during block imports @@ -95,6 +96,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { log.Info("Initialised chain configuration", "config", chainConfig) peers := newServerPeerSet() + merger := core.NewMerger(chainDb) leth := &LightEthereum{ lesCommons: lesCommons{ genesis: genesisHash, @@ -109,7 +111,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { eventMux: stack.EventMux(), reqDist: newRequestDistributor(peers, &mclock.System{}), accountManager: stack.AccountManager(), - engine: ethconfig.CreateConsensusEngine(stack, chainConfig, &config.Ethash, nil, false, chainDb), + merger: merger, + engine: ethconfig.CreateConsensusEngine(stack, chainConfig, &config.Ethash, nil, false, chainDb, merger), bloomRequests: make(chan chan *bloombits.Retrieval), bloomIndexer: core.NewBloomIndexer(chainDb, params.BloomBitsBlocksClient, params.HelperTrieConfirmations), p2pServer: stack.Server(), @@ -138,7 +141,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { } // Note: NewLightChain adds the trusted checkpoint so it needs an ODR with // indexers already set but not started yet - if leth.blockchain, err = light.NewLightChain(leth.odr, leth.chainConfig, leth.engine, checkpoint); err != nil { + if leth.blockchain, err = light.NewLightChain(leth.odr, leth.chainConfig, leth.engine, checkpoint, merger); err != nil { return nil, err } leth.chainReader = leth.blockchain @@ -332,6 +335,7 @@ func (s *LightEthereum) Engine() consensus.Engine { return s.engine } func (s *LightEthereum) LesVersion() int { return int(ClientProtocolVersions[0]) } func (s *LightEthereum) Downloader() *downloader.Downloader { return s.handler.downloader } func (s *LightEthereum) EventMux() *event.TypeMux { return s.eventMux } +func (s *LightEthereum) Merger() *core.Merger { return s.merger } // Protocols returns all the currently configured network protocols to start. func (s *LightEthereum) Protocols() []p2p.Protocol { diff --git a/les/client_handler.go b/les/client_handler.go index 9583bd57ca05..de8bdf92dd8d 100644 --- a/les/client_handler.go +++ b/les/client_handler.go @@ -143,11 +143,13 @@ func (h *clientHandler) handle(p *serverPeer, noInitAnnounce bool) error { connectionTimer.Update(time.Duration(mclock.Now() - connectedAt)) serverConnectionGauge.Update(int64(h.backend.peers.len())) }() - // It's mainly used in testing which requires discarding initial - // signal to prevent syncing. - if !noInitAnnounce { + + // Discard all the announces after the transition + // Also discarding initial signal to prevent syncing during testing. + if !h.backend.merger.LeftPoW() || !noInitAnnounce { h.fetcher.announce(p, &announceData{Hash: p.headInfo.Hash, Number: p.headInfo.Number, Td: p.headInfo.Td}) } + // Mark the peer starts to be served. atomic.StoreUint32(&p.serving, 1) defer atomic.StoreUint32(&p.serving, 0) @@ -212,7 +214,11 @@ func (h *clientHandler) handleMsg(p *serverPeer) error { // Update peer head information first and then notify the announcement p.updateHead(req.Hash, req.Number, req.Td) - h.fetcher.announce(p, &req) + + // Discard all the announces after the transition + if !h.backend.merger.LeftPoW() { + h.fetcher.announce(p, &req) + } } case msg.Code == BlockHeadersMsg: p.Log().Trace("Received block header response message") diff --git a/les/fetcher.go b/les/fetcher.go index d944d32858e7..bfe3aa16ec27 100644 --- a/les/fetcher.go +++ b/les/fetcher.go @@ -71,8 +71,8 @@ type fetcherPeer struct { // These following two fields can track the latest announces // from the peer with limited size for caching. We hold the // assumption that all enqueued announces are td-monotonic. - announces map[common.Hash]*announce // Announcement map - announcesList []common.Hash // FIFO announces list + announces map[common.Hash]*announce // Announcement map + fifo []common.Hash // FIFO announces list } // addAnno enqueues an new trusted announcement. If the queued announces overflow, @@ -87,15 +87,15 @@ func (fp *fetcherPeer) addAnno(anno *announce) { return } fp.announces[hash] = anno - fp.announcesList = append(fp.announcesList, hash) + fp.fifo = append(fp.fifo, hash) // Evict oldest if the announces are oversized. - if len(fp.announcesList)-cachedAnnosThreshold > 0 { - for i := 0; i < len(fp.announcesList)-cachedAnnosThreshold; i++ { - delete(fp.announces, fp.announcesList[i]) + if len(fp.fifo)-cachedAnnosThreshold > 0 { + for i := 0; i < len(fp.fifo)-cachedAnnosThreshold; i++ { + delete(fp.announces, fp.fifo[i]) } - copy(fp.announcesList, fp.announcesList[len(fp.announcesList)-cachedAnnosThreshold:]) - fp.announcesList = fp.announcesList[:cachedAnnosThreshold] + copy(fp.fifo, fp.fifo[len(fp.fifo)-cachedAnnosThreshold:]) + fp.fifo = fp.fifo[:cachedAnnosThreshold] } } @@ -106,8 +106,8 @@ func (fp *fetcherPeer) forwardAnno(td *big.Int) []*announce { cutset int evicted []*announce ) - for ; cutset < len(fp.announcesList); cutset++ { - anno := fp.announces[fp.announcesList[cutset]] + for ; cutset < len(fp.fifo); cutset++ { + anno := fp.announces[fp.fifo[cutset]] if anno == nil { continue // In theory it should never ever happen } @@ -118,8 +118,8 @@ func (fp *fetcherPeer) forwardAnno(td *big.Int) []*announce { delete(fp.announces, anno.data.Hash) } if cutset > 0 { - copy(fp.announcesList, fp.announcesList[cutset:]) - fp.announcesList = fp.announcesList[:len(fp.announcesList)-cutset] + copy(fp.fifo, fp.fifo[cutset:]) + fp.fifo = fp.fifo[:len(fp.fifo)-cutset] } return evicted } diff --git a/les/test_helper.go b/les/test_helper.go index 21d0f191c98a..ea5502395174 100644 --- a/les/test_helper.go +++ b/les/test_helper.go @@ -202,7 +202,7 @@ func newTestClientHandler(backend *backends.SimulatedBackend, odr *LesOdr, index oracle *checkpointoracle.CheckpointOracle ) genesis := gspec.MustCommit(db) - chain, _ := light.NewLightChain(odr, gspec.Config, engine, nil) + chain, _ := light.NewLightChain(odr, gspec.Config, engine, nil, core.NewMerger(rawdb.NewMemoryDatabase())) if indexers != nil { checkpointConfig := ¶ms.CheckpointOracleConfig{ Address: crypto.CreateAddress(bankAddr, 0), @@ -239,6 +239,7 @@ func newTestClientHandler(backend *backends.SimulatedBackend, odr *LesOdr, index engine: engine, blockchain: chain, eventMux: evmux, + merger: core.NewMerger(rawdb.NewMemoryDatabase()), } client.handler = newClientHandler(ulcServers, ulcFraction, nil, client) diff --git a/light/lightchain.go b/light/lightchain.go index c481734ffe97..114aec657792 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -59,6 +59,7 @@ type LightChain struct { chainHeadFeed event.Feed scope event.SubscriptionScope genesisBlock *types.Block + forker *core.ForkChoice bodyCache *lru.Cache // Cache for the most recent block bodies bodyRLPCache *lru.Cache // Cache for the most recent block bodies in RLP encoded format @@ -77,7 +78,7 @@ type LightChain struct { // NewLightChain returns a fully initialised light chain using information // available in the database. It initialises the default Ethereum header // validator. -func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus.Engine, checkpoint *params.TrustedCheckpoint) (*LightChain, error) { +func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus.Engine, checkpoint *params.TrustedCheckpoint, merger *core.Merger) (*LightChain, error) { bodyCache, _ := lru.New(bodyCacheLimit) bodyRLPCache, _ := lru.New(bodyCacheLimit) blockCache, _ := lru.New(blockCacheLimit) @@ -92,6 +93,10 @@ func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus. blockCache: blockCache, engine: engine, } + bc.forker = core.NewForkChoice(bc, merger.LeftPoW(), nil) + merger.SubscribeLeavePoW(func() { + bc.forker.MarkTransitioned() + }) var err error bc.hc, err = core.NewHeaderChain(odr.Database(), config, bc.engine, bc.getProcInterrupt) if err != nil { @@ -369,6 +374,42 @@ func (lc *LightChain) postChainEvents(events []interface{}) { } } +func (lc *LightChain) InsertHeader(header *types.Header) error { + // Verify the header first before obtaing the lock + headers := []*types.Header{header} + if _, err := lc.hc.ValidateHeaderChain(headers, 100); err != nil { + return err + } + // Make sure only one thread manipulates the chain at once + lc.chainmu.Lock() + defer lc.chainmu.Unlock() + + lc.wg.Add(1) + defer lc.wg.Done() + + _, err := lc.hc.WriteHeaders(headers) + log.Info("Inserted header", "number", header.Number, "hash", header.Hash()) + return err +} + +func (lc *LightChain) SetChainHead(header *types.Header) error { + lc.wg.Add(1) + defer lc.wg.Done() + + lc.chainmu.Lock() + defer lc.chainmu.Unlock() + + if err := lc.hc.Reorg([]*types.Header{header}); err != nil { + return err + } + // Emit events + block := types.NewBlockWithHeader(header) + lc.chainFeed.Send(core.ChainEvent{Block: block, Hash: block.Hash()}) + lc.chainHeadFeed.Send(core.ChainHeadEvent{Block: block}) + log.Info("Set the chain head", "number", block.Number(), "hash", block.Hash()) + return nil +} + // InsertHeaderChain attempts to insert the given header chain in to the local // chain, possibly creating a reorg. If an error is returned, it will return the // index number of the failing header as well an error describing what went wrong. @@ -396,25 +437,23 @@ func (lc *LightChain) InsertHeaderChain(chain []*types.Header, checkFreq int) (i lc.wg.Add(1) defer lc.wg.Done() - status, err := lc.hc.InsertHeaderChain(chain, start) + status, err := lc.hc.InsertHeaderChain(chain, start, lc.forker) if err != nil || len(chain) == 0 { return 0, err } // Create chain event for the new head block of this insertion. var ( - events = make([]interface{}, 0, 1) lastHeader = chain[len(chain)-1] block = types.NewBlockWithHeader(lastHeader) ) switch status { case core.CanonStatTy: - events = append(events, core.ChainEvent{Block: block, Hash: block.Hash()}) + lc.chainFeed.Send(core.ChainEvent{Block: block, Hash: block.Hash()}) + lc.chainHeadFeed.Send(core.ChainHeadEvent{Block: block}) case core.SideStatTy: - events = append(events, core.ChainSideEvent{Block: block}) + lc.chainSideFeed.Send(core.ChainSideEvent{Block: block}) } - lc.postChainEvents(events) - return 0, err } diff --git a/light/lightchain_test.go b/light/lightchain_test.go index 8600e56345f6..d6a20823436f 100644 --- a/light/lightchain_test.go +++ b/light/lightchain_test.go @@ -56,7 +56,7 @@ func newCanonical(n int) (ethdb.Database, *LightChain, error) { db := rawdb.NewMemoryDatabase() gspec := core.Genesis{Config: params.TestChainConfig} genesis := gspec.MustCommit(db) - blockchain, _ := NewLightChain(&dummyOdr{db: db, indexerConfig: TestClientIndexerConfig}, gspec.Config, ethash.NewFaker(), nil) + blockchain, _ := NewLightChain(&dummyOdr{db: db, indexerConfig: TestClientIndexerConfig}, gspec.Config, ethash.NewFaker(), nil, core.NewMerger(rawdb.NewMemoryDatabase())) // Create and inject the requested chain if n == 0 { @@ -76,7 +76,7 @@ func newTestLightChain() *LightChain { Config: params.TestChainConfig, } gspec.MustCommit(db) - lc, err := NewLightChain(&dummyOdr{db: db}, gspec.Config, ethash.NewFullFaker(), nil) + lc, err := NewLightChain(&dummyOdr{db: db}, gspec.Config, ethash.NewFullFaker(), nil, core.NewMerger(rawdb.NewMemoryDatabase())) if err != nil { panic(err) } @@ -347,7 +347,7 @@ func TestReorgBadHeaderHashes(t *testing.T) { defer func() { delete(core.BadHashes, headers[3].Hash()) }() // Create a new LightChain and check that it rolled back the state. - ncm, err := NewLightChain(&dummyOdr{db: bc.chainDb}, params.TestChainConfig, ethash.NewFaker(), nil) + ncm, err := NewLightChain(&dummyOdr{db: bc.chainDb}, params.TestChainConfig, ethash.NewFaker(), nil, core.NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("failed to create new chain manager: %v", err) } diff --git a/light/odr_test.go b/light/odr_test.go index fdf657a82ec5..85c2ba86ba0a 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -261,14 +261,14 @@ func testChainOdr(t *testing.T, protocol int, fn odrTestFn) { ) gspec.MustCommit(ldb) // Assemble the test environment - blockchain, _ := core.NewBlockChain(sdb, nil, params.TestChainConfig, ethash.NewFullFaker(), vm.Config{}, nil, nil) + blockchain, _ := core.NewBlockChain(sdb, nil, params.TestChainConfig, ethash.NewFullFaker(), vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) gchain, _ := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), sdb, 4, testChainGen) if _, err := blockchain.InsertChain(gchain); err != nil { t.Fatal(err) } odr := &testOdr{sdb: sdb, ldb: ldb, indexerConfig: TestClientIndexerConfig} - lightchain, err := NewLightChain(odr, params.TestChainConfig, ethash.NewFullFaker(), nil) + lightchain, err := NewLightChain(odr, params.TestChainConfig, ethash.NewFullFaker(), nil, core.NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatal(err) } diff --git a/light/trie_test.go b/light/trie_test.go index e8294cc2a235..1b1d49bcd22f 100644 --- a/light/trie_test.go +++ b/light/trie_test.go @@ -44,7 +44,7 @@ func TestNodeIterator(t *testing.T) { genesis = gspec.MustCommit(fulldb) ) gspec.MustCommit(lightdb) - blockchain, _ := core.NewBlockChain(fulldb, nil, params.TestChainConfig, ethash.NewFullFaker(), vm.Config{}, nil, nil) + blockchain, _ := core.NewBlockChain(fulldb, nil, params.TestChainConfig, ethash.NewFullFaker(), vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) gchain, _ := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), fulldb, 4, testChainGen) if _, err := blockchain.InsertChain(gchain); err != nil { panic(err) diff --git a/light/txpool_test.go b/light/txpool_test.go index cc2651d29ae5..ee1a26df0d2f 100644 --- a/light/txpool_test.go +++ b/light/txpool_test.go @@ -91,7 +91,7 @@ func TestTxPool(t *testing.T) { ) gspec.MustCommit(ldb) // Assemble the test environment - blockchain, _ := core.NewBlockChain(sdb, nil, params.TestChainConfig, ethash.NewFullFaker(), vm.Config{}, nil, nil) + blockchain, _ := core.NewBlockChain(sdb, nil, params.TestChainConfig, ethash.NewFullFaker(), vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) gchain, _ := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), sdb, poolTestBlocks, txPoolTestChainGen) if _, err := blockchain.InsertChain(gchain); err != nil { panic(err) @@ -103,7 +103,7 @@ func TestTxPool(t *testing.T) { discard: make(chan int, 1), mined: make(chan int, 1), } - lightchain, _ := NewLightChain(odr, params.TestChainConfig, ethash.NewFullFaker(), nil) + lightchain, _ := NewLightChain(odr, params.TestChainConfig, ethash.NewFullFaker(), nil, core.NewMerger(rawdb.NewMemoryDatabase())) txPermanent = 50 pool := NewTxPool(params.TestChainConfig, lightchain, relay) ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) diff --git a/miner/miner.go b/miner/miner.go index 1c33b3bd286f..96bc7215b322 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -68,7 +68,7 @@ type Miner struct { wg sync.WaitGroup } -func New(eth Backend, config *Config, chainConfig *params.ChainConfig, mux *event.TypeMux, engine consensus.Engine, isLocalBlock func(block *types.Block) bool) *Miner { +func New(eth Backend, config *Config, chainConfig *params.ChainConfig, mux *event.TypeMux, engine consensus.Engine, isLocalBlock func(header *types.Header) bool, merger *core.Merger) *Miner { miner := &Miner{ eth: eth, mux: mux, @@ -76,7 +76,7 @@ func New(eth Backend, config *Config, chainConfig *params.ChainConfig, mux *even exitCh: make(chan struct{}), startCh: make(chan common.Address), stopCh: make(chan struct{}), - worker: newWorker(config, chainConfig, engine, eth, mux, isLocalBlock, true), + worker: newWorker(config, chainConfig, engine, eth, mux, isLocalBlock, true, merger), } miner.wg.Add(1) go miner.update() diff --git a/miner/miner_test.go b/miner/miner_test.go index da1e472dbd76..69d5c551fa05 100644 --- a/miner/miner_test.go +++ b/miner/miner_test.go @@ -245,7 +245,7 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux) { // Create consensus engine engine := clique.New(chainConfig.Clique, chainDB) // Create Ethereum backend - bc, err := core.NewBlockChain(chainDB, nil, chainConfig, engine, vm.Config{}, nil, nil) + bc, err := core.NewBlockChain(chainDB, nil, chainConfig, engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) if err != nil { t.Fatalf("can't create new chain %v", err) } @@ -257,5 +257,5 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux) { // Create event Mux mux := new(event.TypeMux) // Create Miner - return New(backend, &config, chainConfig, mux, engine, nil), mux + return New(backend, &config, chainConfig, mux, engine, nil, core.NewMerger(rawdb.NewMemoryDatabase())), mux } diff --git a/miner/stress/beacon/main.go b/miner/stress/beacon/main.go new file mode 100644 index 000000000000..cb5b6b56a1a7 --- /dev/null +++ b/miner/stress/beacon/main.go @@ -0,0 +1,489 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// This file contains a miner stress test for the eth1/2 transition +package main + +import ( + "crypto/ecdsa" + "errors" + "io/ioutil" + "math/big" + "math/rand" + "os" + "path/filepath" + "time" + + "github.com/ethereum/go-ethereum/accounts/keystore" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/fdlimit" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/catalyst" + "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/les" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/miner" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/params" +) + +type nodetype int + +const ( + legacyMiningNode nodetype = iota + legacyNormalNode + eth2MiningNode + eth2NormalNode + eth2LightClient +) + +var ( + // transitionDifficulty is the target total difficulty for transition + transitionDifficulty = new(big.Int).Mul(big.NewInt(0), params.MinimumDifficulty) + + // blockInterval is the time interval for creating a new eth2 block + blockInterval = time.Second * 3 + blockIntervalInt = 3 + + // finalizationDist is the block distance for finalizing block + finalizationDist = 10 +) + +type ethNode struct { + typ nodetype + api *catalyst.ConsensusAPI + ethBackend *eth.Ethereum + lesBackend *les.LightEthereum + stack *node.Node + enode *enode.Node +} + +func newNode(typ nodetype, genesis *core.Genesis, enodes []*enode.Node) *ethNode { + var ( + err error + api *catalyst.ConsensusAPI + stack *node.Node + ethBackend *eth.Ethereum + lesBackend *les.LightEthereum + ) + // Start the node and wait until it's up + if typ == eth2LightClient { + stack, lesBackend, api, err = makeLightNode(genesis) + } else { + stack, ethBackend, api, err = makeFullNode(genesis) + } + if err != nil { + panic(err) + } + for stack.Server().NodeInfo().Ports.Listener == 0 { + time.Sleep(250 * time.Millisecond) + } + // Connect the node to all the previous ones + for _, n := range enodes { + stack.Server().AddPeer(n) + } + enode := stack.Server().Self() + + // Inject the signer key and start sealing with it + store := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) + if _, err := store.NewAccount(""); err != nil { + panic(err) + } + return ðNode{ + typ: typ, + api: api, + ethBackend: ethBackend, + lesBackend: lesBackend, + stack: stack, + enode: enode, + } +} + +func (n *ethNode) assembleBlock(parentHash common.Hash, parentTimestamp uint64) (*catalyst.ExecutableData, error) { + if n.typ != eth2MiningNode { + return nil, errors.New("invalid node type") + } + return n.api.AssembleBlock(catalyst.AssembleBlockParams{ + ParentHash: parentHash, + Timestamp: uint64(time.Now().Unix()), + }) +} + +func (n *ethNode) insertBlock(eb catalyst.ExecutableData) error { + if !eth2types(n.typ) { + return errors.New("invalid node type") + } + response, err := n.api.NewBlock(eb) + if err != nil { + return err + } + if !response.Valid { + return errors.New("failed to insert block") + } + return nil +} + +func (n *ethNode) insertBlockAndSetHead(parent *types.Header, ed catalyst.ExecutableData) error { + if !eth2types(n.typ) { + return errors.New("invalid node type") + } + if err := n.insertBlock(ed); err != nil { + return err + } + var config *params.ChainConfig + if n.typ == eth2LightClient { + config = n.lesBackend.BlockChain().Config() + } else { + config = n.ethBackend.BlockChain().Config() + } + block, err := catalyst.InsertBlockParamsToBlock(config, parent, ed) + if err != nil { + return err + } + response, err := n.api.SetHead(block.Hash()) + if err != nil { + return err + } + if !response.Success { + return errors.New("failed to set head") + } + return nil +} + +type nodeManager struct { + genesis *core.Genesis + genesisBlock *types.Block + nodes []*ethNode + enodes []*enode.Node + close chan struct{} +} + +func newNodeManager(genesis *core.Genesis) *nodeManager { + return &nodeManager{ + close: make(chan struct{}), + genesis: genesis, + genesisBlock: genesis.ToBlock(nil), + } +} + +func (mgr *nodeManager) createNode(typ nodetype) { + node := newNode(typ, mgr.genesis, mgr.enodes) + mgr.nodes = append(mgr.nodes, node) + mgr.enodes = append(mgr.enodes, node.enode) +} + +func (mgr *nodeManager) getNodes(typ nodetype) []*ethNode { + var ret []*ethNode + for _, node := range mgr.nodes { + if node.typ == typ { + ret = append(ret, node) + } + } + return ret +} + +func (mgr *nodeManager) startMining() { + for _, node := range append(mgr.getNodes(eth2MiningNode), mgr.getNodes(legacyMiningNode)...) { + if err := node.ethBackend.StartMining(1); err != nil { + panic(err) + } + } +} + +func (mgr *nodeManager) shutdown() { + close(mgr.close) + for _, node := range mgr.nodes { + node.stack.Close() + } +} + +func (mgr *nodeManager) run() { + if len(mgr.nodes) == 0 { + return + } + chain := mgr.nodes[0].ethBackend.BlockChain() + sink := make(chan core.ChainHeadEvent, 1024) + sub := chain.SubscribeChainHeadEvent(sink) + defer sub.Unsubscribe() + + var ( + transitioned bool + parentBlock *types.Block + waitFinalise []*types.Block + ) + timer := time.NewTimer(0) + defer timer.Stop() + <-timer.C // discard the initial tick + + // Handle the by default transition. + if transitionDifficulty.Sign() == 0 { + transitioned = true + parentBlock = mgr.genesisBlock + timer.Reset(blockInterval) + log.Info("Enable the transition by default") + } + + // Handle the block finalization. + checkFinalise := func() { + if parentBlock == nil { + return + } + if len(waitFinalise) == 0 { + return + } + oldest := waitFinalise[0] + if oldest.NumberU64() > parentBlock.NumberU64() { + return + } + distance := parentBlock.NumberU64() - oldest.NumberU64() + if int(distance) < finalizationDist { + return + } + nodes := mgr.getNodes(eth2MiningNode) + nodes = append(nodes, mgr.getNodes(eth2NormalNode)...) + nodes = append(nodes, mgr.getNodes(eth2LightClient)...) + for _, node := range append(nodes) { + node.api.FinalizeBlock(oldest.Hash()) + } + log.Info("Finalised eth2 block", "number", oldest.NumberU64(), "hash", oldest.Hash()) + waitFinalise = waitFinalise[1:] + } + + for { + checkFinalise() + select { + case <-mgr.close: + return + + case ev := <-sink: + if transitioned { + continue + } + td := chain.GetTd(ev.Block.Hash(), ev.Block.NumberU64()) + if td.Cmp(transitionDifficulty) < 0 { + continue + } + transitioned, parentBlock = true, ev.Block + timer.Reset(blockInterval) + log.Info("Transition difficulty reached", "td", td, "target", transitionDifficulty) + + case <-timer.C: + producers := mgr.getNodes(eth2MiningNode) + if len(producers) == 0 { + continue + } + hash, timestamp := parentBlock.Hash(), parentBlock.Time() + if parentBlock.NumberU64() == 0 { + timestamp = uint64(time.Now().Unix()) - uint64(blockIntervalInt) + } + ed, err := producers[0].assembleBlock(hash, timestamp) + if err != nil { + log.Error("Failed to assemble the block", "err", err) + continue + } + block, _ := catalyst.InsertBlockParamsToBlock(chain.Config(), parentBlock.Header(), *ed) + + nodes := mgr.getNodes(eth2MiningNode) + nodes = append(nodes, mgr.getNodes(eth2NormalNode)...) + nodes = append(nodes, mgr.getNodes(eth2LightClient)...) + + for _, node := range nodes { + if err := node.insertBlockAndSetHead(parentBlock.Header(), *ed); err != nil { + log.Error("Failed to insert block", "err", err) + } + } + log.Info("Create and insert eth2 block", "number", ed.Number) + parentBlock = block + waitFinalise = append(waitFinalise, block) + timer.Reset(blockInterval) + } + } +} + +func main() { + log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + fdlimit.Raise(2048) + + // Generate a batch of accounts to seal and fund with + faucets := make([]*ecdsa.PrivateKey, 16) + for i := 0; i < len(faucets); i++ { + faucets[i], _ = crypto.GenerateKey() + } + // Pre-generate the ethash mining DAG so we don't race + ethash.MakeDataset(1, filepath.Join(os.Getenv("HOME"), ".ethash")) + + // Create an Ethash network based off of the Ropsten config + genesis := makeGenesis(faucets) + manager := newNodeManager(genesis) + defer manager.shutdown() + + manager.createNode(eth2NormalNode) + manager.createNode(eth2MiningNode) + manager.createNode(legacyMiningNode) + manager.createNode(legacyNormalNode) + manager.createNode(eth2LightClient) + + // Iterate over all the nodes and start mining + time.Sleep(3 * time.Second) + if transitionDifficulty.Sign() != 0 { + manager.startMining() + } + go manager.run() + + // Start injecting transactions from the faucets like crazy + time.Sleep(3 * time.Second) + nonces := make([]uint64, len(faucets)) + for { + // Pick a random mining node + nodes := manager.getNodes(eth2MiningNode) + + index := rand.Intn(len(faucets)) + node := nodes[index%len(nodes)] + + // Create a self transaction and inject into the pool + tx, err := types.SignTx(types.NewTransaction(nonces[index], crypto.PubkeyToAddress(faucets[index].PublicKey), new(big.Int), 21000, big.NewInt(100000000000+rand.Int63n(65536)), nil), types.HomesteadSigner{}, faucets[index]) + if err != nil { + panic(err) + } + if err := node.ethBackend.TxPool().AddLocal(tx); err != nil { + panic(err) + } + nonces[index]++ + + // Wait if we're too saturated + if pend, _ := node.ethBackend.TxPool().Stats(); pend > 2048 { + time.Sleep(100 * time.Millisecond) + } + } +} + +// makeGenesis creates a custom Ethash genesis block based on some pre-defined +// faucet accounts. +func makeGenesis(faucets []*ecdsa.PrivateKey) *core.Genesis { + genesis := core.DefaultRopstenGenesisBlock() + genesis.Difficulty = params.MinimumDifficulty + genesis.GasLimit = 25000000 + + genesis.Config.ChainID = big.NewInt(18) + genesis.Config.EIP150Hash = common.Hash{} + genesis.BaseFee = big.NewInt(params.InitialBaseFee) + + genesis.Alloc = core.GenesisAlloc{} + for _, faucet := range faucets { + genesis.Alloc[crypto.PubkeyToAddress(faucet.PublicKey)] = core.GenesisAccount{ + Balance: new(big.Int).Exp(big.NewInt(2), big.NewInt(128), nil), + } + } + return genesis +} + +func makeFullNode(genesis *core.Genesis) (*node.Node, *eth.Ethereum, *catalyst.ConsensusAPI, error) { + // Define the basic configurations for the Ethereum node + datadir, _ := ioutil.TempDir("", "") + + config := &node.Config{ + Name: "geth", + Version: params.Version, + DataDir: datadir, + P2P: p2p.Config{ + ListenAddr: "0.0.0.0:0", + NoDiscovery: true, + MaxPeers: 25, + }, + UseLightweightKDF: true, + } + // Create the node and configure a full Ethereum node on it + stack, err := node.New(config) + if err != nil { + return nil, nil, nil, err + } + ethBackend, err := eth.New(stack, ðconfig.Config{ + Genesis: genesis, + NetworkId: genesis.Config.ChainID.Uint64(), + SyncMode: downloader.FullSync, + DatabaseCache: 256, + DatabaseHandles: 256, + TxPool: core.DefaultTxPoolConfig, + GPO: ethconfig.Defaults.GPO, + Ethash: ethconfig.Defaults.Ethash, + Miner: miner.Config{ + GasFloor: genesis.GasLimit * 9 / 10, + GasCeil: genesis.GasLimit * 11 / 10, + GasPrice: big.NewInt(1), + Recommit: 10 * time.Second, // Disable the recommit + }, + LightServ: 100, + LightPeers: 10, + LightNoSyncServe: true, + }) + if err != nil { + return nil, nil, nil, err + } + err = stack.Start() + return stack, ethBackend, catalyst.NewConsensusAPI(ethBackend, nil), err +} + +func makeLightNode(genesis *core.Genesis) (*node.Node, *les.LightEthereum, *catalyst.ConsensusAPI, error) { + // Define the basic configurations for the Ethereum node + datadir, _ := ioutil.TempDir("", "") + + config := &node.Config{ + Name: "geth", + Version: params.Version, + DataDir: datadir, + P2P: p2p.Config{ + ListenAddr: "0.0.0.0:0", + NoDiscovery: true, + MaxPeers: 25, + }, + UseLightweightKDF: true, + } + // Create the node and configure a full Ethereum node on it + stack, err := node.New(config) + if err != nil { + return nil, nil, nil, err + } + lesBackend, err := les.New(stack, ðconfig.Config{ + Genesis: genesis, + NetworkId: genesis.Config.ChainID.Uint64(), + SyncMode: downloader.LightSync, + DatabaseCache: 256, + DatabaseHandles: 256, + TxPool: core.DefaultTxPoolConfig, + GPO: ethconfig.Defaults.GPO, + Ethash: ethconfig.Defaults.Ethash, + LightPeers: 10, + }) + if err != nil { + return nil, nil, nil, err + } + err = stack.Start() + return stack, lesBackend, catalyst.NewConsensusAPI(nil, lesBackend), err +} + +func eth2types(typ nodetype) bool { + if typ == eth2LightClient || typ == eth2NormalNode || typ == eth2MiningNode { + return true + } + return false +} diff --git a/miner/worker.go b/miner/worker.go index 4ef2c8c0da38..dcc823f3afb7 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -128,6 +128,7 @@ type worker struct { engine consensus.Engine eth Backend chain *core.BlockChain + merger *core.Merger // Feeds pendingLogsFeed event.Feed @@ -181,7 +182,7 @@ type worker struct { noempty uint32 // External functions - isLocalBlock func(block *types.Block) bool // Function used to determine whether the specified block is mined by local miner. + isLocalBlock func(header *types.Header) bool // Function used to determine whether the specified block is mined by local miner. // Test hooks newTaskHook func(*task) // Method to call upon receiving a new sealing task. @@ -190,7 +191,7 @@ type worker struct { resubmitHook func(time.Duration, time.Duration) // Method to call upon updating resubmitting interval. } -func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux, isLocalBlock func(*types.Block) bool, init bool) *worker { +func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux, isLocalBlock func(header *types.Header) bool, init bool, merger *core.Merger) *worker { worker := &worker{ config: config, chainConfig: chainConfig, @@ -198,6 +199,7 @@ func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus eth: eth, mux: mux, chain: eth.BlockChain(), + merger: merger, isLocalBlock: isLocalBlock, localUncles: make(map[common.Hash]*types.Block), remoteUncles: make(map[common.Hash]*types.Block), @@ -472,7 +474,7 @@ func (w *worker) mainLoop() { continue } // Add side block to possible uncle block set depending on the author. - if w.isLocalBlock != nil && w.isLocalBlock(ev.Block) { + if w.isLocalBlock != nil && w.isLocalBlock(ev.Block.Header()) { w.localUncles[ev.Block.Hash()] = ev.Block } else { w.remoteUncles[ev.Block.Hash()] = ev.Block @@ -654,7 +656,7 @@ func (w *worker) resultLoop() { logs = append(logs, receipt.Logs...) } // Commit block and state to database. - _, err := w.chain.WriteBlockWithState(block, receipts, logs, task.state, true) + _, err := w.chain.WriteBlockAndSetHead(block, receipts, logs, task.state, true) if err != nil { log.Error("Failed writing block to chain", "err", err) continue @@ -1034,7 +1036,7 @@ func (w *worker) commit(uncles []*types.Header, interval func(), update bool, st if err != nil { return err } - if w.isRunning() { + if w.isRunning() && !w.merger.LeftPoW() { if interval != nil { interval() } diff --git a/miner/worker_test.go b/miner/worker_test.go index 5b35c66dcea8..3b4484adfd2a 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -135,7 +135,7 @@ func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine } genesis := gspec.MustCommit(db) - chain, _ := core.NewBlockChain(db, &core.CacheConfig{TrieDirtyDisabled: true}, gspec.Config, engine, vm.Config{}, nil, nil) + chain, _ := core.NewBlockChain(db, &core.CacheConfig{TrieDirtyDisabled: true}, gspec.Config, engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) txpool := core.NewTxPool(testTxPoolConfig, chainConfig, chain) // Generate a small n-block chain and an uncle block for it @@ -197,7 +197,7 @@ func (b *testWorkerBackend) newRandomTx(creation bool) *types.Transaction { func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, db ethdb.Database, blocks int) (*worker, *testWorkerBackend) { backend := newTestWorkerBackend(t, chainConfig, engine, db, blocks) backend.txPool.AddLocals(pendingTxs) - w := newWorker(testConfig, chainConfig, engine, backend, new(event.TypeMux), nil, false) + w := newWorker(testConfig, chainConfig, engine, backend, new(event.TypeMux), nil, false, core.NewMerger(rawdb.NewMemoryDatabase())) w.setEtherbase(testBankAddress) return w, backend } @@ -232,7 +232,7 @@ func testGenerateBlockAndImport(t *testing.T, isClique bool) { // This test chain imports the mined blocks. db2 := rawdb.NewMemoryDatabase() b.genesis.MustCommit(db2) - chain, _ := core.NewBlockChain(db2, nil, b.chain.Config(), engine, vm.Config{}, nil, nil) + chain, _ := core.NewBlockChain(db2, nil, b.chain.Config(), engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) defer chain.Stop() // Ignore empty commit here for less noise. diff --git a/tests/block_test_util.go b/tests/block_test_util.go index bcf861e09b9c..e3daab9b5d22 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -127,7 +127,7 @@ func (t *BlockTest) Run(snapshotter bool) error { cache.SnapshotLimit = 1 cache.SnapshotWait = true } - chain, err := core.NewBlockChain(db, cache, config, engine, vm.Config{}, nil, nil) + chain, err := core.NewBlockChain(db, cache, config, engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) if err != nil { return err } diff --git a/tests/fuzzers/les/les-fuzzer.go b/tests/fuzzers/les/les-fuzzer.go index 3e1017187345..4aeee43a6656 100644 --- a/tests/fuzzers/les/les-fuzzer.go +++ b/tests/fuzzers/les/les-fuzzer.go @@ -80,7 +80,7 @@ func makechain() (bc *core.BlockChain, addrHashes, txHashes []common.Hash) { addrHashes = append(addrHashes, crypto.Keccak256Hash(addr[:])) txHashes = append(txHashes, tx.Hash()) }) - bc, _ = core.NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + bc, _ = core.NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) if _, err := bc.InsertChain(blocks); err != nil { panic(err) } From ed1c9ceedcc1f8e4cd017e120e3636bac768c044 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Mon, 26 Jul 2021 19:08:48 +0800 Subject: [PATCH 02/32] consensus/beacon, eth: change beacon difficulty to 0 --- consensus/beacon/consensus.go | 47 ++++++++++++++--------------------- eth/catalyst/api.go | 2 +- eth/handler.go | 4 +-- 3 files changed, 21 insertions(+), 32 deletions(-) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index 878cb60fa2c2..ab94cc135d74 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -33,7 +33,7 @@ import ( // Proof-of-stake protocol constants. var ( - beaconDifficulty = common.Big1 // The default block difficulty in the beacon consensus + beaconDifficulty = common.Big0 // The default block difficulty in the beacon consensus beaconNonce = types.EncodeNonce(0) // The default block nonce in the beacon consensus ) @@ -61,8 +61,8 @@ type Beacon struct { ethone consensus.Engine // Classic consensus engine used in the eth1, ethash or clique // transitioned is the flag whether the transition has been triggered. - // It's triggered by receiving the first "NewHead" message from the - // external consensus engine. + // It's triggered by receiving the first "POS_CHAINHEAD_SET" message + // from the external consensus engine. transitioned bool lock sync.RWMutex } @@ -81,7 +81,7 @@ func New(ethone consensus.Engine, transitioned bool) *Beacon { // Author implements consensus.Engine, returning the header's coinbase as the // verified author of the block. func (beacon *Beacon) Author(header *types.Header) (common.Address, error) { - if !beacon.IsPostMergeHeader(header) { + if !beacon.IsPoSHeader(header) { return beacon.ethone.Author(header) } return header.Coinbase, nil @@ -90,7 +90,7 @@ func (beacon *Beacon) Author(header *types.Header) (common.Address, error) { // VerifyHeader checks whether a header conforms to the consensus rules of the // stock Ethereum consensus engine. func (beacon *Beacon) VerifyHeader(chain consensus.ChainHeaderReader, header *types.Header, seal bool) error { - if !beacon.IsPostMergeHeader(header) { + if !beacon.IsPoSHeader(header) { return beacon.ethone.VerifyHeader(chain, header, seal) } // Short circuit if the header is known, or its parent not @@ -110,7 +110,7 @@ func (beacon *Beacon) VerifyHeader(chain consensus.ChainHeaderReader, header *ty // concurrently. The method returns a quit channel to abort the operations and // a results channel to retrieve the async verifications. func (beacon *Beacon) VerifyHeaders(chain consensus.ChainHeaderReader, headers []*types.Header, seals []bool) (chan<- struct{}, <-chan error) { - if !beacon.IsPostMergeHeader(headers[len(headers)-1]) { + if !beacon.IsPoSHeader(headers[len(headers)-1]) { return beacon.ethone.VerifyHeaders(chain, headers, seals) } var ( @@ -119,7 +119,7 @@ func (beacon *Beacon) VerifyHeaders(chain consensus.ChainHeaderReader, headers [ preSeals []bool ) for index, header := range headers { - if beacon.IsPostMergeHeader(header) { + if beacon.IsPoSHeader(header) { preHeaders = headers[:index] postHeaders = headers[index:] preSeals = seals[:index] @@ -171,7 +171,7 @@ func (beacon *Beacon) VerifyHeaders(chain consensus.ChainHeaderReader, headers [ // VerifyUncles verifies that the given block's uncles conform to the consensus // rules of the stock Ethereum consensus engine. func (beacon *Beacon) VerifyUncles(chain consensus.ChainReader, block *types.Block) error { - if !beacon.IsPostMergeHeader(block.Header()) { + if !beacon.IsPoSHeader(block.Header()) { return beacon.ethone.VerifyUncles(chain, block) } // Verify that there is no uncle block. It's explicitly disabled in the beacon @@ -295,7 +295,7 @@ func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types. // Finalize is different with Prepare, it can be used in both block generation // and verification. So determine the consensus rules by header type. - if !beacon.IsPostMergeHeader(header) { + if !beacon.IsPoSHeader(header) { beacon.ethone.Finalize(chain, header, state, txs, uncles) return } @@ -312,7 +312,7 @@ func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea // FinalizeAndAssemble is different with Prepare, it can be used in both block // generation and verification. So determine the consensus rules by header type. - if !beacon.IsPostMergeHeader(header) { + if !beacon.IsPoSHeader(header) { return beacon.ethone.FinalizeAndAssemble(chain, header, state, txs, uncles, receipts) } // Finalize and assemble the block @@ -329,7 +329,7 @@ func (beacon *Beacon) Seal(chain consensus.ChainHeaderReader, block *types.Block beacon.lock.RLock() defer beacon.lock.RUnlock() - if !beacon.IsPostMergeHeader(block.Header()) { + if !beacon.IsPoSHeader(block.Header()) { return beacon.ethone.Seal(chain, block, results, stop) } // The seal verification is done by the external consensus engine, @@ -368,25 +368,14 @@ func (beacon *Beacon) Close() error { return beacon.ethone.Close() } -// IsPostMergeHeader reports the header belongs to the PoS-stage with some special fields. -// This function is not suitable for a part of APIs like Prepare or CalcDifficulty because -// the header difficulty is not set yet. -func (beacon *Beacon) IsPostMergeHeader(header *types.Header) bool { - // These fields can be used to filter out ethash block - if header.Difficulty.Cmp(beaconDifficulty) != 0 { - return false - } - if header.MixDigest != (common.Hash{}) { - return false - } - if header.Nonce != beaconNonce { - return false - } - // Extra field can be used to filter out clique block - if len(header.Extra) != 0 { - return false +// IsPoSHeader reports the header belongs to the PoS-stage with some special fields. +// This function is not suitable for a part of APIs like Prepare or CalcDifficulty +// because the header difficulty is not set yet. +func (beacon *Beacon) IsPoSHeader(header *types.Header) bool { + if header.Difficulty == nil { + return false // we should never enter here. } - return true + return header.Difficulty.Cmp(beaconDifficulty) == 0 } // MarkTransitioned sets the transitioned flag. diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index e6b40416e3fc..158fa56e6e9f 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -308,7 +308,7 @@ func InsertBlockParamsToBlock(config *chainParams.ChainConfig, parent *types.Hea TxHash: types.DeriveSha(types.Transactions(txs), trie.NewStackTrie(nil)), ReceiptHash: params.ReceiptRoot, Bloom: types.BytesToBloom(params.LogsBloom), - Difficulty: big.NewInt(1), + Difficulty: common.Big0, Number: number, GasLimit: params.GasLimit, GasUsed: params.GasUsed, diff --git a/eth/handler.go b/eth/handler.go index 1dcb2559b17d..2a7051b6be59 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -201,7 +201,7 @@ func newHandler(config *handlerConfig) (*handler, error) { // should only come from the trusted consensus layer instead of // p2p network. if beacon, ok := h.chain.Engine().(*beacon.Beacon); ok { - if beacon.IsPostMergeHeader(header) { + if beacon.IsPoSHeader(header) { return errors.New("unexpected post-merge header") } } @@ -487,7 +487,7 @@ func (h *handler) BroadcastBlock(block *types.Block, propagate bool) { } // Disable the block propagation if it's the post-merge block. if beacon, ok := h.chain.Engine().(*beacon.Beacon); ok { - if beacon.IsPostMergeHeader(block.Header()) { + if beacon.IsPoSHeader(block.Header()) { return } } From e234c251684aa15c400057e13536dfdf71668e51 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Fri, 30 Jul 2021 10:36:09 +0800 Subject: [PATCH 03/32] eth: updates --- eth/handler.go | 15 +++++++-------- eth/handler_eth.go | 12 ++++++------ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/eth/handler.go b/eth/handler.go index 2a7051b6be59..f86e3a2f0f84 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -249,21 +249,20 @@ func newHandler(config *handlerConfig) (*handler, error) { // after the transition. In theory block gossip should be disabled // entirely whenever the transition is started. But in order to // handle the transition boundary reorg in the consensus-layer, - // the legacy blocks are still accepted but the chain head won't - // be updated. + // the legacy blocks are still accepted, but only for the terminal + // pow blocks. Spec: https://github.com/ethereum/EIPs/blob/master/EIPS/eip-3675.md#halt-the-importing-of-pow-blocks for i, block := range blocks { if err := h.chain.InsertBlock(block); err != nil { return i, err } } return 0, nil - } else { - n, err := h.chain.InsertChain(blocks) - if err == nil { - atomic.StoreUint32(&h.acceptTxs, 1) // Mark initial sync done on any fetcher import - } - return n, err } + n, err := h.chain.InsertChain(blocks) + if err == nil { + atomic.StoreUint32(&h.acceptTxs, 1) // Mark initial sync done on any fetcher import + } + return n, err } h.blockFetcher = fetcher.NewBlockFetcher(false, nil, h.chain.GetBlockByHash, validator, h.BroadcastBlock, heighter, nil, inserter, h.removePeer) diff --git a/eth/handler_eth.go b/eth/handler_eth.go index d6e8ac47eda1..1788a5849c1b 100644 --- a/eth/handler_eth.go +++ b/eth/handler_eth.go @@ -181,10 +181,10 @@ func (h *ethHandler) handleBodies(peer *eth.Peer, txs [][]*types.Transaction, un // batch of block announcements for the local node to process. func (h *ethHandler) handleBlockAnnounces(peer *eth.Peer, hashes []common.Hash, numbers []uint64) error { // Drop all incoming block announces from the p2p network if - // the chain already entered the pos stage. TODO perhaps we - // should return the error here to disconnect the legacy node. + // the chain already entered the pos stage and disconnect the + // remote peer. if h.merger.EnteredPoS() { - return nil + return errors.New("unexpected block announces") } // Schedule all the unknown hashes for retrieval var ( @@ -207,10 +207,10 @@ func (h *ethHandler) handleBlockAnnounces(peer *eth.Peer, hashes []common.Hash, // block broadcast for the local node to process. func (h *ethHandler) handleBlockBroadcast(peer *eth.Peer, block *types.Block, td *big.Int) error { // Drop all incoming block announces from the p2p network if - // the chain already entered the pos stage. TODO perhaps we - // should return the error here to disconnect the legacy node. + // the chain already entered the pos stage and disconnect the + // remote peer. if h.merger.EnteredPoS() { - return nil + return errors.New("unexpected block announces") } // Schedule the block for import h.blockFetcher.Enqueue(peer.ID(), block) From e3a3374549401a5813a27d64f90afceba8ab29c9 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Fri, 30 Jul 2021 14:16:45 +0800 Subject: [PATCH 04/32] all: add terminalBlockDifficulty config, fix rebasing issues --- consensus/beacon/consensus.go | 7 +++---- core/blockchain.go | 6 ++++-- core/state_processor_test.go | 2 +- eth/catalyst/api_test.go | 27 +++++++++++++-------------- eth/handler.go | 9 +++++++++ les/client_handler.go | 4 +++- miner/stress/beacon/main.go | 33 ++++++++++++++++++++++++++++----- 7 files changed, 61 insertions(+), 27 deletions(-) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index ab94cc135d74..6126e2571fcb 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -52,7 +52,7 @@ var ( // Beacon is a consensus engine combines the ethereum 1 consensus and proof-of-stake // algorithm. There is a special flag inside to decide whether to use legacy consensus // rules or new rules. The transition rule is described in the eth1/2 merge spec. -// https://hackmd.io/@n0ble/ethereum_consensus_upgrade_mainnet_perspective#Transition-process +// https://github.com/ethereum/EIPs/blob/master/EIPS/eip-3675.md // // The beacon here is a half-functional consensus engine with partial functions which // is only used for necessary consensus checks. The legacy consensus engine can be any @@ -78,8 +78,7 @@ func New(ethone consensus.Engine, transitioned bool) *Beacon { } } -// Author implements consensus.Engine, returning the header's coinbase as the -// verified author of the block. +// Author implements consensus.Engine, returning the verified author of the block. func (beacon *Beacon) Author(header *types.Header) (common.Address, error) { if !beacon.IsPoSHeader(header) { return beacon.ethone.Author(header) @@ -182,7 +181,7 @@ func (beacon *Beacon) VerifyUncles(chain consensus.ChainReader, block *types.Blo } // verifyHeader checks whether a header conforms to the consensus rules of the -// stock Ethereum consensus engine. The difference between the beacon and ethash is +// stock Ethereum consensus engine. The difference between the beacon and classic is // (a) the difficulty, mixhash, nonce, extradata and unclehash are expected // to be the desired constants // (b) the timestamp is not verified anymore diff --git a/core/blockchain.go b/core/blockchain.go index a2553737a560..fbd4381601c9 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -239,6 +239,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par Preimages: cacheConfig.Preimages, }), quit: make(chan struct{}), + chainmu: syncx.NewClosableMutex(), bodyCache: bodyCache, bodyRLPCache: bodyRLPCache, receiptsCache: receiptsCache, @@ -2105,8 +2106,8 @@ func (bc *BlockChain) insertBlock(block *types.Block) error { } // If the header is a banned one, straight out abort if BadHashes[block.Hash()] { - bc.reportBlock(block, nil, ErrBlacklistedHash) - return ErrBlacklistedHash + bc.reportBlock(block, nil, ErrBannedHash) + return ErrBannedHash } // Retrieve the parent block and it's state to execute on top start := time.Now() @@ -2197,6 +2198,7 @@ func (bc *BlockChain) SetChainHead(newBlock *types.Block) error { func (bc *BlockChain) update() { futureTimer := time.NewTicker(5 * time.Second) defer futureTimer.Stop() + defer bc.wg.Done() for { select { case <-futureTimer.C: diff --git a/core/state_processor_test.go b/core/state_processor_test.go index ee493416f456..62f1737ddd39 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -260,7 +260,7 @@ func TestStateProcessorErrors(t *testing.T) { }, } genesis = gspec.MustCommit(db) - blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(nil)) + blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) ) defer blockchain.Stop() for i, tt := range []struct { diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index a3d7c41e697e..1753fdaaba0c 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -75,20 +75,19 @@ func generateTestChainWithFork(n int, fork int) (*core.Genesis, []*types.Block, } db := rawdb.NewMemoryDatabase() config := ¶ms.ChainConfig{ - ChainID: big.NewInt(1337), - HomesteadBlock: big.NewInt(0), - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - ByzantiumBlock: big.NewInt(0), - ConstantinopleBlock: big.NewInt(0), - PetersburgBlock: big.NewInt(0), - IstanbulBlock: big.NewInt(0), - MuirGlacierBlock: big.NewInt(0), - BerlinBlock: big.NewInt(0), - LondonBlock: big.NewInt(0), - TerminalTotalDifficulty: big.NewInt(0), - Ethash: new(params.EthashConfig), + ChainID: big.NewInt(1337), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + Ethash: new(params.EthashConfig), } genesis := &core.Genesis{ Config: config, diff --git a/eth/handler.go b/eth/handler.go index f86e3a2f0f84..96fe3ab872cc 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -252,6 +252,15 @@ func newHandler(config *handlerConfig) (*handler, error) { // the legacy blocks are still accepted, but only for the terminal // pow blocks. Spec: https://github.com/ethereum/EIPs/blob/master/EIPS/eip-3675.md#halt-the-importing-of-pow-blocks for i, block := range blocks { + ptd := h.chain.GetTd(block.ParentHash(), block.NumberU64()-1) + if ptd == nil { + return 0, nil + } + td := new(big.Int).Add(ptd, block.Difficulty()) + if !h.chain.Config().IsTerminalPoWBlock(ptd, td) { + log.Info("Filtered out non-termimal pow block", "number", block.NumberU64(), "hash", block.Hash()) + return 0, nil + } if err := h.chain.InsertBlock(block); err != nil { return i, err } diff --git a/les/client_handler.go b/les/client_handler.go index de8bdf92dd8d..3125869f6a0c 100644 --- a/les/client_handler.go +++ b/les/client_handler.go @@ -18,6 +18,7 @@ package les import ( "context" + "fmt" "math/big" "math/rand" "sync" @@ -146,7 +147,8 @@ func (h *clientHandler) handle(p *serverPeer, noInitAnnounce bool) error { // Discard all the announces after the transition // Also discarding initial signal to prevent syncing during testing. - if !h.backend.merger.LeftPoW() || !noInitAnnounce { + if !(noInitAnnounce || h.backend.merger.LeftPoW()) { + fmt.Print("adf") h.fetcher.announce(p, &announceData{Hash: p.headInfo.Hash, Number: p.headInfo.Number, Td: p.headInfo.Td}) } diff --git a/miner/stress/beacon/main.go b/miner/stress/beacon/main.go index cb5b6b56a1a7..30e6e8ee044d 100644 --- a/miner/stress/beacon/main.go +++ b/miner/stress/beacon/main.go @@ -57,9 +57,26 @@ const ( eth2LightClient ) +func (typ nodetype) String() string { + switch typ { + case legacyMiningNode: + return "legacyMiningNode" + case legacyNormalNode: + return "legacyNormalNode" + case eth2MiningNode: + return "eth2MiningNode" + case eth2NormalNode: + return "eth2NormalNode" + case eth2LightClient: + return "eth2LightClient" + default: + return "undefined" + } +} + var ( // transitionDifficulty is the target total difficulty for transition - transitionDifficulty = new(big.Int).Mul(big.NewInt(0), params.MinimumDifficulty) + transitionDifficulty = new(big.Int).Mul(big.NewInt(20), params.MinimumDifficulty) // blockInterval is the time interval for creating a new eth2 block blockInterval = time.Second * 3 @@ -285,7 +302,7 @@ func (mgr *nodeManager) run() { } transitioned, parentBlock = true, ev.Block timer.Reset(blockInterval) - log.Info("Transition difficulty reached", "td", td, "target", transitionDifficulty) + log.Info("Transition difficulty reached", "td", td, "target", transitionDifficulty, "number", ev.Block.NumberU64(), "hash", ev.Block.Hash()) case <-timer.C: producers := mgr.getNodes(eth2MiningNode) @@ -309,7 +326,7 @@ func (mgr *nodeManager) run() { for _, node := range nodes { if err := node.insertBlockAndSetHead(parentBlock.Header(), *ed); err != nil { - log.Error("Failed to insert block", "err", err) + log.Error("Failed to insert block", "type", node.typ, "err", err) } } log.Info("Create and insert eth2 block", "number", ed.Number) @@ -387,6 +404,7 @@ func makeGenesis(faucets []*ecdsa.PrivateKey) *core.Genesis { genesis.Config.ChainID = big.NewInt(18) genesis.Config.EIP150Hash = common.Hash{} genesis.BaseFee = big.NewInt(params.InitialBaseFee) + genesis.Config.TerminalTotalDifficulty = transitionDifficulty genesis.Alloc = core.GenesisAlloc{} for _, faucet := range faucets { @@ -417,7 +435,7 @@ func makeFullNode(genesis *core.Genesis) (*node.Node, *eth.Ethereum, *catalyst.C if err != nil { return nil, nil, nil, err } - ethBackend, err := eth.New(stack, ðconfig.Config{ + econfig := ðconfig.Config{ Genesis: genesis, NetworkId: genesis.Config.ChainID.Uint64(), SyncMode: downloader.FullSync, @@ -435,10 +453,15 @@ func makeFullNode(genesis *core.Genesis) (*node.Node, *eth.Ethereum, *catalyst.C LightServ: 100, LightPeers: 10, LightNoSyncServe: true, - }) + } + ethBackend, err := eth.New(stack, econfig) if err != nil { return nil, nil, nil, err } + _, err = les.NewLesServer(stack, ethBackend, econfig) + if err != nil { + log.Crit("Failed to create the LES server", "err", err) + } err = stack.Start() return stack, ethBackend, catalyst.NewConsensusAPI(ethBackend, nil), err } From da9c5a4924754e8aad7ca033a6a7392f4c1af3a7 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 20 Sep 2021 13:48:03 +0200 Subject: [PATCH 05/32] eth: implemented merge interop spec --- consensus/beacon/consensus.go | 15 +- core/blockchain.go | 7 +- core/chain_makers.go | 2 +- core/headerchain.go | 2 +- core/types/block.go | 6 + eth/catalyst/api.go | 283 +++++++++++++++++++++++--------- eth/catalyst/api_test.go | 125 +++++++++++--- eth/catalyst/api_types.go | 63 ++++--- eth/catalyst/gen_blockparams.go | 22 ++- eth/catalyst/gen_ed.go | 91 ++++++---- les/client_handler.go | 2 - light/lightchain.go | 8 +- miner/stress/beacon/main.go | 25 +-- rpc/errors.go | 10 ++ 14 files changed, 465 insertions(+), 196 deletions(-) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index 6126e2571fcb..20981ca3e56e 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -42,14 +42,13 @@ var ( // codebase, inherently breaking if the engine is swapped out. Please put common // error types into the consensus package. var ( - errTooManyUncles = errors.New("too many uncles") - errInvalidDifficulty = errors.New("invalid difficulty") - errInvalidMixDigest = errors.New("invalid mix digest") - errInvalidNonce = errors.New("invalid nonce") - errInvalidUncleHash = errors.New("invalid uncle hash") + errTooManyUncles = errors.New("too many uncles") + errInvalidMixDigest = errors.New("invalid mix digest") + errInvalidNonce = errors.New("invalid nonce") + errInvalidUncleHash = errors.New("invalid uncle hash") ) -// Beacon is a consensus engine combines the ethereum 1 consensus and proof-of-stake +// Beacon is a consensus engine combines the eth1 consensus and proof-of-stake // algorithm. There is a special flag inside to decide whether to use legacy consensus // rules or new rules. The transition rule is described in the eth1/2 merge spec. // https://github.com/ethereum/EIPs/blob/master/EIPS/eip-3675.md @@ -58,7 +57,7 @@ var ( // is only used for necessary consensus checks. The legacy consensus engine can be any // engine implements the consensus interface(except the beacon itself). type Beacon struct { - ethone consensus.Engine // Classic consensus engine used in the eth1, ethash or clique + ethone consensus.Engine // Classic consensus engine used in eth1, e.g. ethash or clique // transitioned is the flag whether the transition has been triggered. // It's triggered by receiving the first "POS_CHAINHEAD_SET" message @@ -67,7 +66,7 @@ type Beacon struct { lock sync.RWMutex } -// New creates a consensus engine with the given embedded ethereum 1 engine. +// New creates a consensus engine with the given embedded eth1 engine. func New(ethone consensus.Engine, transitioned bool) *Beacon { if _, ok := ethone.(*Beacon); ok { panic("nested consensus engine") diff --git a/core/blockchain.go b/core/blockchain.go index fbd4381601c9..7354e358da57 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -2058,11 +2058,10 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { return nil } -// InsertBlock accepts a single block and the given consensus engine as the -// parameters. It will firstly execute the block, run the necessary verfication +// InsertBlock executes the block, runs the necessary verification // upon it and then persist the block and the associate state into the database. // The key difference between the InsertChain is it won't do the canonical chain -// udpating. It relays on the additional SetChainHead call to finalize the entire +// updating. It relies on the additional SetChainHead call to finalize the entire // procedure. func (bc *BlockChain) InsertBlock(block *types.Block) error { if !bc.chainmu.TryLock() { @@ -2162,7 +2161,7 @@ func (bc *BlockChain) insertBlock(block *types.Block) error { blockWriteTimer.Update(time.Since(substart) - statedb.AccountCommits - statedb.StorageCommits - statedb.SnapshotCommits) blockInsertTimer.UpdateSince(start) - log.Info("Inserted block", "number", block.Number(), "hash", block.Hash(), "txs", len(block.Transactions()), "elasped", common.PrettyDuration(time.Since(start))) + log.Info("Inserted block", "number", block.Number(), "hash", block.Hash(), "txs", len(block.Transactions()), "elapsed", common.PrettyDuration(time.Since(start))) return nil } diff --git a/core/chain_makers.go b/core/chain_makers.go index 5316f0d84e01..403e64b34c31 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -206,7 +206,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse b.header = makeHeader(chainreader, parent, statedb, b.engine) // Set the difficulty for clique block. The chain maker doesn't have access - // to a chain, so the difficulty will be lets unset (nil). Set it here to the + // to a chain, so the difficulty will be left unset (nil). Set it here to the // correct value. if b.header.Difficulty == nil { b.header.Difficulty = big.NewInt(2) diff --git a/core/headerchain.go b/core/headerchain.go index 6975c9b2a032..4c01afda7410 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -197,7 +197,7 @@ func (hc *HeaderChain) Reorg(headers []*types.Header) error { // WriteHeaders writes a chain of headers into the local chain, given that the // parents are already known. The chain head header won't be updated in this -// function, the addtional setChainHead is expected in order to finish the entire +// function, the additional setChainHead is expected in order to finish the entire // procedure. func (hc *HeaderChain) WriteHeaders(headers []*types.Header) (int, error) { if len(headers) == 0 { diff --git a/core/types/block.go b/core/types/block.go index 360f1eb47c2b..92e5cb77275a 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -85,6 +85,12 @@ type Header struct { // BaseFee was added by EIP-1559 and is ignored in legacy headers. BaseFee *big.Int `json:"baseFeePerGas" rlp:"optional"` + + /* + TODO (MariusVanDerWijden) Add this field once needed + // Random was added during the merge and contains the BeaconState randomness + Random common.Hash `json:"random" rlp:"optional"` + */ } // field type overrides for gencodec diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 158fa56e6e9f..12ab85954bb1 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -21,9 +21,11 @@ import ( "errors" "fmt" "math/big" + "os" "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/consensus/misc" @@ -35,16 +37,25 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" chainParams "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/trie" ) +var ( + VALID = GenericStringResponse{"VALID"} + INVALID = GenericStringResponse{"INVALID"} + SYNCING = GenericStringResponse{"SYNCING"} + UnknownHeader = rpc.CustomError{Code: 4, Message: "unknown header"} + UnknownPayload = rpc.CustomError{Code: 5, Message: "unknown payload"} +) + // Register adds catalyst APIs to the full node. func Register(stack *node.Node, backend *eth.Ethereum) error { log.Warn("Catalyst mode enabled", "protocol", "eth") stack.RegisterAPIs([]rpc.API{ { - Namespace: "consensus", + Namespace: "engine", Version: "1.0", Service: NewConsensusAPI(backend, nil), Public: true, @@ -58,7 +69,7 @@ func RegisterLight(stack *node.Node, backend *les.LightEthereum) error { log.Warn("Catalyst mode enabled", "protocol", "les") stack.RegisterAPIs([]rpc.API{ { - Namespace: "consensus", + Namespace: "engine", Version: "1.0", Service: NewConsensusAPI(nil, backend), Public: true, @@ -68,22 +79,29 @@ func RegisterLight(stack *node.Node, backend *les.LightEthereum) error { } type ConsensusAPI struct { - light bool - eth *eth.Ethereum - les *les.LightEthereum - engine consensus.Engine // engine is the post-merge consensus engine, only for block creation - syncer *syncer // syncer is responsible for triggering chain sync + light bool + eth *eth.Ethereum + les *les.LightEthereum + engine consensus.Engine // engine is the post-merge consensus engine, only for block creation + syncer *syncer // syncer is responsible for triggering chain sync + preparedBlocks map[int]*ExecutableData } func NewConsensusAPI(eth *eth.Ethereum, les *les.LightEthereum) *ConsensusAPI { var engine consensus.Engine if eth == nil { + if les.BlockChain().Config().TerminalTotalDifficulty == nil { + panic("Catalyst started without valid total difficulty") + } if b, ok := les.Engine().(*beacon.Beacon); ok { engine = beacon.New(b.InnerEngine(), true) } else { engine = beacon.New(les.Engine(), true) } } else { + if eth.BlockChain().Config().TerminalTotalDifficulty == nil { + panic("Catalyst started without valid total difficulty") + } if b, ok := eth.Engine().(*beacon.Beacon); ok { engine = beacon.New(b.InnerEngine(), true) } else { @@ -91,11 +109,12 @@ func NewConsensusAPI(eth *eth.Ethereum, les *les.LightEthereum) *ConsensusAPI { } } return &ConsensusAPI{ - light: eth == nil, - eth: eth, - les: les, - engine: engine, - syncer: newSyncer(), + light: eth == nil, + eth: eth, + les: les, + engine: engine, + syncer: newSyncer(), + preparedBlocks: make(map[int]*ExecutableData), } } @@ -155,9 +174,90 @@ func (api *ConsensusAPI) makeEnv(parent *types.Block, header *types.Header) (*bl return env, nil } +func (api *ConsensusAPI) PreparePayload(params AssembleBlockParams) (hexutil.Uint64, error) { + data, err := api.assembleBlock(params) + if err != nil { + return hexutil.Uint64(0), err + } + id := len(api.preparedBlocks) + api.preparedBlocks[id] = data + return hexutil.Uint64(id), nil +} + +func (api *ConsensusAPI) GetPayload(PayloadID hexutil.Uint64) (*ExecutableData, error) { + data, ok := api.preparedBlocks[int(PayloadID)] + if !ok { + return nil, &UnknownPayload + } + return data, nil +} + +// ConsensusValidated is called to mark a block as valid, so +// that data that is no longer needed can be removed. +func (api *ConsensusAPI) ConsensusValidated(params ConsensusValidatedParams) error { + switch params.Status { + case VALID.Status: + // Finalize the transition if it's the first `FinalisedBlock` event. + merger := api.merger() + if !merger.EnteredPoS() { + merger.EnterPoS() + } + return nil //api.setHead(params.BlockHash) + case INVALID.Status: + // TODO (MariusVanDerWijden) delete the block from the bc + return nil + default: + return errors.New("invalid params.status") + } +} + +func (api *ConsensusAPI) ForkchoiceUpdated(params ForkChoiceParams) error { + return api.setHead(params.FinalizedBlockHash) +} + +// ExecutePayload creates an Eth1 block, inserts it in the chain, and returns the status of the chain. +func (api *ConsensusAPI) ExecutePayload(params ExecutableData) (GenericStringResponse, error) { + if api.light { + parent := api.les.BlockChain().GetHeaderByHash(params.ParentHash) + if parent == nil { + return INVALID, fmt.Errorf("could not find parent %x", params.ParentHash) + } + block, err := ExecutableDataToBlock(api.les.BlockChain().Config(), parent, params) + if err != nil { + return INVALID, err + } + if err = api.les.BlockChain().InsertHeader(block.Header()); err != nil { + return INVALID, err + } + return VALID, nil + } + parent := api.eth.BlockChain().GetBlockByHash(params.ParentHash) + if parent == nil { + return INVALID, fmt.Errorf("could not find parent %x", params.ParentHash) + } + if !api.eth.Synced() { + td := api.eth.BlockChain().GetTdByHash(parent.Hash()) + if td != nil && td.Cmp(api.eth.BlockChain().Config().TerminalTotalDifficulty) > 0 { + api.eth.SetSynced() + } else { + // TODO (MariusVanDerWijden) if the node is not synced and we received a finalized block + // we should trigger the reverse header sync here. + return SYNCING, errors.New("node is not synced yet") + } + } + block, err := ExecutableDataToBlock(api.eth.BlockChain().Config(), parent.Header(), params) + if err != nil { + return INVALID, err + } + if err := api.eth.BlockChain().InsertBlock(block); err != nil { + return INVALID, err + } + return VALID, nil +} + // AssembleBlock creates a new block, inserts it into the chain, and returns the "execution // data" required for eth2 clients to process the new block. -func (api *ConsensusAPI) AssembleBlock(params AssembleBlockParams) (*ExecutableData, error) { +func (api *ConsensusAPI) assembleBlock(params AssembleBlockParams) (*ExecutableData, error) { if api.light { return nil, errors.New("not supported") } @@ -179,10 +279,7 @@ func (api *ConsensusAPI) AssembleBlock(params AssembleBlockParams) (*ExecutableD time.Sleep(wait) } pending := api.eth.TxPool().Pending(true) - coinbase, err := api.eth.Etherbase() - if err != nil { - return nil, err - } + coinbase := params.FeeRecipient num := parent.Number() header := &types.Header{ ParentHash: parent.Hash(), @@ -195,8 +292,7 @@ func (api *ConsensusAPI) AssembleBlock(params AssembleBlockParams) (*ExecutableD if config := api.eth.BlockChain().Config(); config.IsLondon(header.Number) { header.BaseFee = misc.CalcBaseFee(config, parent.Header()) } - err = api.engine.Prepare(bc, header) - if err != nil { + if err := api.engine.Prepare(bc, header); err != nil { return nil, err } env, err := api.makeEnv(parent, header) @@ -258,19 +354,7 @@ func (api *ConsensusAPI) AssembleBlock(params AssembleBlockParams) (*ExecutableD if err != nil { return nil, err } - return &ExecutableData{ - BlockHash: block.Hash(), - ParentHash: block.ParentHash(), - Miner: block.Coinbase(), - StateRoot: block.Root(), - Number: block.NumberU64(), - GasLimit: block.GasLimit(), - GasUsed: block.GasUsed(), - Timestamp: block.Time(), - ReceiptRoot: block.ReceiptHash(), - LogsBloom: block.Bloom().Bytes(), - Transactions: encodeTransactions(block.Transactions()), - }, nil + return BlockToExecutableData(block, params.Random), nil } func encodeTransactions(txs []*types.Transaction) [][]byte { @@ -293,7 +377,7 @@ func decodeTransactions(enc [][]byte) ([]*types.Transaction, error) { return txs, nil } -func InsertBlockParamsToBlock(config *chainParams.ChainConfig, parent *types.Header, params ExecutableData) (*types.Block, error) { +func ExecutableDataToBlock(config *chainParams.ChainConfig, parent *types.Header, params ExecutableData) (*types.Block, error) { txs, err := decodeTransactions(params.Transactions) if err != nil { return nil, err @@ -303,7 +387,7 @@ func InsertBlockParamsToBlock(config *chainParams.ChainConfig, parent *types.Hea header := &types.Header{ ParentHash: params.ParentHash, UncleHash: types.EmptyUncleHash, - Coinbase: params.Miner, + Coinbase: params.Coinbase, Root: params.StateRoot, TxHash: types.DeriveSha(types.Transactions(txs), trie.NewStackTrie(nil)), ReceiptHash: params.ReceiptRoot, @@ -313,6 +397,7 @@ func InsertBlockParamsToBlock(config *chainParams.ChainConfig, parent *types.Hea GasLimit: params.GasLimit, GasUsed: params.GasUsed, Time: params.Timestamp, + // TODO (MariusVanDerWijden) add params.Random to header once required } if config.IsLondon(number) { header.BaseFee = misc.CalcBaseFee(config, parent) @@ -324,55 +409,33 @@ func InsertBlockParamsToBlock(config *chainParams.ChainConfig, parent *types.Hea return block, nil } -// NewBlock creates an Eth1 block, inserts it in the chain, and either returns true, -// or false + an error. This is a bit redundant for go, but simplifies things on the -// eth2 side. -func (api *ConsensusAPI) NewBlock(params ExecutableData) (*NewBlockResponse, error) { - if api.light { - parent := api.les.BlockChain().GetHeaderByHash(params.ParentHash) - if parent == nil { - return &NewBlockResponse{false}, fmt.Errorf("could not find parent %x", params.ParentHash) - } - block, err := InsertBlockParamsToBlock(api.les.BlockChain().Config(), parent, params) - if err != nil { - return nil, err - } - err = api.les.BlockChain().InsertHeader(block.Header()) - return &NewBlockResponse{err == nil}, err - } - parent := api.eth.BlockChain().GetBlockByHash(params.ParentHash) - if parent == nil { - return &NewBlockResponse{false}, fmt.Errorf("could not find parent %x", params.ParentHash) - } - block, err := InsertBlockParamsToBlock(api.eth.BlockChain().Config(), parent.Header(), params) - if err != nil { - return nil, err +func BlockToExecutableData(block *types.Block, random common.Hash) *ExecutableData { + return &ExecutableData{ + BlockHash: block.Hash(), + ParentHash: block.ParentHash(), + Coinbase: block.Coinbase(), + StateRoot: block.Root(), + Number: block.NumberU64(), + GasLimit: block.GasLimit(), + GasUsed: block.GasUsed(), + Timestamp: block.Time(), + ReceiptRoot: block.ReceiptHash(), + LogsBloom: block.Bloom().Bytes(), + Transactions: encodeTransactions(block.Transactions()), + Random: random, } - err = api.eth.BlockChain().InsertBlock(block) - return &NewBlockResponse{err == nil}, err } // Used in tests to add a the list of transactions from a block to the tx pool. -func (api *ConsensusAPI) addBlockTxs(block *types.Block) error { - for _, tx := range block.Transactions() { +func (api *ConsensusAPI) insertTransactions(txs types.Transactions) error { + for _, tx := range txs { api.eth.TxPool().AddLocal(tx) } return nil } -// FinalizeBlock is called to mark a block as synchronized, so -// that data that is no longer needed can be removed. -func (api *ConsensusAPI) FinalizeBlock(blockHash common.Hash) (*GenericResponse, error) { - // Finalize the transition if it's the first `FinalisedBlock` event. - merger := api.merger() - if !merger.EnteredPoS() { - merger.EnterPoS() - } - return &GenericResponse{true}, nil -} - -// SetHead is called to perform a force choice. -func (api *ConsensusAPI) SetHead(newHead common.Hash) (*GenericResponse, error) { +// setHead is called to perform a force choice. +func (api *ConsensusAPI) setHead(newHead common.Hash) error { // Trigger the transition if it's the first `NewHead` event. merger := api.merger() if !merger.LeftPoW() { @@ -381,30 +444,30 @@ func (api *ConsensusAPI) SetHead(newHead common.Hash) (*GenericResponse, error) if api.light { headHeader := api.les.BlockChain().CurrentHeader() if headHeader.Hash() == newHead { - return &GenericResponse{true}, nil + return nil } newHeadHeader := api.les.BlockChain().GetHeaderByHash(newHead) if newHeadHeader == nil { - return &GenericResponse{false}, nil + return &UnknownHeader } if err := api.les.BlockChain().SetChainHead(newHeadHeader); err != nil { - return &GenericResponse{false}, nil + return err } - return &GenericResponse{true}, nil + return nil } headBlock := api.eth.BlockChain().CurrentBlock() if headBlock.Hash() == newHead { - return &GenericResponse{true}, nil + return nil } newHeadBlock := api.eth.BlockChain().GetBlockByHash(newHead) if newHeadBlock == nil { - return &GenericResponse{false}, nil + return &UnknownHeader } if err := api.eth.BlockChain().SetChainHead(newHeadBlock); err != nil { - return &GenericResponse{false}, nil + return err } api.eth.SetSynced() - return &GenericResponse{true}, nil + return nil } // Helper function, return the merger instance. @@ -414,3 +477,59 @@ func (api *ConsensusAPI) merger() *core.Merger { } return api.eth.Merger() } + +// Helper API for the merge f2f + +func (api *ConsensusAPI) ExportChain(path string) error { + if api.light { + return errors.New("cannot export chain in light mode") + } + f, err := os.Create(path) + if err != nil { + return err + } + return api.eth.BlockChain().Export(f) +} + +func (api *ConsensusAPI) ImportChain(path string) error { + if api.light { + return errors.New("cannot import chain in light mode") + } + f, err := os.Open(path) + if err != nil { + return err + } + for { + var block types.Block + if err := block.DecodeRLP(rlp.NewStream(f, 0)); err != nil { + break + } + if err := api.eth.BlockChain().InsertBlock(&block); err != nil { + return err + } + } + return nil +} + +func (api *ConsensusAPI) ExportExecutableData(path string) error { + if api.light { + return errors.New("cannot export chain in light mode") + } + f, err := os.Create(path) + if err != nil { + return err + } + for i := uint64(0); i < api.eth.BlockChain().CurrentBlock().NumberU64(); i++ { + block := api.eth.BlockChain().GetBlockByNumber(i) + exec := BlockToExecutableData(block, common.Hash{}) + b, err := exec.MarshalJSON() + if err != nil { + return err + } + if _, err := f.Write(b); err != nil { + return err + } + f.Write([]byte("\n")) + } + return nil +} diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 1753fdaaba0c..710f6b376b43 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -46,6 +46,7 @@ var ( func generatePreMergeChain(n int) (*core.Genesis, []*types.Block) { db := rawdb.NewMemoryDatabase() config := params.AllEthashProtocolChanges + config.TerminalTotalDifficulty = big.NewInt(100000000000000) genesis := &core.Genesis{ Config: config, Alloc: core.GenesisAlloc{testAddr: {Balance: testBalance}}, @@ -129,7 +130,7 @@ func TestEth2AssembleBlock(t *testing.T) { ParentHash: blocks[9].Hash(), Timestamp: blocks[9].Time() + 5, } - execData, err := api.AssembleBlock(blockParams) + execData, err := api.assembleBlock(blockParams) if err != nil { t.Fatalf("error producing block, err=%v", err) } @@ -146,12 +147,12 @@ func TestEth2AssembleBlockWithAnotherBlocksTxs(t *testing.T) { api := NewConsensusAPI(ethservice, nil) // Put the 10th block's tx in the pool and produce a new block - api.addBlockTxs(blocks[9]) + api.insertTransactions(blocks[9].Transactions()) blockParams := AssembleBlockParams{ ParentHash: blocks[8].Hash(), Timestamp: blocks[8].Time() + 5, } - execData, err := api.AssembleBlock(blockParams) + execData, err := api.assembleBlock(blockParams) if err != nil { t.Fatalf("error producing block, err=%v", err) } @@ -160,6 +161,32 @@ func TestEth2AssembleBlockWithAnotherBlocksTxs(t *testing.T) { } } +func TestEth2PrepareAndGetPayload(t *testing.T) { + genesis, blocks := generatePreMergeChain(10) + n, ethservice := startEthService(t, genesis, blocks[:9]) + defer n.Close() + + api := NewConsensusAPI(ethservice, nil) + + // Put the 10th block's tx in the pool and produce a new block + api.insertTransactions(blocks[9].Transactions()) + blockParams := AssembleBlockParams{ + ParentHash: blocks[8].Hash(), + Timestamp: blocks[8].Time() + 5, + } + respID, err := api.PreparePayload(blockParams) + if err != nil { + t.Fatalf("error preparing payload, err=%v", err) + } + execData, err := api.GetPayload(respID) + if err != nil { + t.Fatalf("error getting payload, err=%v", err) + } + if len(execData.Transactions) != blocks[9].Transactions().Len() { + t.Fatalf("invalid number of transactions %d != 1", len(execData.Transactions)) + } +} + func checkLogEvents(t *testing.T, logsCh <-chan []*types.Log, rmLogsCh <-chan core.RemovedLogsEvent, wantNew, wantRemoved int) { t.Helper() @@ -199,22 +226,22 @@ func TestEth2NewBlock(t *testing.T) { for i := 0; i < 10; i++ { statedb, _ := ethservice.BlockChain().StateAt(parent.Root()) nonce := statedb.GetNonce(testAddr) - tx, err := types.SignTx(types.NewContractCreation(nonce, new(big.Int), 1000000, big.NewInt(2*params.InitialBaseFee), logCode), types.LatestSigner(ethservice.BlockChain().Config()), testKey) + tx, _ := types.SignTx(types.NewContractCreation(nonce, new(big.Int), 1000000, big.NewInt(2*params.InitialBaseFee), logCode), types.LatestSigner(ethservice.BlockChain().Config()), testKey) ethservice.TxPool().AddLocal(tx) - execData, err := api.AssembleBlock(AssembleBlockParams{ + execData, err := api.assembleBlock(AssembleBlockParams{ ParentHash: parent.Hash(), Timestamp: parent.Time() + 5, }) if err != nil { t.Fatalf("Failed to create the executable data %v", err) } - block, err := InsertBlockParamsToBlock(ethservice.BlockChain().Config(), parent.Header(), *execData) + block, err := ExecutableDataToBlock(ethservice.BlockChain().Config(), parent.Header(), *execData) if err != nil { t.Fatalf("Failed to convert executable data to block %v", err) } - newResp, err := api.NewBlock(*execData) - if err != nil || !newResp.Valid { + newResp, err := api.ExecutePayload(*execData) + if err != nil || newResp.Status != "VALID" { t.Fatalf("Failed to insert block: %v", err) } if ethservice.BlockChain().CurrentBlock().NumberU64() != block.NumberU64()-1 { @@ -222,8 +249,7 @@ func TestEth2NewBlock(t *testing.T) { } checkLogEvents(t, newLogCh, rmLogsCh, 0, 0) - setResp, err := api.SetHead(block.Hash()) - if err != nil || !setResp.Success { + if err := api.ForkChoiceUpdated(ForkChoiceParams{HeadBlockHash: block.Hash(), FinalizedBlockHash: block.Hash()}); err != nil { t.Fatalf("Failed to insert block: %v", err) } if ethservice.BlockChain().CurrentBlock().NumberU64() != block.NumberU64() { @@ -240,27 +266,29 @@ func TestEth2NewBlock(t *testing.T) { ) parent = preMergeBlocks[len(preMergeBlocks)-1] for i := 0; i < 10; i++ { - execData, err := api.AssembleBlock(AssembleBlockParams{ + execData, err := api.assembleBlock(AssembleBlockParams{ ParentHash: parent.Hash(), Timestamp: parent.Time() + 6, }) if err != nil { t.Fatalf("Failed to create the executable data %v", err) } - block, err := InsertBlockParamsToBlock(ethservice.BlockChain().Config(), parent.Header(), *execData) + block, err := ExecutableDataToBlock(ethservice.BlockChain().Config(), parent.Header(), *execData) if err != nil { t.Fatalf("Failed to convert executable data to block %v", err) } - newResp, err := api.NewBlock(*execData) - if err != nil || !newResp.Valid { + newResp, err := api.ExecutePayload(*execData) + if err != nil || newResp.Status != "VALID" { t.Fatalf("Failed to insert block: %v", err) } if ethservice.BlockChain().CurrentBlock().NumberU64() != head { t.Fatalf("Chain head shouldn't be updated") } - setResp, err := api.SetHead(block.Hash()) - if err != nil || !setResp.Success { + if err := api.ConsensusValidated(ConsensusValidatedParams{BlockHash: block.Hash(), Status: "VALID"}); err != nil { + t.Fatalf("Failed to insert block: %v", err) + } + if err := api.ForkChoiceUpdated(ForkChoiceParams{FinalizedBlockHash: block.Hash(), HeadBlockHash: block.Hash()}); err != nil { t.Fatalf("Failed to insert block: %v", err) } if ethservice.BlockChain().CurrentBlock().NumberU64() != block.NumberU64() { @@ -284,26 +312,25 @@ func TestEth2DeepReorg(t *testing.T) { t.Errorf("Block %d not pruned", parent.NumberU64()) } for i := 0; i < 10; i++ { - execData, err := api.AssembleBlock(AssembleBlockParams{ + execData, err := api.assembleBlock(AssembleBlockParams{ ParentHash: parent.Hash(), Timestamp: parent.Time() + 5, }) if err != nil { t.Fatalf("Failed to create the executable data %v", err) } - block, err := InsertBlockParamsToBlock(ethservice.BlockChain().Config(), parent.Header(), *execData) + block, err := ExecutableDataToBlock(ethservice.BlockChain().Config(), parent.Header(), *execData) if err != nil { t.Fatalf("Failed to convert executable data to block %v", err) } - newResp, err := api.NewBlock(*execData) - if err != nil || !newResp.Valid { + newResp, err := api.ExecutePayload(*execData) + if err != nil || newResp.Status != "VALID" { t.Fatalf("Failed to insert block: %v", err) } if ethservice.BlockChain().CurrentBlock().NumberU64() != head { t.Fatalf("Chain head shouldn't be updated") } - setResp, err := api.SetHead(block.Hash()) - if err != nil || !setResp.Success { + if err := api.ForkChoiceUpdated(ForkChoiceParams{HeadBlockHash: block.Hash(), FinalizedBlockHash: block.Hash()}); err != nil { t.Fatalf("Failed to insert block: %v", err) } if ethservice.BlockChain().CurrentBlock().NumberU64() != block.NumberU64() { @@ -335,6 +362,60 @@ func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block) t.Fatal("can't import test blocks:", err) } ethservice.SetEtherbase(testAddr) + ethservice.SetSynced() return n, ethservice } + +func TestFullAPI(t *testing.T) { + genesis, preMergeBlocks := generatePreMergeChain(10) + n, ethservice := startEthService(t, genesis, preMergeBlocks) + defer n.Close() + var ( + api = NewConsensusAPI(ethservice, nil) + parent = ethservice.BlockChain().CurrentBlock() + // This EVM code generates a log when the contract is created. + logCode = common.Hex2Bytes("60606040525b7f24ec1d3ff24c2f6ff210738839dbc339cd45a5294d85c79361016243157aae7b60405180905060405180910390a15b600a8060416000396000f360606040526008565b00") + ) + for i := 0; i < 10; i++ { + statedb, _ := ethservice.BlockChain().StateAt(parent.Root()) + nonce := statedb.GetNonce(testAddr) + tx, _ := types.SignTx(types.NewContractCreation(nonce, new(big.Int), 1000000, big.NewInt(2*params.InitialBaseFee), logCode), types.LatestSigner(ethservice.BlockChain().Config()), testKey) + ethservice.TxPool().AddLocal(tx) + + params := AssembleBlockParams{ + ParentHash: parent.Hash(), + Timestamp: parent.Time() + 1, + Random: crypto.Keccak256Hash([]byte{byte(i)}), + FeeRecipient: parent.Coinbase(), + } + resp, err := api.PreparePayload(params) + if err != nil { + t.Fatalf("can't prepare payload: %v", err) + } + payload, err := api.GetPayload(resp) + if err != nil { + t.Fatalf("can't get payload: %v", err) + } + execResp, err := api.ExecutePayload(*payload) + if err != nil { + t.Fatalf("can't execute payload: %v", err) + } + if execResp.Status != VALID.Status { + t.Fatalf("invalid status: %v", execResp.Status) + } + + if err := api.ConsensusValidated(ConsensusValidatedParams{BlockHash: payload.BlockHash, Status: VALID.Status}); err != nil { + t.Fatalf("failed to validate consensus: %v", err) + } + + if err := api.ForkChoiceUpdated(ForkChoiceParams{HeadBlockHash: payload.BlockHash, FinalizedBlockHash: payload.BlockHash}); err != nil { + t.Fatalf("Failed to insert block: %v", err) + } + if ethservice.BlockChain().CurrentBlock().NumberU64() != payload.Number { + t.Fatalf("Chain head should be updated") + } + parent = ethservice.BlockChain().CurrentBlock() + + } +} diff --git a/eth/catalyst/api_types.go b/eth/catalyst/api_types.go index 4f6c18ce4217..0efeb1aae14f 100644 --- a/eth/catalyst/api_types.go +++ b/eth/catalyst/api_types.go @@ -23,10 +23,12 @@ import ( //go:generate go run github.com/fjl/gencodec -type AssembleBlockParams -field-override assembleBlockParamsMarshaling -out gen_blockparams.go -// Structure described at https://hackmd.io/T9x2mMA4S7us8tJwEB3FDQ +// Structure described at https://github.com/ethereum/execution-apis/pull/74 type AssembleBlockParams struct { - ParentHash common.Hash `json:"parentHash" gencodec:"required"` - Timestamp uint64 `json:"timestamp" gencodec:"required"` + ParentHash common.Hash `json:"parentHash" gencodec:"required"` + Timestamp uint64 `json:"timestamp" gencodec:"required"` + Random common.Hash `json:"random" gencodec:"required"` + FeeRecipient common.Address `json:"feeRecipient" gencodec:"required"` } // JSON type overrides for assembleBlockParams. @@ -36,29 +38,34 @@ type assembleBlockParamsMarshaling struct { //go:generate go run github.com/fjl/gencodec -type ExecutableData -field-override executableDataMarshaling -out gen_ed.go -// Structure described at https://notes.ethereum.org/@n0ble/rayonism-the-merge-spec#Parameters1 +// Structure described at https://github.com/ethereum/execution-apis/pull/74/files type ExecutableData struct { - BlockHash common.Hash `json:"blockHash" gencodec:"required"` - ParentHash common.Hash `json:"parentHash" gencodec:"required"` - Miner common.Address `json:"miner" gencodec:"required"` - StateRoot common.Hash `json:"stateRoot" gencodec:"required"` - Number uint64 `json:"number" gencodec:"required"` - GasLimit uint64 `json:"gasLimit" gencodec:"required"` - GasUsed uint64 `json:"gasUsed" gencodec:"required"` - Timestamp uint64 `json:"timestamp" gencodec:"required"` - ReceiptRoot common.Hash `json:"receiptsRoot" gencodec:"required"` - LogsBloom []byte `json:"logsBloom" gencodec:"required"` - Transactions [][]byte `json:"transactions" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + ParentHash common.Hash `json:"parentHash" gencodec:"required"` + Coinbase common.Address `json:"coinbase" gencodec:"required"` + StateRoot common.Hash `json:"stateRoot" gencodec:"required"` + ReceiptRoot common.Hash `json:"receiptRoot" gencodec:"required"` + LogsBloom []byte `json:"logsBloom" gencodec:"required"` + Random common.Hash `json:"random" gencodec:"required"` + Number uint64 `json:"blockNumber" gencodec:"required"` + GasLimit uint64 `json:"gasLimit" gencodec:"required"` + GasUsed uint64 `json:"gasUsed" gencodec:"required"` + Timestamp uint64 `json:"timestamp" gencodec:"required"` + ExtraData []byte `json:"extraData" gencodec:"required"` + BaseFeePerGas uint64 `json:"baseFeePerGas" gencodec:"required"` + Transactions [][]byte `json:"transactions" gencodec:"required"` } // JSON type overrides for executableData. type executableDataMarshaling struct { - Number hexutil.Uint64 - GasLimit hexutil.Uint64 - GasUsed hexutil.Uint64 - Timestamp hexutil.Uint64 - LogsBloom hexutil.Bytes - Transactions []hexutil.Bytes + Number hexutil.Uint64 + GasLimit hexutil.Uint64 + GasUsed hexutil.Uint64 + Timestamp hexutil.Uint64 + BaseFeePerGas hexutil.Uint64 + ExtraData hexutil.Bytes + LogsBloom hexutil.Bytes + Transactions []hexutil.Bytes } type NewBlockResponse struct { @@ -68,3 +75,17 @@ type NewBlockResponse struct { type GenericResponse struct { Success bool `json:"success"` } + +type GenericStringResponse struct { + Status string `json:"status"` +} + +type ConsensusValidatedParams struct { + BlockHash common.Hash `json:"blockHash"` + Status string `json:"status"` +} + +type ForkChoiceParams struct { + HeadBlockHash common.Hash `json:"headBlockHash"` + FinalizedBlockHash common.Hash `json:"finalizedBlockHash"` +} diff --git a/eth/catalyst/gen_blockparams.go b/eth/catalyst/gen_blockparams.go index 4b4182629960..9928c12908a9 100644 --- a/eth/catalyst/gen_blockparams.go +++ b/eth/catalyst/gen_blockparams.go @@ -15,20 +15,26 @@ var _ = (*assembleBlockParamsMarshaling)(nil) // MarshalJSON marshals as JSON. func (a AssembleBlockParams) MarshalJSON() ([]byte, error) { type AssembleBlockParams struct { - ParentHash common.Hash `json:"parentHash" gencodec:"required"` - Timestamp hexutil.Uint64 `json:"timestamp" gencodec:"required"` + ParentHash common.Hash `json:"parentHash" gencodec:"required"` + Timestamp hexutil.Uint64 `json:"timestamp" gencodec:"required"` + Random common.Hash `json:"random" gencodec:"required"` + FeeRecipient common.Address `json:"feeRecipient" gencodec:"required"` } var enc AssembleBlockParams enc.ParentHash = a.ParentHash enc.Timestamp = hexutil.Uint64(a.Timestamp) + enc.Random = a.Random + enc.FeeRecipient = a.FeeRecipient return json.Marshal(&enc) } // UnmarshalJSON unmarshals from JSON. func (a *AssembleBlockParams) UnmarshalJSON(input []byte) error { type AssembleBlockParams struct { - ParentHash *common.Hash `json:"parentHash" gencodec:"required"` - Timestamp *hexutil.Uint64 `json:"timestamp" gencodec:"required"` + ParentHash *common.Hash `json:"parentHash" gencodec:"required"` + Timestamp *hexutil.Uint64 `json:"timestamp" gencodec:"required"` + Random *common.Hash `json:"random" gencodec:"required"` + FeeRecipient *common.Address `json:"feeRecipient" gencodec:"required"` } var dec AssembleBlockParams if err := json.Unmarshal(input, &dec); err != nil { @@ -42,5 +48,13 @@ func (a *AssembleBlockParams) UnmarshalJSON(input []byte) error { return errors.New("missing required field 'timestamp' for AssembleBlockParams") } a.Timestamp = uint64(*dec.Timestamp) + if dec.Random == nil { + return errors.New("missing required field 'random' for AssembleBlockParams") + } + a.Random = *dec.Random + if dec.FeeRecipient == nil { + return errors.New("missing required field 'feeRecipient' for AssembleBlockParams") + } + a.FeeRecipient = *dec.FeeRecipient return nil } diff --git a/eth/catalyst/gen_ed.go b/eth/catalyst/gen_ed.go index 850d8617a5ae..bc99a4a0954e 100644 --- a/eth/catalyst/gen_ed.go +++ b/eth/catalyst/gen_ed.go @@ -15,29 +15,35 @@ var _ = (*executableDataMarshaling)(nil) // MarshalJSON marshals as JSON. func (e ExecutableData) MarshalJSON() ([]byte, error) { type ExecutableData struct { - BlockHash common.Hash `json:"blockHash" gencodec:"required"` - ParentHash common.Hash `json:"parentHash" gencodec:"required"` - Miner common.Address `json:"miner" gencodec:"required"` - StateRoot common.Hash `json:"stateRoot" gencodec:"required"` - Number hexutil.Uint64 `json:"number" gencodec:"required"` - GasLimit hexutil.Uint64 `json:"gasLimit" gencodec:"required"` - GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"` - Timestamp hexutil.Uint64 `json:"timestamp" gencodec:"required"` - ReceiptRoot common.Hash `json:"receiptsRoot" gencodec:"required"` - LogsBloom hexutil.Bytes `json:"logsBloom" gencodec:"required"` - Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + ParentHash common.Hash `json:"parentHash" gencodec:"required"` + Coinbase common.Address `json:"coinbase" gencodec:"required"` + StateRoot common.Hash `json:"stateRoot" gencodec:"required"` + ReceiptRoot common.Hash `json:"receiptRoot" gencodec:"required"` + LogsBloom hexutil.Bytes `json:"logsBloom" gencodec:"required"` + Random common.Hash `json:"random" gencodec:"required"` + Number hexutil.Uint64 `json:"blockNumber" gencodec:"required"` + GasLimit hexutil.Uint64 `json:"gasLimit" gencodec:"required"` + GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"` + Timestamp hexutil.Uint64 `json:"timestamp" gencodec:"required"` + ExtraData hexutil.Bytes `json:"extraData" gencodec:"required"` + BaseFeePerGas hexutil.Uint64 `json:"baseFeePerGas" gencodec:"required"` + Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` } var enc ExecutableData enc.BlockHash = e.BlockHash enc.ParentHash = e.ParentHash - enc.Miner = e.Miner + enc.Coinbase = e.Coinbase enc.StateRoot = e.StateRoot + enc.ReceiptRoot = e.ReceiptRoot + enc.LogsBloom = e.LogsBloom + enc.Random = e.Random enc.Number = hexutil.Uint64(e.Number) enc.GasLimit = hexutil.Uint64(e.GasLimit) enc.GasUsed = hexutil.Uint64(e.GasUsed) enc.Timestamp = hexutil.Uint64(e.Timestamp) - enc.ReceiptRoot = e.ReceiptRoot - enc.LogsBloom = e.LogsBloom + enc.ExtraData = e.ExtraData + enc.BaseFeePerGas = hexutil.Uint64(e.BaseFeePerGas) if e.Transactions != nil { enc.Transactions = make([]hexutil.Bytes, len(e.Transactions)) for k, v := range e.Transactions { @@ -50,17 +56,20 @@ func (e ExecutableData) MarshalJSON() ([]byte, error) { // UnmarshalJSON unmarshals from JSON. func (e *ExecutableData) UnmarshalJSON(input []byte) error { type ExecutableData struct { - BlockHash *common.Hash `json:"blockHash" gencodec:"required"` - ParentHash *common.Hash `json:"parentHash" gencodec:"required"` - Miner *common.Address `json:"miner" gencodec:"required"` - StateRoot *common.Hash `json:"stateRoot" gencodec:"required"` - Number *hexutil.Uint64 `json:"number" gencodec:"required"` - GasLimit *hexutil.Uint64 `json:"gasLimit" gencodec:"required"` - GasUsed *hexutil.Uint64 `json:"gasUsed" gencodec:"required"` - Timestamp *hexutil.Uint64 `json:"timestamp" gencodec:"required"` - ReceiptRoot *common.Hash `json:"receiptsRoot" gencodec:"required"` - LogsBloom *hexutil.Bytes `json:"logsBloom" gencodec:"required"` - Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` + BlockHash *common.Hash `json:"blockHash" gencodec:"required"` + ParentHash *common.Hash `json:"parentHash" gencodec:"required"` + Coinbase *common.Address `json:"coinbase" gencodec:"required"` + StateRoot *common.Hash `json:"stateRoot" gencodec:"required"` + ReceiptRoot *common.Hash `json:"receiptRoot" gencodec:"required"` + LogsBloom *hexutil.Bytes `json:"logsBloom" gencodec:"required"` + Random *common.Hash `json:"random" gencodec:"required"` + Number *hexutil.Uint64 `json:"blockNumber" gencodec:"required"` + GasLimit *hexutil.Uint64 `json:"gasLimit" gencodec:"required"` + GasUsed *hexutil.Uint64 `json:"gasUsed" gencodec:"required"` + Timestamp *hexutil.Uint64 `json:"timestamp" gencodec:"required"` + ExtraData *hexutil.Bytes `json:"extraData" gencodec:"required"` + BaseFeePerGas *hexutil.Uint64 `json:"baseFeePerGas" gencodec:"required"` + Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` } var dec ExecutableData if err := json.Unmarshal(input, &dec); err != nil { @@ -74,16 +83,28 @@ func (e *ExecutableData) UnmarshalJSON(input []byte) error { return errors.New("missing required field 'parentHash' for ExecutableData") } e.ParentHash = *dec.ParentHash - if dec.Miner == nil { - return errors.New("missing required field 'miner' for ExecutableData") + if dec.Coinbase == nil { + return errors.New("missing required field 'coinbase' for ExecutableData") } - e.Miner = *dec.Miner + e.Coinbase = *dec.Coinbase if dec.StateRoot == nil { return errors.New("missing required field 'stateRoot' for ExecutableData") } e.StateRoot = *dec.StateRoot + if dec.ReceiptRoot == nil { + return errors.New("missing required field 'receiptRoot' for ExecutableData") + } + e.ReceiptRoot = *dec.ReceiptRoot + if dec.LogsBloom == nil { + return errors.New("missing required field 'logsBloom' for ExecutableData") + } + e.LogsBloom = *dec.LogsBloom + if dec.Random == nil { + return errors.New("missing required field 'random' for ExecutableData") + } + e.Random = *dec.Random if dec.Number == nil { - return errors.New("missing required field 'number' for ExecutableData") + return errors.New("missing required field 'blockNumber' for ExecutableData") } e.Number = uint64(*dec.Number) if dec.GasLimit == nil { @@ -98,14 +119,14 @@ func (e *ExecutableData) UnmarshalJSON(input []byte) error { return errors.New("missing required field 'timestamp' for ExecutableData") } e.Timestamp = uint64(*dec.Timestamp) - if dec.ReceiptRoot == nil { - return errors.New("missing required field 'receiptsRoot' for ExecutableData") + if dec.ExtraData == nil { + return errors.New("missing required field 'extraData' for ExecutableData") } - e.ReceiptRoot = *dec.ReceiptRoot - if dec.LogsBloom == nil { - return errors.New("missing required field 'logsBloom' for ExecutableData") + e.ExtraData = *dec.ExtraData + if dec.BaseFeePerGas == nil { + return errors.New("missing required field 'baseFeePerGas' for ExecutableData") } - e.LogsBloom = *dec.LogsBloom + e.BaseFeePerGas = uint64(*dec.BaseFeePerGas) if dec.Transactions == nil { return errors.New("missing required field 'transactions' for ExecutableData") } diff --git a/les/client_handler.go b/les/client_handler.go index 3125869f6a0c..fd78ab34877f 100644 --- a/les/client_handler.go +++ b/les/client_handler.go @@ -18,7 +18,6 @@ package les import ( "context" - "fmt" "math/big" "math/rand" "sync" @@ -148,7 +147,6 @@ func (h *clientHandler) handle(p *serverPeer, noInitAnnounce bool) error { // Discard all the announces after the transition // Also discarding initial signal to prevent syncing during testing. if !(noInitAnnounce || h.backend.merger.LeftPoW()) { - fmt.Print("adf") h.fetcher.announce(p, &announceData{Hash: p.headInfo.Hash, Number: p.headInfo.Number, Td: p.headInfo.Td}) } diff --git a/light/lightchain.go b/light/lightchain.go index 114aec657792..0244d295229c 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -375,7 +375,7 @@ func (lc *LightChain) postChainEvents(events []interface{}) { } func (lc *LightChain) InsertHeader(header *types.Header) error { - // Verify the header first before obtaing the lock + // Verify the header first before obtaining the lock headers := []*types.Header{header} if _, err := lc.hc.ValidateHeaderChain(headers, 100); err != nil { return err @@ -393,12 +393,12 @@ func (lc *LightChain) InsertHeader(header *types.Header) error { } func (lc *LightChain) SetChainHead(header *types.Header) error { - lc.wg.Add(1) - defer lc.wg.Done() - lc.chainmu.Lock() defer lc.chainmu.Unlock() + lc.wg.Add(1) + defer lc.wg.Done() + if err := lc.hc.Reorg([]*types.Header{header}); err != nil { return err } diff --git a/miner/stress/beacon/main.go b/miner/stress/beacon/main.go index 30e6e8ee044d..46bf6d7e1476 100644 --- a/miner/stress/beacon/main.go +++ b/miner/stress/beacon/main.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/fdlimit" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" @@ -122,6 +123,7 @@ func newNode(typ nodetype, genesis *core.Genesis, enodes []*enode.Node) *ethNode enode := stack.Server().Self() // Inject the signer key and start sealing with it + stack.AccountManager().AddBackend(keystore.NewPlaintextKeyStore("beacon-stress")) store := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) if _, err := store.NewAccount(""); err != nil { panic(err) @@ -140,21 +142,24 @@ func (n *ethNode) assembleBlock(parentHash common.Hash, parentTimestamp uint64) if n.typ != eth2MiningNode { return nil, errors.New("invalid node type") } - return n.api.AssembleBlock(catalyst.AssembleBlockParams{ + payload, err := n.api.PreparePayload(catalyst.AssembleBlockParams{ ParentHash: parentHash, Timestamp: uint64(time.Now().Unix()), }) + if err != nil { + return nil, err + } + return n.api.GetPayload(hexutil.Uint64(payload.PayloadID)) } func (n *ethNode) insertBlock(eb catalyst.ExecutableData) error { if !eth2types(n.typ) { return errors.New("invalid node type") } - response, err := n.api.NewBlock(eb) + newResp, err := n.api.ExecutePayload(eb) if err != nil { return err - } - if !response.Valid { + } else if newResp.Status != "VALID" { return errors.New("failed to insert block") } return nil @@ -173,17 +178,13 @@ func (n *ethNode) insertBlockAndSetHead(parent *types.Header, ed catalyst.Execut } else { config = n.ethBackend.BlockChain().Config() } - block, err := catalyst.InsertBlockParamsToBlock(config, parent, ed) + block, err := catalyst.ExecutableDataToBlock(config, parent, ed) if err != nil { return err } - response, err := n.api.SetHead(block.Hash()) - if err != nil { + if err := n.api.ConsensusValidated(catalyst.ConsensusValidatedParams{BlockHash: block.Hash(), Status: "VALID"}); err != nil { return err } - if !response.Success { - return errors.New("failed to set head") - } return nil } @@ -280,7 +281,7 @@ func (mgr *nodeManager) run() { nodes = append(nodes, mgr.getNodes(eth2NormalNode)...) nodes = append(nodes, mgr.getNodes(eth2LightClient)...) for _, node := range append(nodes) { - node.api.FinalizeBlock(oldest.Hash()) + node.api.ConsensusValidated(catalyst.ConsensusValidatedParams{BlockHash: oldest.Hash(), Status: catalyst.VALID.Status}) } log.Info("Finalised eth2 block", "number", oldest.NumberU64(), "hash", oldest.Hash()) waitFinalise = waitFinalise[1:] @@ -318,7 +319,7 @@ func (mgr *nodeManager) run() { log.Error("Failed to assemble the block", "err", err) continue } - block, _ := catalyst.InsertBlockParamsToBlock(chain.Config(), parentBlock.Header(), *ed) + block, _ := catalyst.ExecutableDataToBlock(chain.Config(), parentBlock.Header(), *ed) nodes := mgr.getNodes(eth2MiningNode) nodes = append(nodes, mgr.getNodes(eth2NormalNode)...) diff --git a/rpc/errors.go b/rpc/errors.go index 4c06a745fbd8..184275244a79 100644 --- a/rpc/errors.go +++ b/rpc/errors.go @@ -54,6 +54,7 @@ var ( _ Error = new(invalidRequestError) _ Error = new(invalidMessageError) _ Error = new(invalidParamsError) + _ Error = new(CustomError) ) const defaultErrorCode = -32000 @@ -101,3 +102,12 @@ type invalidParamsError struct{ message string } func (e *invalidParamsError) ErrorCode() int { return -32602 } func (e *invalidParamsError) Error() string { return e.message } + +type CustomError struct { + Code int + Message string +} + +func (e *CustomError) ErrorCode() int { return e.Code } + +func (e *CustomError) Error() string { return e.Message } From 10372582115a916086f2337537a88826ad2bc042 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Fri, 1 Oct 2021 09:52:22 +0200 Subject: [PATCH 06/32] internal/ethapi: update to v1.0.0.alpha.2 This commit updates the code to the new spec, moving payloadId into it's own object. It also fixes an issue with finalizing an empty blockhash. It also properly sets the basefee --- cmd/geth/config.go | 3 + cmd/geth/main.go | 1 + cmd/utils/flags.go | 4 ++ consensus/beacon/consensus.go | 4 +- eth/backend.go | 12 ++++ eth/catalyst/api.go | 83 +++++++++++++++++++------ eth/catalyst/api_test.go | 110 +++++++++++++++++++++------------- eth/catalyst/api_types.go | 17 +++++- eth/catalyst/gen_ed.go | 13 ++-- eth/catalyst/gen_payload.go | 36 +++++++++++ miner/worker.go | 13 ++++ 11 files changed, 224 insertions(+), 72 deletions(-) create mode 100644 eth/catalyst/gen_payload.go diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 487dd53d6251..44de9903e141 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -158,6 +158,9 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { if ctx.GlobalIsSet(utils.OverrideArrowGlacierFlag.Name) { cfg.Eth.OverrideArrowGlacier = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideArrowGlacierFlag.Name)) } + if ctx.GlobalIsSet(utils.OverrideTotalTerminalDifficulty.Name) { + cfg.Eth.Genesis.Config.TerminalTotalDifficulty = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideTotalTerminalDifficulty.Name)) + } backend, _ := utils.RegisterEthService(stack, &cfg.Eth, ctx.GlobalBool(utils.CatalystFlag.Name)) // Configure GraphQL if requested diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 9d80ce0bed85..881fd9bde665 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -67,6 +67,7 @@ var ( utils.USBFlag, utils.SmartCardDaemonPathFlag, utils.OverrideArrowGlacierFlag, + utils.OverrideTotalTerminalDifficulty, utils.EthashCacheDirFlag, utils.EthashCachesInMemoryFlag, utils.EthashCachesOnDiskFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 3403bc231372..6c2fd68817f0 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -240,6 +240,10 @@ var ( Name: "override.arrowglacier", Usage: "Manually specify Arrow Glacier fork-block, overriding the bundled setting", } + OverrideTotalTerminalDifficulty = cli.Uint64Flag{ + Name: "override.totalterminaldifficulty", + Usage: "Manually specify TotalTerminalDifficulty, overriding the bundled setting", + } // Light server and client settings LightServeFlag = cli.IntFlag{ Name: "light.serve", diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index 20981ca3e56e..c8e49341074f 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -186,8 +186,8 @@ func (beacon *Beacon) VerifyUncles(chain consensus.ChainReader, block *types.Blo // (b) the timestamp is not verified anymore func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, parent *types.Header) error { // Ensure that the header's extra-data section is of a reasonable size - if len(header.Extra) != 0 { - return fmt.Errorf("non-empty extra-data(%d)", len(header.Extra)) + if len(header.Extra) > 32 { + return fmt.Errorf("extra-data longer than 32 bytes (%d)", len(header.Extra)) } // Verify the block's difficulty to ensure it's the default constant if beaconDifficulty.Cmp(header.Difficulty) != 0 { diff --git a/eth/backend.go b/eth/backend.go index ff98191fabe8..4a8658f9c3a8 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" @@ -478,6 +479,17 @@ func (s *Ethereum) StartMining(threads int) error { } clique.Authorize(eb, wallet.SignData) } + // Authorize nested engines too + if cl, ok := s.engine.(*beacon.Beacon); ok { + if clique, ok := cl.InnerEngine().(*clique.Clique); ok { + wallet, err := s.accountManager.Find(accounts.Account{Address: eb}) + if wallet == nil || err != nil { + log.Error("Etherbase account unavailable locally", "err", err) + return fmt.Errorf("signer missing: %v", err) + } + clique.Authorize(eb, wallet.SignData) + } + } // If mining is started, we can disable the transaction rejection mechanism // introduced to speed sync times. atomic.StoreUint32(&s.handler.acceptTxs, 1) diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 12ab85954bb1..bdf3e22ecfeb 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -18,6 +18,7 @@ package catalyst import ( + "bytes" "errors" "fmt" "math/big" @@ -174,14 +175,14 @@ func (api *ConsensusAPI) makeEnv(parent *types.Block, header *types.Header) (*bl return env, nil } -func (api *ConsensusAPI) PreparePayload(params AssembleBlockParams) (hexutil.Uint64, error) { +func (api *ConsensusAPI) PreparePayload(params AssembleBlockParams) (*PayloadResponse, error) { data, err := api.assembleBlock(params) if err != nil { - return hexutil.Uint64(0), err + return nil, err } id := len(api.preparedBlocks) api.preparedBlocks[id] = data - return hexutil.Uint64(id), nil + return &PayloadResponse{PayloadID: uint64(id)}, nil } func (api *ConsensusAPI) GetPayload(PayloadID hexutil.Uint64) (*ExecutableData, error) { @@ -200,6 +201,9 @@ func (api *ConsensusAPI) ConsensusValidated(params ConsensusValidatedParams) err // Finalize the transition if it's the first `FinalisedBlock` event. merger := api.merger() if !merger.EnteredPoS() { + if err := api.checkTerminalTotalDifficulty(params.BlockHash); err != nil { + return err + } merger.EnterPoS() } return nil //api.setHead(params.BlockHash) @@ -212,7 +216,14 @@ func (api *ConsensusAPI) ConsensusValidated(params ConsensusValidatedParams) err } func (api *ConsensusAPI) ForkchoiceUpdated(params ForkChoiceParams) error { - return api.setHead(params.FinalizedBlockHash) + var emptyHash = common.Hash{} + if !bytes.Equal(params.FinalizedBlockHash[:], emptyHash[:]) { + if err := api.checkTerminalTotalDifficulty(params.FinalizedBlockHash); err != nil { + return err + } + return api.setHead(params.FinalizedBlockHash) + } + return nil } // ExecutePayload creates an Eth1 block, inserts it in the chain, and returns the status of the chain. @@ -235,15 +246,20 @@ func (api *ConsensusAPI) ExecutePayload(params ExecutableData) (GenericStringRes if parent == nil { return INVALID, fmt.Errorf("could not find parent %x", params.ParentHash) } + + td := api.eth.BlockChain().GetTdByHash(parent.Hash()) + ttd := api.eth.BlockChain().Config().TerminalTotalDifficulty if !api.eth.Synced() { - td := api.eth.BlockChain().GetTdByHash(parent.Hash()) - if td != nil && td.Cmp(api.eth.BlockChain().Config().TerminalTotalDifficulty) > 0 { + if td.Cmp(ttd) > 0 { + // first pos block api.eth.SetSynced() } else { // TODO (MariusVanDerWijden) if the node is not synced and we received a finalized block // we should trigger the reverse header sync here. return SYNCING, errors.New("node is not synced yet") } + } else if td.Cmp(ttd) < 0 { + return INVALID, fmt.Errorf("can not execute payload on top of block with low td got: %v threshold %v", td, ttd) } block, err := ExecutableDataToBlock(api.eth.BlockChain().Config(), parent.Header(), params) if err != nil { @@ -286,7 +302,7 @@ func (api *ConsensusAPI) assembleBlock(params AssembleBlockParams) (*ExecutableD Number: num.Add(num, common.Big1), Coinbase: coinbase, GasLimit: parent.GasLimit(), // Keep the gas limit constant in this prototype - Extra: []byte{}, + Extra: []byte{}, // TODO (MariusVanDerWijden) properly set extra data Time: params.Timestamp, } if config := api.eth.BlockChain().Config(); config.IsLondon(header.Number) { @@ -382,6 +398,9 @@ func ExecutableDataToBlock(config *chainParams.ChainConfig, parent *types.Header if err != nil { return nil, err } + if len(params.ExtraData) > 32 { + return nil, fmt.Errorf("invalid extradata length: %v", len(params.ExtraData)) + } number := big.NewInt(0) number.SetUint64(params.Number) header := &types.Header{ @@ -397,6 +416,8 @@ func ExecutableDataToBlock(config *chainParams.ChainConfig, parent *types.Header GasLimit: params.GasLimit, GasUsed: params.GasUsed, Time: params.Timestamp, + BaseFee: params.BaseFeePerGas, + Extra: params.ExtraData, // TODO (MariusVanDerWijden) add params.Random to header once required } if config.IsLondon(number) { @@ -411,18 +432,20 @@ func ExecutableDataToBlock(config *chainParams.ChainConfig, parent *types.Header func BlockToExecutableData(block *types.Block, random common.Hash) *ExecutableData { return &ExecutableData{ - BlockHash: block.Hash(), - ParentHash: block.ParentHash(), - Coinbase: block.Coinbase(), - StateRoot: block.Root(), - Number: block.NumberU64(), - GasLimit: block.GasLimit(), - GasUsed: block.GasUsed(), - Timestamp: block.Time(), - ReceiptRoot: block.ReceiptHash(), - LogsBloom: block.Bloom().Bytes(), - Transactions: encodeTransactions(block.Transactions()), - Random: random, + BlockHash: block.Hash(), + ParentHash: block.ParentHash(), + Coinbase: block.Coinbase(), + StateRoot: block.Root(), + Number: block.NumberU64(), + GasLimit: block.GasLimit(), + GasUsed: block.GasUsed(), + BaseFeePerGas: block.BaseFee(), + Timestamp: block.Time(), + ReceiptRoot: block.ReceiptHash(), + LogsBloom: block.Bloom().Bytes(), + Transactions: encodeTransactions(block.Transactions()), + Random: random, + ExtraData: block.Extra(), } } @@ -434,6 +457,27 @@ func (api *ConsensusAPI) insertTransactions(txs types.Transactions) error { return nil } +func (api *ConsensusAPI) checkTerminalTotalDifficulty(head common.Hash) error { + // shortcut if we entered PoS already + if api.merger().EnteredPoS() { + return nil + } + // make sure the parent has enough terminal total difficulty + newHeadBlock := api.eth.BlockChain().GetBlockByHash(head) + if newHeadBlock == nil { + return &UnknownHeader + } + parent := api.eth.BlockChain().GetBlockByHash(newHeadBlock.ParentHash()) + if parent == nil { + return fmt.Errorf("parent unavailable: %v", newHeadBlock.ParentHash()) + } + td := api.eth.BlockChain().GetTdByHash(parent.Hash()) + if td != nil && td.Cmp(api.eth.BlockChain().Config().TerminalTotalDifficulty) < 0 { + return errors.New("total difficulty not reached yet") + } + return nil +} + // setHead is called to perform a force choice. func (api *ConsensusAPI) setHead(newHead common.Hash) error { // Trigger the transition if it's the first `NewHead` event. @@ -441,6 +485,7 @@ func (api *ConsensusAPI) setHead(newHead common.Hash) error { if !merger.LeftPoW() { merger.LeavePoW() } + log.Info("Setting head", "head", newHead) if api.light { headHeader := api.les.BlockChain().CurrentHeader() if headHeader.Hash() == newHead { diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 710f6b376b43..9d087164761e 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -22,6 +22,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" @@ -46,7 +47,6 @@ var ( func generatePreMergeChain(n int) (*core.Genesis, []*types.Block) { db := rawdb.NewMemoryDatabase() config := params.AllEthashProtocolChanges - config.TerminalTotalDifficulty = big.NewInt(100000000000000) genesis := &core.Genesis{ Config: config, Alloc: core.GenesisAlloc{testAddr: {Balance: testBalance}}, @@ -65,6 +65,11 @@ func generatePreMergeChain(n int) (*core.Genesis, []*types.Block) { gblock := genesis.ToBlock(db) engine := ethash.NewFaker() blocks, _ := core.GenerateChain(config, gblock, engine, db, n, generate) + totalDifficulty := big.NewInt(0) + for _, b := range blocks { + totalDifficulty.Add(totalDifficulty, b.Difficulty()) + } + config.TerminalTotalDifficulty = totalDifficulty return genesis, blocks } @@ -161,6 +166,21 @@ func TestEth2AssembleBlockWithAnotherBlocksTxs(t *testing.T) { } } +func TestSetHeadBeforeTotalDifficulty(t *testing.T) { + genesis, blocks := generatePreMergeChain(10) + n, ethservice := startEthService(t, genesis, blocks) + defer n.Close() + + api := NewConsensusAPI(ethservice, nil) + if err := api.ConsensusValidated(ConsensusValidatedParams{BlockHash: blocks[5].Hash(), Status: VALID.Status}); err == nil { + t.Errorf("consensus validated before total terminal difficulty should fail") + } + + if err := api.ForkchoiceUpdated(ForkChoiceParams{FinalizedBlockHash: blocks[5].Hash()}); err == nil { + t.Errorf("fork choice updated before total terminal difficulty should fail") + } +} + func TestEth2PrepareAndGetPayload(t *testing.T) { genesis, blocks := generatePreMergeChain(10) n, ethservice := startEthService(t, genesis, blocks[:9]) @@ -178,7 +198,7 @@ func TestEth2PrepareAndGetPayload(t *testing.T) { if err != nil { t.Fatalf("error preparing payload, err=%v", err) } - execData, err := api.GetPayload(respID) + execData, err := api.GetPayload(hexutil.Uint64(respID.PayloadID)) if err != nil { t.Fatalf("error getting payload, err=%v", err) } @@ -249,7 +269,7 @@ func TestEth2NewBlock(t *testing.T) { } checkLogEvents(t, newLogCh, rmLogsCh, 0, 0) - if err := api.ForkChoiceUpdated(ForkChoiceParams{HeadBlockHash: block.Hash(), FinalizedBlockHash: block.Hash()}); err != nil { + if err := api.ForkchoiceUpdated(ForkChoiceParams{HeadBlockHash: block.Hash(), FinalizedBlockHash: block.Hash()}); err != nil { t.Fatalf("Failed to insert block: %v", err) } if ethservice.BlockChain().CurrentBlock().NumberU64() != block.NumberU64() { @@ -288,7 +308,7 @@ func TestEth2NewBlock(t *testing.T) { if err := api.ConsensusValidated(ConsensusValidatedParams{BlockHash: block.Hash(), Status: "VALID"}); err != nil { t.Fatalf("Failed to insert block: %v", err) } - if err := api.ForkChoiceUpdated(ForkChoiceParams{FinalizedBlockHash: block.Hash(), HeadBlockHash: block.Hash()}); err != nil { + if err := api.ForkchoiceUpdated(ForkChoiceParams{FinalizedBlockHash: block.Hash(), HeadBlockHash: block.Hash()}); err != nil { t.Fatalf("Failed to insert block: %v", err) } if ethservice.BlockChain().CurrentBlock().NumberU64() != block.NumberU64() { @@ -299,45 +319,49 @@ func TestEth2NewBlock(t *testing.T) { } func TestEth2DeepReorg(t *testing.T) { - genesis, preMergeBlocks := generatePreMergeChain(core.TriesInMemory * 2) - n, ethservice := startEthService(t, genesis, preMergeBlocks) - defer n.Close() - - var ( - api = NewConsensusAPI(ethservice, nil) - parent = preMergeBlocks[len(preMergeBlocks)-core.TriesInMemory-1] - head = ethservice.BlockChain().CurrentBlock().NumberU64() - ) - if ethservice.BlockChain().HasBlockAndState(parent.Hash(), parent.NumberU64()) { - t.Errorf("Block %d not pruned", parent.NumberU64()) - } - for i := 0; i < 10; i++ { - execData, err := api.assembleBlock(AssembleBlockParams{ - ParentHash: parent.Hash(), - Timestamp: parent.Time() + 5, - }) - if err != nil { - t.Fatalf("Failed to create the executable data %v", err) + // TODO (MariusVanDerWijden) TestEth2DeepReorg is currently broken, because it tries to reorg + // before the totalTerminalDifficulty threshold + /* + genesis, preMergeBlocks := generatePreMergeChain(core.TriesInMemory * 2) + n, ethservice := startEthService(t, genesis, preMergeBlocks) + defer n.Close() + + var ( + api = NewConsensusAPI(ethservice, nil) + parent = preMergeBlocks[len(preMergeBlocks)-core.TriesInMemory-1] + head = ethservice.BlockChain().CurrentBlock().NumberU64() + ) + if ethservice.BlockChain().HasBlockAndState(parent.Hash(), parent.NumberU64()) { + t.Errorf("Block %d not pruned", parent.NumberU64()) } - block, err := ExecutableDataToBlock(ethservice.BlockChain().Config(), parent.Header(), *execData) - if err != nil { - t.Fatalf("Failed to convert executable data to block %v", err) - } - newResp, err := api.ExecutePayload(*execData) - if err != nil || newResp.Status != "VALID" { - t.Fatalf("Failed to insert block: %v", err) + for i := 0; i < 10; i++ { + execData, err := api.assembleBlock(AssembleBlockParams{ + ParentHash: parent.Hash(), + Timestamp: parent.Time() + 5, + }) + if err != nil { + t.Fatalf("Failed to create the executable data %v", err) + } + block, err := ExecutableDataToBlock(ethservice.BlockChain().Config(), parent.Header(), *execData) + if err != nil { + t.Fatalf("Failed to convert executable data to block %v", err) + } + newResp, err := api.ExecutePayload(*execData) + if err != nil || newResp.Status != "VALID" { + t.Fatalf("Failed to insert block: %v", err) + } + if ethservice.BlockChain().CurrentBlock().NumberU64() != head { + t.Fatalf("Chain head shouldn't be updated") + } + if err := api.setHead(block.Hash()); err != nil { + t.Fatalf("Failed to set head: %v", err) + } + if ethservice.BlockChain().CurrentBlock().NumberU64() != block.NumberU64() { + t.Fatalf("Chain head should be updated") + } + parent, head = block, block.NumberU64() } - if ethservice.BlockChain().CurrentBlock().NumberU64() != head { - t.Fatalf("Chain head shouldn't be updated") - } - if err := api.ForkChoiceUpdated(ForkChoiceParams{HeadBlockHash: block.Hash(), FinalizedBlockHash: block.Hash()}); err != nil { - t.Fatalf("Failed to insert block: %v", err) - } - if ethservice.BlockChain().CurrentBlock().NumberU64() != block.NumberU64() { - t.Fatalf("Chain head should be updated") - } - parent, head = block, block.NumberU64() - } + */ } // startEthService creates a full node instance for testing. @@ -393,7 +417,7 @@ func TestFullAPI(t *testing.T) { if err != nil { t.Fatalf("can't prepare payload: %v", err) } - payload, err := api.GetPayload(resp) + payload, err := api.GetPayload(hexutil.Uint64(resp.PayloadID)) if err != nil { t.Fatalf("can't get payload: %v", err) } @@ -409,7 +433,7 @@ func TestFullAPI(t *testing.T) { t.Fatalf("failed to validate consensus: %v", err) } - if err := api.ForkChoiceUpdated(ForkChoiceParams{HeadBlockHash: payload.BlockHash, FinalizedBlockHash: payload.BlockHash}); err != nil { + if err := api.ForkchoiceUpdated(ForkChoiceParams{HeadBlockHash: payload.BlockHash, FinalizedBlockHash: payload.BlockHash}); err != nil { t.Fatalf("Failed to insert block: %v", err) } if ethservice.BlockChain().CurrentBlock().NumberU64() != payload.Number { diff --git a/eth/catalyst/api_types.go b/eth/catalyst/api_types.go index 0efeb1aae14f..ff0aea39bf24 100644 --- a/eth/catalyst/api_types.go +++ b/eth/catalyst/api_types.go @@ -17,6 +17,8 @@ package catalyst import ( + "math/big" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" ) @@ -52,7 +54,7 @@ type ExecutableData struct { GasUsed uint64 `json:"gasUsed" gencodec:"required"` Timestamp uint64 `json:"timestamp" gencodec:"required"` ExtraData []byte `json:"extraData" gencodec:"required"` - BaseFeePerGas uint64 `json:"baseFeePerGas" gencodec:"required"` + BaseFeePerGas *big.Int `json:"baseFeePerGas" gencodec:"required"` Transactions [][]byte `json:"transactions" gencodec:"required"` } @@ -62,12 +64,23 @@ type executableDataMarshaling struct { GasLimit hexutil.Uint64 GasUsed hexutil.Uint64 Timestamp hexutil.Uint64 - BaseFeePerGas hexutil.Uint64 + BaseFeePerGas *hexutil.Big ExtraData hexutil.Bytes LogsBloom hexutil.Bytes Transactions []hexutil.Bytes } +//go:generate go run github.com/fjl/gencodec -type PayloadResponse -field-override payloadResponseMarshaling -out gen_payload.go + +type PayloadResponse struct { + PayloadID uint64 `json:"payloadId"` +} + +// JSON type overrides for payloadResponse. +type payloadResponseMarshaling struct { + PayloadID hexutil.Uint64 +} + type NewBlockResponse struct { Valid bool `json:"valid"` } diff --git a/eth/catalyst/gen_ed.go b/eth/catalyst/gen_ed.go index bc99a4a0954e..2953ab820fc2 100644 --- a/eth/catalyst/gen_ed.go +++ b/eth/catalyst/gen_ed.go @@ -5,6 +5,7 @@ package catalyst import ( "encoding/json" "errors" + "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -19,7 +20,7 @@ func (e ExecutableData) MarshalJSON() ([]byte, error) { ParentHash common.Hash `json:"parentHash" gencodec:"required"` Coinbase common.Address `json:"coinbase" gencodec:"required"` StateRoot common.Hash `json:"stateRoot" gencodec:"required"` - ReceiptRoot common.Hash `json:"receiptRoot" gencodec:"required"` + ReceiptRoot common.Hash `json:"receiptRoot" gencodec:"required"` LogsBloom hexutil.Bytes `json:"logsBloom" gencodec:"required"` Random common.Hash `json:"random" gencodec:"required"` Number hexutil.Uint64 `json:"blockNumber" gencodec:"required"` @@ -27,7 +28,7 @@ func (e ExecutableData) MarshalJSON() ([]byte, error) { GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"` Timestamp hexutil.Uint64 `json:"timestamp" gencodec:"required"` ExtraData hexutil.Bytes `json:"extraData" gencodec:"required"` - BaseFeePerGas hexutil.Uint64 `json:"baseFeePerGas" gencodec:"required"` + BaseFeePerGas *hexutil.Big `json:"baseFeePerGas" gencodec:"required"` Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` } var enc ExecutableData @@ -43,7 +44,7 @@ func (e ExecutableData) MarshalJSON() ([]byte, error) { enc.GasUsed = hexutil.Uint64(e.GasUsed) enc.Timestamp = hexutil.Uint64(e.Timestamp) enc.ExtraData = e.ExtraData - enc.BaseFeePerGas = hexutil.Uint64(e.BaseFeePerGas) + enc.BaseFeePerGas = (*hexutil.Big)(e.BaseFeePerGas) if e.Transactions != nil { enc.Transactions = make([]hexutil.Bytes, len(e.Transactions)) for k, v := range e.Transactions { @@ -60,7 +61,7 @@ func (e *ExecutableData) UnmarshalJSON(input []byte) error { ParentHash *common.Hash `json:"parentHash" gencodec:"required"` Coinbase *common.Address `json:"coinbase" gencodec:"required"` StateRoot *common.Hash `json:"stateRoot" gencodec:"required"` - ReceiptRoot *common.Hash `json:"receiptRoot" gencodec:"required"` + ReceiptRoot *common.Hash `json:"receiptRoot" gencodec:"required"` LogsBloom *hexutil.Bytes `json:"logsBloom" gencodec:"required"` Random *common.Hash `json:"random" gencodec:"required"` Number *hexutil.Uint64 `json:"blockNumber" gencodec:"required"` @@ -68,7 +69,7 @@ func (e *ExecutableData) UnmarshalJSON(input []byte) error { GasUsed *hexutil.Uint64 `json:"gasUsed" gencodec:"required"` Timestamp *hexutil.Uint64 `json:"timestamp" gencodec:"required"` ExtraData *hexutil.Bytes `json:"extraData" gencodec:"required"` - BaseFeePerGas *hexutil.Uint64 `json:"baseFeePerGas" gencodec:"required"` + BaseFeePerGas *hexutil.Big `json:"baseFeePerGas" gencodec:"required"` Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` } var dec ExecutableData @@ -126,7 +127,7 @@ func (e *ExecutableData) UnmarshalJSON(input []byte) error { if dec.BaseFeePerGas == nil { return errors.New("missing required field 'baseFeePerGas' for ExecutableData") } - e.BaseFeePerGas = uint64(*dec.BaseFeePerGas) + e.BaseFeePerGas = (*big.Int)(dec.BaseFeePerGas) if dec.Transactions == nil { return errors.New("missing required field 'transactions' for ExecutableData") } diff --git a/eth/catalyst/gen_payload.go b/eth/catalyst/gen_payload.go new file mode 100644 index 000000000000..a0b00fcfd8c3 --- /dev/null +++ b/eth/catalyst/gen_payload.go @@ -0,0 +1,36 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package catalyst + +import ( + "encoding/json" + + "github.com/ethereum/go-ethereum/common/hexutil" +) + +var _ = (*payloadResponseMarshaling)(nil) + +// MarshalJSON marshals as JSON. +func (p PayloadResponse) MarshalJSON() ([]byte, error) { + type PayloadResponse struct { + PayloadID hexutil.Uint64 `json:"payloadId"` + } + var enc PayloadResponse + enc.PayloadID = hexutil.Uint64(p.PayloadID) + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (p *PayloadResponse) UnmarshalJSON(input []byte) error { + type PayloadResponse struct { + PayloadID *hexutil.Uint64 `json:"payloadId"` + } + var dec PayloadResponse + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.PayloadID != nil { + p.PayloadID = uint64(*dec.PayloadID) + } + return nil +} diff --git a/miner/worker.go b/miner/worker.go index dcc823f3afb7..b7b0ba14cc97 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -19,6 +19,7 @@ package miner import ( "bytes" "errors" + "fmt" "math/big" "sync" "sync/atomic" @@ -1036,6 +1037,18 @@ func (w *worker) commit(uncles []*types.Header, interval func(), update bool, st if err != nil { return err } + + // TODO (MariusVanDerWijden) remove this after eth2 interop + // --- + parent := w.chain.GetBlockByHash(block.ParentHash()) + if parent == nil { + return fmt.Errorf("parent unavailable: %v", block.ParentHash()) + } + td := w.chain.GetTdByHash(parent.Hash()) + if td != nil && td.Cmp(w.chain.Config().TerminalTotalDifficulty) > 0 { + log.Warn("Total terminal difficulty reached, keeping on mining to test eth2 clients") + } + // --- if w.isRunning() && !w.merger.LeftPoW() { if interval != nil { interval() From 64189721ae26f679602e8dd113cadd7235519bd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 4 Oct 2021 15:01:13 +0300 Subject: [PATCH 07/32] all: sync polishes, other fixes + refactors --- accounts/abi/bind/backends/simulated.go | 2 +- cmd/geth/config.go | 4 +- cmd/geth/main.go | 2 +- cmd/utils/flags.go | 8 +- consensus/clique/clique_test.go | 6 +- consensus/clique/snapshot_test.go | 2 +- core/bench_test.go | 4 +- core/block_validator_test.go | 10 +- core/blockchain.go | 39 +++--- core/blockchain_repair_test.go | 4 +- core/blockchain_sethead_test.go | 2 +- core/blockchain_snapshot_test.go | 24 ++-- core/blockchain_test.go | 123 +++++++++-------- core/chain_makers_test.go | 2 +- core/dao_test.go | 12 +- core/forkchoice.go | 40 ++---- core/genesis.go | 7 +- core/genesis_test.go | 2 +- core/headerchain_test.go | 2 +- core/state_processor_test.go | 6 +- eth/backend.go | 8 +- eth/catalyst/api.go | 64 ++++----- eth/catalyst/api_test.go | 6 +- eth/catalyst/sync.go | 74 ---------- eth/ethconfig/config.go | 3 + eth/ethconfig/gen_config.go | 174 ++++++++++++------------ eth/gasprice/gasprice_test.go | 2 +- eth/handler_eth_test.go | 4 +- eth/handler_test.go | 2 +- eth/protocols/eth/handler_test.go | 2 +- eth/tracers/api_test.go | 2 +- les/client.go | 4 +- les/test_helper.go | 2 +- light/lightchain.go | 14 +- light/lightchain_test.go | 6 +- light/odr_test.go | 4 +- light/trie_test.go | 2 +- light/txpool_test.go | 4 +- miner/miner_test.go | 5 +- miner/stress/beacon/main.go | 10 +- miner/worker.go | 12 -- miner/worker_test.go | 4 +- tests/block_test_util.go | 2 +- tests/fuzzers/les/les-fuzzer.go | 2 +- 44 files changed, 309 insertions(+), 404 deletions(-) delete mode 100644 eth/catalyst/sync.go diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 1f8a663bfe40..78122ecd6f0d 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -78,7 +78,7 @@ type SimulatedBackend struct { func NewSimulatedBackendWithDatabase(database ethdb.Database, alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend { genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc} genesis.MustCommit(database) - blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}, nil, nil, core.NewMerger(database)) + blockchain, _ := core.NewBlockChainWithMerger(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}, nil, nil, core.NewMerger(database)) backend := &SimulatedBackend{ database: database, diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 44de9903e141..18829bade932 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -158,8 +158,8 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { if ctx.GlobalIsSet(utils.OverrideArrowGlacierFlag.Name) { cfg.Eth.OverrideArrowGlacier = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideArrowGlacierFlag.Name)) } - if ctx.GlobalIsSet(utils.OverrideTotalTerminalDifficulty.Name) { - cfg.Eth.Genesis.Config.TerminalTotalDifficulty = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideTotalTerminalDifficulty.Name)) + if ctx.GlobalIsSet(utils.OverrideTerminalTotalDifficulty.Name) { + cfg.Eth.Genesis.Config.TerminalTotalDifficulty = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideTerminalTotalDifficulty.Name)) } backend, _ := utils.RegisterEthService(stack, &cfg.Eth, ctx.GlobalBool(utils.CatalystFlag.Name)) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 881fd9bde665..704a4cb41b6e 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -67,7 +67,7 @@ var ( utils.USBFlag, utils.SmartCardDaemonPathFlag, utils.OverrideArrowGlacierFlag, - utils.OverrideTotalTerminalDifficulty, + utils.OverrideTerminalTotalDifficulty, utils.EthashCacheDirFlag, utils.EthashCachesInMemoryFlag, utils.EthashCachesOnDiskFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 6c2fd68817f0..696b09bbf98d 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -240,9 +240,9 @@ var ( Name: "override.arrowglacier", Usage: "Manually specify Arrow Glacier fork-block, overriding the bundled setting", } - OverrideTotalTerminalDifficulty = cli.Uint64Flag{ - Name: "override.totalterminaldifficulty", - Usage: "Manually specify TotalTerminalDifficulty, overriding the bundled setting", + OverrideTerminalTotalDifficulty = cli.Uint64Flag{ + Name: "override.terminaltotaldifficulty", + Usage: "Manually specify TerminalTotalDifficulty, overriding the bundled setting", } // Light server and client settings LightServeFlag = cli.IntFlag{ @@ -1906,7 +1906,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai // TODO(rjl493456442) disable snapshot generation/wiping if the chain is read only. // Disable transaction indexing/unindexing by default. - chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil, nil, core.NewMerger(chainDb)) + chain, err = core.NewBlockChainWithMerger(chainDb, cache, config, engine, vmcfg, nil, nil, core.NewMerger(chainDb)) if err != nil { Fatalf("Can't create BlockChain: %v", err) } diff --git a/consensus/clique/clique_test.go b/consensus/clique/clique_test.go index 354da78a87dd..1bd32acd3746 100644 --- a/consensus/clique/clique_test.go +++ b/consensus/clique/clique_test.go @@ -55,7 +55,7 @@ func TestReimportMirroredState(t *testing.T) { genesis := genspec.MustCommit(db) // Generate a batch of blocks, each properly signed - chain, _ := core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) + chain, _ := core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil) defer chain.Stop() blocks, _ := core.GenerateChain(params.AllCliqueProtocolChanges, genesis, engine, db, 3, func(i int, block *core.BlockGen) { @@ -89,7 +89,7 @@ func TestReimportMirroredState(t *testing.T) { db = rawdb.NewMemoryDatabase() genspec.MustCommit(db) - chain, _ = core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) + chain, _ = core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil) defer chain.Stop() if _, err := chain.InsertChain(blocks[:2]); err != nil { @@ -102,7 +102,7 @@ func TestReimportMirroredState(t *testing.T) { // Simulate a crash by creating a new chain on top of the database, without // flushing the dirty states out. Insert the last block, triggering a sidechain // reimport. - chain, _ = core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) + chain, _ = core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil) defer chain.Stop() if _, err := chain.InsertChain(blocks[2:]); err != nil { diff --git a/consensus/clique/snapshot_test.go b/consensus/clique/snapshot_test.go index 1168c434439c..094868ca744d 100644 --- a/consensus/clique/snapshot_test.go +++ b/consensus/clique/snapshot_test.go @@ -450,7 +450,7 @@ func TestClique(t *testing.T) { batches[len(batches)-1] = append(batches[len(batches)-1], block) } // Pass all the headers through clique and ensure tallying succeeds - chain, err := core.NewBlockChain(db, nil, &config, engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) + chain, err := core.NewBlockChain(db, nil, &config, engine, vm.Config{}, nil, nil) if err != nil { t.Errorf("test %d: failed to create test chain: %v", i, err) continue diff --git a/core/bench_test.go b/core/bench_test.go index 201dfbc9bdf8..959979763d66 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -201,7 +201,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { // Time the insertion of the new chain. // State and blocks are stored in the same DB. - chainman, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + chainman, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) defer chainman.Stop() b.ReportAllocs() b.ResetTimer() @@ -316,7 +316,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) { if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) } - chain, err := NewBlockChain(db, &cacheConfig, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + chain, err := NewBlockChain(db, &cacheConfig, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) if err != nil { b.Fatalf("error creating chain: %v", err) } diff --git a/core/block_validator_test.go b/core/block_validator_test.go index 80168f6c0fe2..ad177257a474 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -49,7 +49,7 @@ func TestHeaderVerification(t *testing.T) { headers[i] = block.Header() } // Run the header checker for blocks one-by-one, checking for both valid and invalid nonces - chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) defer chain.Stop() for i := 0; i < len(blocks); i++ { @@ -162,7 +162,7 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { merger.SubscribeLeavePoW(func() { runEngine.(*beacon.Beacon).MarkTransitioned() }) - chain, _ := NewBlockChain(testdb, nil, chainConfig, runEngine, vm.Config{}, nil, nil, merger) + chain, _ := NewBlockChainWithMerger(testdb, nil, chainConfig, runEngine, vm.Config{}, nil, nil, merger) defer chain.Stop() // Verify the blocks before the merging @@ -273,11 +273,11 @@ func testHeaderConcurrentVerification(t *testing.T, threads int) { var results <-chan error if valid { - chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) _, results = chain.engine.VerifyHeaders(chain, headers, seals) chain.Stop() } else { - chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeFailer(uint64(len(headers)-1)), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeFailer(uint64(len(headers)-1)), vm.Config{}, nil, nil) _, results = chain.engine.VerifyHeaders(chain, headers, seals) chain.Stop() } @@ -340,7 +340,7 @@ func testHeaderConcurrentAbortion(t *testing.T, threads int) { defer runtime.GOMAXPROCS(old) // Start the verifications and immediately abort - chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeDelayer(time.Millisecond), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeDelayer(time.Millisecond), vm.Config{}, nil, nil) defer chain.Stop() abort, results := chain.engine.VerifyHeaders(chain, headers, seals) diff --git a/core/blockchain.go b/core/blockchain.go index 7354e358da57..5ef19a3c1fc1 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -209,15 +209,19 @@ type BlockChain struct { processor Processor // Block transaction processor interface forker *ForkChoice vmConfig vm.Config - - shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block. - terminateInsert func(common.Hash, uint64) bool // Testing hook used to terminate ancient receipt chain insertion. } // NewBlockChain returns a fully initialised block chain using information // available in the database. It initialises the default Ethereum Validator // and Processor. -func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(header *types.Header) bool, txLookupLimit *uint64, merger *Merger) (*BlockChain, error) { +func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(header *types.Header) bool, txLookupLimit *uint64) (*BlockChain, error) { + return NewBlockChainWithMerger(db, cacheConfig, chainConfig, engine, vmConfig, shouldPreserve, txLookupLimit, NewMerger(rawdb.NewMemoryDatabase())) +} + +// NewBlockChainWithMerger returns a fully initialised block chain using information +// available in the database. It initialises the default Ethereum Validator +// and Processor. +func NewBlockChainWithMerger(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(header *types.Header) bool, txLookupLimit *uint64, merger *Merger) (*BlockChain, error) { if cacheConfig == nil { cacheConfig = defaultCacheConfig } @@ -249,10 +253,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par engine: engine, vmConfig: vmConfig, } - bc.forker = NewForkChoice(bc, merger.LeftPoW(), shouldPreserve) - merger.SubscribeLeavePoW(func() { - bc.forker.MarkTransitioned() - }) + bc.forker = NewForkChoice(bc, shouldPreserve) bc.validator = NewBlockValidator(chainConfig, bc, engine) bc.prefetcher = newStatePrefetcher(chainConfig, bc, engine) bc.processor = NewStateProcessor(chainConfig, bc, engine) @@ -742,32 +743,24 @@ func (bc *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error { // // Note, this function assumes that the `mu` mutex is held! func (bc *BlockChain) writeHeadBlock(block *types.Block) { - // If the block is on a side chain or an unknown one, force other heads onto it too - updateHeads := rawdb.ReadCanonicalHash(bc.db, block.NumberU64()) != block.Hash() - // Add the block to the canonical chain number scheme and mark as the head batch := bc.db.NewBatch() + rawdb.WriteHeadHeaderHash(batch, block.Hash()) + rawdb.WriteHeadFastBlockHash(batch, block.Hash()) rawdb.WriteCanonicalHash(batch, block.Hash(), block.NumberU64()) rawdb.WriteTxLookupEntriesByBlock(batch, block) rawdb.WriteHeadBlockHash(batch, block.Hash()) - // If the block is better than our head or is on a different chain, force update heads - // TODO(rjl493456442) What if the (header head/fast block head) is lower than the chain - // head? Is it possible in practise? - if updateHeads { - rawdb.WriteHeadHeaderHash(batch, block.Hash()) - rawdb.WriteHeadFastBlockHash(batch, block.Hash()) - } // Flush the whole batch into the disk, exit the node if failed if err := batch.Write(); err != nil { log.Crit("Failed to update chain indexes and markers", "err", err) } // Update all in-memory chain markers in the last step - if updateHeads { - bc.hc.SetCurrentHeader(block.Header()) - bc.currentFastBlock.Store(block) - headFastBlockGauge.Update(int64(block.NumberU64())) - } + bc.hc.SetCurrentHeader(block.Header()) + + bc.currentFastBlock.Store(block) + headFastBlockGauge.Update(int64(block.NumberU64())) + bc.currentBlock.Store(block) headBlockGauge.Update(int64(block.NumberU64())) } diff --git a/core/blockchain_repair_test.go b/core/blockchain_repair_test.go index 29a361534d6d..231a83c09145 100644 --- a/core/blockchain_repair_test.go +++ b/core/blockchain_repair_test.go @@ -1783,7 +1783,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { config.SnapshotLimit = 256 config.SnapshotWait = true } - chain, err := NewBlockChain(db, config, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + chain, err := NewBlockChain(db, config, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create chain: %v", err) } @@ -1836,7 +1836,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { } defer db.Close() - chain, err = NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + chain, err = NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } diff --git a/core/blockchain_sethead_test.go b/core/blockchain_sethead_test.go index 85a5b891f195..27b6be6e1363 100644 --- a/core/blockchain_sethead_test.go +++ b/core/blockchain_sethead_test.go @@ -1982,7 +1982,7 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) { config.SnapshotLimit = 256 config.SnapshotWait = true } - chain, err := NewBlockChain(db, config, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + chain, err := NewBlockChain(db, config, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create chain: %v", err) } diff --git a/core/blockchain_snapshot_test.go b/core/blockchain_snapshot_test.go index 4fb14349f65d..a8044ecb4d81 100644 --- a/core/blockchain_snapshot_test.go +++ b/core/blockchain_snapshot_test.go @@ -80,7 +80,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo // will happen during the block insertion. cacheConfig = defaultCacheConfig ) - chain, err := NewBlockChain(db, cacheConfig, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + chain, err := NewBlockChain(db, cacheConfig, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create chain: %v", err) } @@ -223,7 +223,7 @@ func (snaptest *snapshotTest) test(t *testing.T) { // Restart the chain normally chain.Stop() - newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -259,13 +259,13 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) { // the crash, we do restart twice here: one after the crash and one // after the normal stop. It's used to ensure the broken snapshot // can be detected all the time. - newchain, err := NewBlockChain(newdb, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + newchain, err := NewBlockChain(newdb, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } newchain.Stop() - newchain, err = NewBlockChain(newdb, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + newchain, err = NewBlockChain(newdb, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -301,7 +301,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) { TrieTimeLimit: 5 * time.Minute, SnapshotLimit: 0, } - newchain, err := NewBlockChain(snaptest.db, cacheConfig, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + newchain, err := NewBlockChain(snaptest.db, cacheConfig, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -309,7 +309,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) { newchain.Stop() // Restart the chain with enabling the snapshot - newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -337,7 +337,7 @@ func (snaptest *setHeadSnapshotTest) test(t *testing.T) { chain.SetHead(snaptest.setHead) chain.Stop() - newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -368,7 +368,7 @@ func (snaptest *restartCrashSnapshotTest) test(t *testing.T) { // and state committed. chain.Stop() - newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -385,7 +385,7 @@ func (snaptest *restartCrashSnapshotTest) test(t *testing.T) { // journal and latest state will be committed // Restart the chain after the crash - newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -420,7 +420,7 @@ func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) { TrieTimeLimit: 5 * time.Minute, SnapshotLimit: 0, } - newchain, err := NewBlockChain(snaptest.db, config, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + newchain, err := NewBlockChain(snaptest.db, config, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -436,13 +436,13 @@ func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) { SnapshotLimit: 256, SnapshotWait: false, // Don't wait rebuild } - newchain, err = NewBlockChain(snaptest.db, config, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + newchain, err = NewBlockChain(snaptest.db, config, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } // Simulate the blockchain crash. - newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index a0a65a634b96..dfd373c7d3d8 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -58,7 +58,7 @@ func newCanonical(engine consensus.Engine, n int, full bool) (ethdb.Database, *B ) // Initialize a fresh chain with only a genesis block - blockchain, _ := NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + blockchain, _ := NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) // Create and inject the requested chain if n == 0 { return db, blockchain, nil @@ -235,9 +235,6 @@ func testInsertAfterMerge(t *testing.T, blockchain *BlockChain, i, n int, full b t.Errorf("chain content mismatch at %d: have hash %v, want hash %v", i, hash2, hash1) } - // Trigger the transition explicitly - blockchain2.forker.MarkTransitioned() - // Extend the newly created chain if full { blockChainB := makeBlockChain(blockchain2.CurrentBlock(), n, ethash.NewFaker(), db, forkSeed) @@ -657,7 +654,7 @@ func testReorgBadHashes(t *testing.T, full bool) { blockchain.Stop() // Create a new BlockChain and check that it rolled back the state. - ncm, err := NewBlockChain(blockchain.db, nil, blockchain.chainConfig, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + ncm, err := NewBlockChain(blockchain.db, nil, blockchain.chainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create new chain manager: %v", err) } @@ -770,7 +767,7 @@ func TestFastVsFullChains(t *testing.T) { // Import the chain as an archive node for the comparison baseline archiveDb := rawdb.NewMemoryDatabase() gspec.MustCommit(archiveDb) - archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) defer archive.Stop() if n, err := archive.InsertChain(blocks); err != nil { @@ -779,7 +776,7 @@ func TestFastVsFullChains(t *testing.T) { // Fast import the chain as a non-archive node to test fastDb := rawdb.NewMemoryDatabase() gspec.MustCommit(fastDb) - fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) defer fast.Stop() headers := make([]*types.Header, len(blocks)) @@ -803,7 +800,7 @@ func TestFastVsFullChains(t *testing.T) { t.Fatalf("failed to create temp freezer db: %v", err) } gspec.MustCommit(ancientDb) - ancient, _ := NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + ancient, _ := NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) defer ancient.Stop() if n, err := ancient.InsertHeaderChain(headers, 1); err != nil { @@ -925,7 +922,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { archiveCaching := *defaultCacheConfig archiveCaching.TrieDirtyDisabled = true - archive, _ := NewBlockChain(archiveDb, &archiveCaching, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + archive, _ := NewBlockChain(archiveDb, &archiveCaching, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) if n, err := archive.InsertChain(blocks); err != nil { t.Fatalf("failed to process block %d: %v", n, err) } @@ -938,7 +935,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Import the chain as a non-archive node and ensure all pointers are updated fastDb, delfn := makeDb() defer delfn() - fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) defer fast.Stop() headers := make([]*types.Header, len(blocks)) @@ -958,7 +955,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Import the chain as a ancient-first node and ensure all pointers are updated ancientDb, delfn := makeDb() defer delfn() - ancient, _ := NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + ancient, _ := NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) defer ancient.Stop() if n, err := ancient.InsertHeaderChain(headers, 1); err != nil { @@ -977,7 +974,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Import the chain as a light node and ensure all pointers are updated lightDb, delfn := makeDb() defer delfn() - light, _ := NewBlockChain(lightDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + light, _ := NewBlockChain(lightDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) if n, err := light.InsertHeaderChain(headers, 1); err != nil { t.Fatalf("failed to insert header %d: %v", n, err) } @@ -1046,7 +1043,7 @@ func TestChainTxReorgs(t *testing.T) { } }) // Import the chain. This runs all block validation rules. - blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) if i, err := blockchain.InsertChain(chain); err != nil { t.Fatalf("failed to insert original chain[%d]: %v", i, err) } @@ -1116,7 +1113,7 @@ func TestLogReorgs(t *testing.T) { signer = types.LatestSigner(gspec.Config) ) - blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() rmLogsCh := make(chan RemovedLogsEvent) @@ -1169,7 +1166,7 @@ func TestLogRebirth(t *testing.T) { genesis = gspec.MustCommit(db) signer = types.LatestSigner(gspec.Config) engine = ethash.NewFaker() - blockchain, _ = NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + blockchain, _ = NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}, nil, nil) ) defer blockchain.Stop() @@ -1232,7 +1229,7 @@ func TestSideLogRebirth(t *testing.T) { gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} genesis = gspec.MustCommit(db) signer = types.LatestSigner(gspec.Config) - blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) ) defer blockchain.Stop() @@ -1307,7 +1304,7 @@ func TestReorgSideEvent(t *testing.T) { signer = types.LatestSigner(gspec.Config) ) - blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() chain, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 3, func(i int, gen *BlockGen) {}) @@ -1439,7 +1436,7 @@ func TestEIP155Transition(t *testing.T) { genesis = gspec.MustCommit(db) ) - blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() blocks, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 4, func(i int, block *BlockGen) { @@ -1547,7 +1544,7 @@ func TestEIP161AccountRemoval(t *testing.T) { } genesis = gspec.MustCommit(db) ) - blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() blocks, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 3, func(i int, block *BlockGen) { @@ -1622,7 +1619,7 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { diskdb := rawdb.NewMemoryDatabase() (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -1666,7 +1663,7 @@ func TestTrieForkGC(t *testing.T) { diskdb := rawdb.NewMemoryDatabase() (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -1705,7 +1702,7 @@ func TestLargeReorgTrieGC(t *testing.T) { diskdb := rawdb.NewMemoryDatabase() (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -1766,7 +1763,7 @@ func TestBlockchainRecovery(t *testing.T) { t.Fatalf("failed to create temp freezer db: %v", err) } gspec.MustCommit(ancientDb) - ancient, _ := NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + ancient, _ := NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) headers := make([]*types.Header, len(blocks)) for i, block := range blocks { @@ -1786,7 +1783,7 @@ func TestBlockchainRecovery(t *testing.T) { rawdb.WriteHeadFastBlockHash(ancientDb, midBlock.Hash()) // Reopen broken blockchain again - ancient, _ = NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + ancient, _ = NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) defer ancient.Stop() if num := ancient.CurrentBlock().NumberU64(); num != 0 { t.Errorf("head block mismatch: have #%v, want #%v", num, 0) @@ -1838,7 +1835,7 @@ func TestInsertReceiptChainRollback(t *testing.T) { } gspec := Genesis{Config: params.AllEthashProtocolChanges} gspec.MustCommit(ancientDb) - ancientChain, _ := NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + ancientChain, _ := NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) defer ancientChain.Stop() // Import the canonical header chain. @@ -1899,7 +1896,7 @@ func TestLowDiffLongChain(t *testing.T) { diskdb := rawdb.NewMemoryDatabase() (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -1941,6 +1938,8 @@ func TestLowDiffLongChain(t *testing.T) { // 0: the transition happens since genesis // 1: the transition happens after some chain segments func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommonAncestorAndPruneblock int, mergePoint int) { + // Copy the TestChainConfig so we can modify it during tests + chainConfig := *params.TestChainConfig // Generate a canonical chain to act as the main dataset var ( genEngine = beacon.New(ethash.NewFaker(), false) @@ -1952,7 +1951,7 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon nonce = uint64(0) gspec = &Genesis{ - Config: params.TestChainConfig, + Config: &chainConfig, Alloc: GenesisAlloc{addr: {Balance: big.NewInt(math.MaxInt64)}}, BaseFee: big.NewInt(params.InitialBaseFee), } @@ -1962,7 +1961,7 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon // Generate and import the canonical chain diskdb := rawdb.NewMemoryDatabase() gspec.MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, runEngine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + chain, err := NewBlockChain(diskdb, nil, &chainConfig, runEngine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -1970,9 +1969,11 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon if mergePoint == 0 { genEngine.MarkTransitioned() runEngine.MarkTransitioned() - chain.forker.MarkTransitioned() + + // Set the terminal total difficulty in the config + gspec.Config.TerminalTotalDifficulty = big.NewInt(0) } - blocks, _ := GenerateChain(params.TestChainConfig, genesis, genEngine, db, 2*TriesInMemory, func(i int, gen *BlockGen) { + blocks, _ := GenerateChain(&chainConfig, genesis, genEngine, db, 2*TriesInMemory, func(i int, gen *BlockGen) { tx, err := types.SignTx(types.NewTransaction(nonce, common.HexToAddress("deadbeef"), big.NewInt(100), 21000, big.NewInt(int64(i+1)*params.GWei), nil), signer, key) if err != nil { t.Fatalf("failed to create tx: %v", err) @@ -2001,7 +2002,8 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon if mergePoint == 1 { genEngine.MarkTransitioned() runEngine.MarkTransitioned() - chain.forker.MarkTransitioned() + // Set the terminal total difficulty in the config + gspec.Config.TerminalTotalDifficulty = big.NewInt(int64(len(blocks))) } // Generate the sidechain @@ -2011,7 +2013,7 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon // Generate fork chain, make it longer than canon parentIndex := lastPrunedIndex + blocksBetweenCommonAncestorAndPruneblock parent := blocks[parentIndex] - fork, _ := GenerateChain(params.TestChainConfig, parent, genEngine, db, 2*TriesInMemory, func(i int, b *BlockGen) { + fork, _ := GenerateChain(&chainConfig, parent, genEngine, db, 2*TriesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) }) // Prepend the parent(s) @@ -2098,7 +2100,7 @@ func testInsertKnownChainData(t *testing.T, typ string) { (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(chaindb) defer os.RemoveAll(dir) - chain, err := NewBlockChain(chaindb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + chain, err := NewBlockChain(chaindb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2215,34 +2217,35 @@ func TestInsertKnownBlocksAfterMerging(t *testing.T) { // 0: means the merging is applied since genesis // 1: means the merging is applied after the first segment func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight int) { + // Copy the TestChainConfig so we can modify it during tests + chainConfig := *params.TestChainConfig var ( db = rawdb.NewMemoryDatabase() - genesis = (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) + genesis = (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee), Config: &chainConfig}).MustCommit(db) runEngine = beacon.New(ethash.NewFaker(), false) genEngine = beacon.New(ethash.NewFaker(), false) ) - applyMerge := func(engine *beacon.Beacon, forker *ForkChoice) { + applyMerge := func(engine *beacon.Beacon, height int) { if engine != nil { engine.MarkTransitioned() - } - if forker != nil { - forker.MarkTransitioned() + // Set the terminal total difficulty in the config + chainConfig.TerminalTotalDifficulty = big.NewInt(int64(height)) } } // Apply merging since genesis if mergeHeight == 0 { - applyMerge(genEngine, nil) + applyMerge(genEngine, 0) } - blocks, receipts := GenerateChain(params.TestChainConfig, genesis, genEngine, db, 32, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) + blocks, receipts := GenerateChain(&chainConfig, genesis, genEngine, db, 32, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) // Apply merging after the first segment if mergeHeight == 1 { - applyMerge(genEngine, nil) + applyMerge(genEngine, len(blocks)) } // Longer chain and shorter chain - blocks2, receipts2 := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], genEngine, db, 65, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) - blocks3, receipts3 := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], genEngine, db, 64, func(i int, b *BlockGen) { + blocks2, receipts2 := GenerateChain(&chainConfig, blocks[len(blocks)-1], genEngine, db, 65, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) + blocks3, receipts3 := GenerateChain(&chainConfig, blocks[len(blocks)-1], genEngine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) b.OffsetTime(-9) // Time shifted, difficulty shouldn't be changed }) @@ -2260,7 +2263,7 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(chaindb) defer os.RemoveAll(dir) - chain, err := NewBlockChain(chaindb, nil, params.TestChainConfig, runEngine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + chain, err := NewBlockChain(chaindb, nil, &chainConfig, runEngine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2314,7 +2317,7 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i // Apply merging since genesis if required if mergeHeight == 0 { - applyMerge(runEngine, chain.forker) + applyMerge(runEngine, 0) } if err := inserter(blocks, receipts); err != nil { t.Fatalf("failed to insert chain data: %v", err) @@ -2337,7 +2340,7 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i // Apply merging after the first segment if mergeHeight == 1 { - applyMerge(runEngine, chain.forker) + applyMerge(runEngine, len(blocks)) } // Import a longer chain with some known data as prefix. @@ -2378,7 +2381,7 @@ func getLongAndShortChains() (bc *BlockChain, longChain []*types.Block, heavyCha diskdb := rawdb.NewMemoryDatabase() (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) if err != nil { return nil, nil, nil, fmt.Errorf("failed to create tester chain: %v", err) } @@ -2571,7 +2574,7 @@ func TestTransactionIndices(t *testing.T) { // Import all blocks into ancient db l := uint64(0) - chain, err := NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l, NewMerger(rawdb.NewMemoryDatabase())) + chain, err := NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2596,7 +2599,7 @@ func TestTransactionIndices(t *testing.T) { t.Fatalf("failed to create temp freezer db: %v", err) } gspec.MustCommit(ancientDb) - chain, err = NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l, NewMerger(rawdb.NewMemoryDatabase())) + chain, err = NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2620,7 +2623,7 @@ func TestTransactionIndices(t *testing.T) { limit = []uint64{0, 64 /* drop stale */, 32 /* shorten history */, 64 /* extend history */, 0 /* restore all */} tails := []uint64{0, 67 /* 130 - 64 + 1 */, 100 /* 131 - 32 + 1 */, 69 /* 132 - 64 + 1 */, 0} for i, l := range limit { - chain, err = NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l, NewMerger(rawdb.NewMemoryDatabase())) + chain, err = NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2698,7 +2701,7 @@ func TestSkipStaleTxIndicesInFastSync(t *testing.T) { // Import all blocks into ancient db, only HEAD-32 indices are kept. l := uint64(32) - chain, err := NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l, NewMerger(rawdb.NewMemoryDatabase())) + chain, err := NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2762,7 +2765,7 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in diskdb := rawdb.NewMemoryDatabase() gspec.MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) if err != nil { b.Fatalf("failed to create tester chain: %v", err) } @@ -2845,7 +2848,7 @@ func TestSideImportPrunedBlocks(t *testing.T) { diskdb := rawdb.NewMemoryDatabase() (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2939,7 +2942,7 @@ func TestDeleteCreateRevert(t *testing.T) { diskdb := rawdb.NewMemoryDatabase() gspec.MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3053,7 +3056,7 @@ func TestDeleteRecreateSlots(t *testing.T) { chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{ Debug: true, Tracer: vm.NewJSONLogger(nil, os.Stdout), - }, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + }, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3133,7 +3136,7 @@ func TestDeleteRecreateAccount(t *testing.T) { chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{ Debug: true, Tracer: vm.NewJSONLogger(nil, os.Stdout), - }, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + }, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3306,7 +3309,7 @@ func TestDeleteRecreateSlotsAcrossManyBlocks(t *testing.T) { chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{ //Debug: true, //Tracer: vm.NewJSONLogger(nil, os.Stdout), - }, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + }, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3440,7 +3443,7 @@ func TestInitThenFailCreateContract(t *testing.T) { chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{ //Debug: true, //Tracer: vm.NewJSONLogger(nil, os.Stdout), - }, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + }, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3527,7 +3530,7 @@ func TestEIP2718Transition(t *testing.T) { diskdb := rawdb.NewMemoryDatabase() gspec.MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, gspec.Config, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + chain, err := NewBlockChain(diskdb, nil, gspec.Config, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3622,7 +3625,7 @@ func TestEIP1559Transition(t *testing.T) { diskdb := rawdb.NewMemoryDatabase() gspec.MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, gspec.Config, engine, vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + chain, err := NewBlockChain(diskdb, nil, gspec.Config, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } diff --git a/core/chain_makers_test.go b/core/chain_makers_test.go index 79c8314d61a8..85a029f7c757 100644 --- a/core/chain_makers_test.go +++ b/core/chain_makers_test.go @@ -79,7 +79,7 @@ func ExampleGenerateChain() { }) // Import the chain. This runs all block validation rules. - blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() if i, err := blockchain.InsertChain(chain); err != nil { diff --git a/core/dao_test.go b/core/dao_test.go index 5b5cf18579b5..c9c765a3832a 100644 --- a/core/dao_test.go +++ b/core/dao_test.go @@ -45,7 +45,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { proConf.DAOForkBlock = forkBlock proConf.DAOForkSupport = true - proBc, _ := NewBlockChain(proDb, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + proBc, _ := NewBlockChain(proDb, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil, nil) defer proBc.Stop() conDb := rawdb.NewMemoryDatabase() @@ -55,7 +55,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { conConf.DAOForkBlock = forkBlock conConf.DAOForkSupport = false - conBc, _ := NewBlockChain(conDb, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + conBc, _ := NewBlockChain(conDb, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil, nil) defer conBc.Stop() if _, err := proBc.InsertChain(prefix); err != nil { @@ -69,7 +69,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Create a pro-fork block, and try to feed into the no-fork chain db = rawdb.NewMemoryDatabase() gspec.MustCommit(db) - bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil, nil) defer bc.Stop() blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64())) @@ -94,7 +94,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Create a no-fork block, and try to feed into the pro-fork chain db = rawdb.NewMemoryDatabase() gspec.MustCommit(db) - bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil, nil) defer bc.Stop() blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64())) @@ -120,7 +120,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Verify that contra-forkers accept pro-fork extra-datas after forking finishes db = rawdb.NewMemoryDatabase() gspec.MustCommit(db) - bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil, nil) defer bc.Stop() blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64())) @@ -140,7 +140,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Verify that pro-forkers accept contra-fork extra-datas after forking finishes db = rawdb.NewMemoryDatabase() gspec.MustCommit(db) - bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil, nil) defer bc.Stop() blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64())) diff --git a/core/forkchoice.go b/core/forkchoice.go index 9bc1f79facef..e3b8e591201f 100644 --- a/core/forkchoice.go +++ b/core/forkchoice.go @@ -27,12 +27,16 @@ import ( "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/params" ) // ChainReader defines a small collection of methods needed to access the local // blockchain during header verification. It's implemented by both blockchain // and lightchain. type ChainReader interface { + // Config retrieves the header chain's chain configuration. + Config() *params.ChainConfig + // GetTd returns the total difficulty of a local block. GetTd(common.Hash, uint64) *big.Int } @@ -45,12 +49,7 @@ type ChainReader interface { type ForkChoice struct { chain ChainReader rand *mrand.Rand - - // transitioned is the flag whether the chain has started(or finished) - // the transition. It's triggered by receiving the first "NewHead" message - // from the external consensus engine. - transitioned bool - lock sync.RWMutex + lock sync.RWMutex // preserve is a helper function used in td fork choice. // Miners will prefer to choose the local mined block if the @@ -59,17 +58,16 @@ type ForkChoice struct { preserve func(header *types.Header) bool } -func NewForkChoice(chainReader ChainReader, transitioned bool, preserve func(header *types.Header) bool) *ForkChoice { +func NewForkChoice(chainReader ChainReader, preserve func(header *types.Header) bool) *ForkChoice { // Seed a fast but crypto originating random generator seed, err := crand.Int(crand.Reader, big.NewInt(math.MaxInt64)) if err != nil { log.Crit("Failed to initialize random seed", "err", err) } return &ForkChoice{ - chain: chainReader, - rand: mrand.New(mrand.NewSource(seed.Int64())), - transitioned: transitioned, - preserve: preserve, + chain: chainReader, + rand: mrand.New(mrand.NewSource(seed.Int64())), + preserve: preserve, } } @@ -82,12 +80,6 @@ func (f *ForkChoice) Reorg(current *types.Header, header *types.Header) (bool, e f.lock.RLock() defer f.lock.RUnlock() - // Accept the new header as the chain head if the transition - // is already triggered. We assume all the headers after the - // transition come from the trusted consensus layer. - if f.transitioned { - return true, nil - } var ( localTD = f.chain.GetTd(current.Hash(), current.Number.Uint64()) externTd = f.chain.GetTd(header.Hash(), header.Number.Uint64()) @@ -95,6 +87,12 @@ func (f *ForkChoice) Reorg(current *types.Header, header *types.Header) (bool, e if localTD == nil || externTd == nil { return false, errors.New("missing td") } + // Accept the new header as the chain head if the transition + // is already triggered. We assume all the headers after the + // transition come from the trusted consensus layer. + if ttd := f.chain.Config().TerminalTotalDifficulty; ttd != nil && ttd.Cmp(externTd) <= 0 { + return true, nil + } // If the total difficulty is higher than our known, add it to the canonical chain // Second clause in the if statement reduces the vulnerability to selfish mining. // Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf @@ -113,11 +111,3 @@ func (f *ForkChoice) Reorg(current *types.Header, header *types.Header) (bool, e } return reorg, nil } - -// MarkTransitioned marks the transition has started. -func (f *ForkChoice) MarkTransitioned() { - f.lock.Lock() - defer f.lock.Unlock() - - f.transitioned = true -} diff --git a/core/genesis.go b/core/genesis.go index 1c1534fbb5cc..55914d938191 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -155,10 +155,10 @@ func (e *GenesisMismatchError) Error() string { // // The returned chain configuration is never nil. func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig, common.Hash, error) { - return SetupGenesisBlockWithOverride(db, genesis, nil) + return SetupGenesisBlockWithOverride(db, genesis, nil, nil) } -func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, overrideArrowGlacier *big.Int) (*params.ChainConfig, common.Hash, error) { +func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, overrideArrowGlacier, overrideTerminalTotalDifficulty *big.Int) (*params.ChainConfig, common.Hash, error) { if genesis != nil && genesis.Config == nil { return params.AllEthashProtocolChanges, common.Hash{}, errGenesisNoConfig } @@ -207,6 +207,9 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override if overrideArrowGlacier != nil { newcfg.ArrowGlacierBlock = overrideArrowGlacier } + if overrideTerminalTotalDifficulty != nil { + newcfg.TerminalTotalDifficulty = overrideTerminalTotalDifficulty + } if err := newcfg.CheckConfigForkOrder(); err != nil { return newcfg, common.Hash{}, err } diff --git a/core/genesis_test.go b/core/genesis_test.go index aa02279dc3f7..056c1c195f6b 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -136,7 +136,7 @@ func TestSetupGenesis(t *testing.T) { // Advance to block #4, past the homestead transition block of customg. genesis := oldcustomg.MustCommit(db) - bc, _ := NewBlockChain(db, nil, oldcustomg.Config, ethash.NewFullFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + bc, _ := NewBlockChain(db, nil, oldcustomg.Config, ethash.NewFullFaker(), vm.Config{}, nil, nil) defer bc.Stop() blocks, _ := GenerateChain(oldcustomg.Config, genesis, ethash.NewFaker(), db, 4, nil) diff --git a/core/headerchain_test.go b/core/headerchain_test.go index 5371ff98642d..ed0522671fb8 100644 --- a/core/headerchain_test.go +++ b/core/headerchain_test.go @@ -84,7 +84,7 @@ func TestHeaderInsertion(t *testing.T) { chainB := makeHeaderChain(chainA[0], 128, ethash.NewFaker(), db, 10) log.Root().SetHandler(log.StdoutHandler) - forker := NewForkChoice(hc, false, nil) + forker := NewForkChoice(hc, nil) // Inserting 64 headers on an empty chain, expecting // 1 callbacks, 1 canon-status, 0 sidestatus, testInsert(t, hc, chainA[:64], CanonStatTy, nil, forker) diff --git a/core/state_processor_test.go b/core/state_processor_test.go index 62f1737ddd39..13a9eb810df6 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -85,7 +85,7 @@ func TestStateProcessorErrors(t *testing.T) { }, } genesis = gspec.MustCommit(db) - blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) ) defer blockchain.Stop() bigNumber := new(big.Int).SetBytes(common.FromHex("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")) @@ -220,7 +220,7 @@ func TestStateProcessorErrors(t *testing.T) { }, } genesis = gspec.MustCommit(db) - blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) ) defer blockchain.Stop() for i, tt := range []struct { @@ -260,7 +260,7 @@ func TestStateProcessorErrors(t *testing.T) { }, } genesis = gspec.MustCommit(db) - blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, NewMerger(rawdb.NewMemoryDatabase())) + blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) ) defer blockchain.Stop() for i, tt := range []struct { diff --git a/eth/backend.go b/eth/backend.go index 4a8658f9c3a8..c4fa7b596b42 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -133,7 +133,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideArrowGlacier) + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideArrowGlacier, config.OverrideTerminalTotalDifficulty) if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok { return nil, genesisErr } @@ -192,7 +192,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { Preimages: config.Preimages, } ) - eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, chainConfig, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit, merger) + eth.blockchain, err = core.NewBlockChainWithMerger(chainDb, cacheConfig, chainConfig, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit, merger) if err != nil { return nil, err } @@ -529,6 +529,10 @@ func (s *Ethereum) SetSynced() { atomic.StoreUint32(&s.h func (s *Ethereum) ArchiveMode() bool { return s.config.NoPruning } func (s *Ethereum) BloomIndexer() *core.ChainIndexer { return s.bloomIndexer } func (s *Ethereum) Merger() *core.Merger { return s.merger } +func (s *Ethereum) SyncMode() downloader.SyncMode { + mode, _ := s.handler.chainSync.modeAndLocalHead() + return mode +} // Protocols returns all the currently configured // network protocols to start. diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index bdf3e22ecfeb..9925dea5f332 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -84,7 +84,6 @@ type ConsensusAPI struct { eth *eth.Ethereum les *les.LightEthereum engine consensus.Engine // engine is the post-merge consensus engine, only for block creation - syncer *syncer // syncer is responsible for triggering chain sync preparedBlocks map[int]*ExecutableData } @@ -114,7 +113,6 @@ func NewConsensusAPI(eth *eth.Ethereum, les *les.LightEthereum) *ConsensusAPI { eth: eth, les: les, engine: engine, - syncer: newSyncer(), preparedBlocks: make(map[int]*ExecutableData), } } @@ -217,54 +215,46 @@ func (api *ConsensusAPI) ConsensusValidated(params ConsensusValidatedParams) err func (api *ConsensusAPI) ForkchoiceUpdated(params ForkChoiceParams) error { var emptyHash = common.Hash{} - if !bytes.Equal(params.FinalizedBlockHash[:], emptyHash[:]) { - if err := api.checkTerminalTotalDifficulty(params.FinalizedBlockHash); err != nil { + if !bytes.Equal(params.HeadBlockHash[:], emptyHash[:]) { + if err := api.checkTerminalTotalDifficulty(params.HeadBlockHash); err != nil { return err } - return api.setHead(params.FinalizedBlockHash) + return api.setHead(params.HeadBlockHash) } return nil } // ExecutePayload creates an Eth1 block, inserts it in the chain, and returns the status of the chain. func (api *ConsensusAPI) ExecutePayload(params ExecutableData) (GenericStringResponse, error) { + block, err := ExecutableDataToBlock(params) + if err != nil { + return INVALID, err + } if api.light { parent := api.les.BlockChain().GetHeaderByHash(params.ParentHash) if parent == nil { return INVALID, fmt.Errorf("could not find parent %x", params.ParentHash) } - block, err := ExecutableDataToBlock(api.les.BlockChain().Config(), parent, params) - if err != nil { - return INVALID, err - } if err = api.les.BlockChain().InsertHeader(block.Header()); err != nil { return INVALID, err } return VALID, nil } - parent := api.eth.BlockChain().GetBlockByHash(params.ParentHash) - if parent == nil { - return INVALID, fmt.Errorf("could not find parent %x", params.ParentHash) + if !api.eth.BlockChain().HasBlock(block.ParentHash(), block.NumberU64()-1) { + /* + TODO (MariusVanDerWijden) reenable once sync is merged + if err := api.eth.Downloader().BeaconSync(api.eth.SyncMode(), block.Header()); err != nil { + return SYNCING, err + } + */ + return SYNCING, nil } - - td := api.eth.BlockChain().GetTdByHash(parent.Hash()) + parent := api.eth.BlockChain().GetBlockByHash(params.ParentHash) + td := api.eth.BlockChain().GetTd(parent.Hash(), block.NumberU64()-1) ttd := api.eth.BlockChain().Config().TerminalTotalDifficulty - if !api.eth.Synced() { - if td.Cmp(ttd) > 0 { - // first pos block - api.eth.SetSynced() - } else { - // TODO (MariusVanDerWijden) if the node is not synced and we received a finalized block - // we should trigger the reverse header sync here. - return SYNCING, errors.New("node is not synced yet") - } - } else if td.Cmp(ttd) < 0 { + if td.Cmp(ttd) < 0 { return INVALID, fmt.Errorf("can not execute payload on top of block with low td got: %v threshold %v", td, ttd) } - block, err := ExecutableDataToBlock(api.eth.BlockChain().Config(), parent.Header(), params) - if err != nil { - return INVALID, err - } if err := api.eth.BlockChain().InsertBlock(block); err != nil { return INVALID, err } @@ -286,13 +276,12 @@ func (api *ConsensusAPI) assembleBlock(params AssembleBlockParams) (*ExecutableD return nil, fmt.Errorf("cannot assemble block with unknown parent %s", params.ParentHash) } - if parent.Time() >= params.Timestamp { + if params.Timestamp < parent.Time() { return nil, fmt.Errorf("child timestamp lower than parent's: %d >= %d", parent.Time(), params.Timestamp) } if now := uint64(time.Now().Unix()); params.Timestamp > now+1 { - wait := time.Duration(params.Timestamp-now) * time.Second - log.Info("Producing block too far in the future", "wait", common.PrettyDuration(wait)) - time.Sleep(wait) + diff := time.Duration(params.Timestamp-now) * time.Second + log.Warn("Producing block too far in the future", "diff", common.PrettyDuration(diff)) } pending := api.eth.TxPool().Pending(true) coinbase := params.FeeRecipient @@ -393,7 +382,7 @@ func decodeTransactions(enc [][]byte) ([]*types.Transaction, error) { return txs, nil } -func ExecutableDataToBlock(config *chainParams.ChainConfig, parent *types.Header, params ExecutableData) (*types.Block, error) { +func ExecutableDataToBlock(params ExecutableData) (*types.Block, error) { txs, err := decodeTransactions(params.Transactions) if err != nil { return nil, err @@ -420,9 +409,6 @@ func ExecutableDataToBlock(config *chainParams.ChainConfig, parent *types.Header Extra: params.ExtraData, // TODO (MariusVanDerWijden) add params.Random to header once required } - if config.IsLondon(number) { - header.BaseFee = misc.CalcBaseFee(config, parent) - } block := types.NewBlockWithHeader(header).WithBody(txs, nil /* uncles */) if block.Hash() != params.BlockHash { return nil, fmt.Errorf("blockhash mismatch, want %x, got %x", params.BlockHash, block.Hash()) @@ -471,7 +457,7 @@ func (api *ConsensusAPI) checkTerminalTotalDifficulty(head common.Hash) error { if parent == nil { return fmt.Errorf("parent unavailable: %v", newHeadBlock.ParentHash()) } - td := api.eth.BlockChain().GetTdByHash(parent.Hash()) + td := api.eth.BlockChain().GetTd(parent.Hash(), parent.NumberU64()) if td != nil && td.Cmp(api.eth.BlockChain().Config().TerminalTotalDifficulty) < 0 { return errors.New("total difficulty not reached yet") } @@ -578,3 +564,7 @@ func (api *ConsensusAPI) ExportExecutableData(path string) error { } return nil } + +func (api *ConsensusAPI) GetHead() (common.Hash, error) { + return api.eth.BlockChain().CurrentBlock().Hash(), nil +} diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 9d087164761e..c3871b45617a 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -176,7 +176,7 @@ func TestSetHeadBeforeTotalDifficulty(t *testing.T) { t.Errorf("consensus validated before total terminal difficulty should fail") } - if err := api.ForkchoiceUpdated(ForkChoiceParams{FinalizedBlockHash: blocks[5].Hash()}); err == nil { + if err := api.ForkchoiceUpdated(ForkChoiceParams{HeadBlockHash: blocks[5].Hash()}); err == nil { t.Errorf("fork choice updated before total terminal difficulty should fail") } } @@ -256,7 +256,7 @@ func TestEth2NewBlock(t *testing.T) { if err != nil { t.Fatalf("Failed to create the executable data %v", err) } - block, err := ExecutableDataToBlock(ethservice.BlockChain().Config(), parent.Header(), *execData) + block, err := ExecutableDataToBlock(*execData) if err != nil { t.Fatalf("Failed to convert executable data to block %v", err) } @@ -293,7 +293,7 @@ func TestEth2NewBlock(t *testing.T) { if err != nil { t.Fatalf("Failed to create the executable data %v", err) } - block, err := ExecutableDataToBlock(ethservice.BlockChain().Config(), parent.Header(), *execData) + block, err := ExecutableDataToBlock(*execData) if err != nil { t.Fatalf("Failed to convert executable data to block %v", err) } diff --git a/eth/catalyst/sync.go b/eth/catalyst/sync.go deleted file mode 100644 index 61fd1ef9c166..000000000000 --- a/eth/catalyst/sync.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2021 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package catalyst - -import ( - "sync" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/log" -) - -type syncer struct { - running bool - newBlocks map[common.Hash]*types.Block - lock sync.Mutex -} - -func newSyncer() *syncer { - return &syncer{ - newBlocks: make(map[common.Hash]*types.Block), - } -} - -// onNewBlock is the action for receiving new block event -func (s *syncer) onNewBlock(block *types.Block) { - s.lock.Lock() - defer s.lock.Unlock() - - if s.running { - return - } - s.newBlocks[block.Hash()] = block -} - -func (s *syncer) hasBlock(hash common.Hash) bool { - s.lock.Lock() - defer s.lock.Unlock() - - _, present := s.newBlocks[hash] - return present -} - -// onNewHead is the action for receiving new head event -func (s *syncer) onNewHead(head common.Hash) { - s.lock.Lock() - defer s.lock.Unlock() - - if s.running { - return - } - _, present := s.newBlocks[head] - if !present { - log.Error("Chain head is set with an unknown header") - return - } - s.running = true - - // todo call the SetHead function exposed by the downloader -} diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index bebc78566bb2..121750ada730 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -205,6 +205,9 @@ type Config struct { // Arrow Glacier block override (TODO: remove after the fork) OverrideArrowGlacier *big.Int `toml:",omitempty"` + + // OverrideTerminalTotalDifficulty (TODO: remove after the fork) + OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` } // CreateConsensusEngine creates a consensus engine for the given chain configuration. diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 1f1ee3aafb00..70a9649bff83 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -18,48 +18,49 @@ import ( // MarshalTOML marshals as TOML. func (c Config) MarshalTOML() (interface{}, error) { type Config struct { - Genesis *core.Genesis `toml:",omitempty"` - NetworkId uint64 - SyncMode downloader.SyncMode - EthDiscoveryURLs []string - SnapDiscoveryURLs []string - NoPruning bool - NoPrefetch bool - TxLookupLimit uint64 `toml:",omitempty"` - Whitelist map[uint64]common.Hash `toml:"-"` - LightServ int `toml:",omitempty"` - LightIngress int `toml:",omitempty"` - LightEgress int `toml:",omitempty"` - LightPeers int `toml:",omitempty"` - LightNoPrune bool `toml:",omitempty"` - LightNoSyncServe bool `toml:",omitempty"` - SyncFromCheckpoint bool `toml:",omitempty"` - UltraLightServers []string `toml:",omitempty"` - UltraLightFraction int `toml:",omitempty"` - UltraLightOnlyAnnounce bool `toml:",omitempty"` - SkipBcVersionCheck bool `toml:"-"` - DatabaseHandles int `toml:"-"` - DatabaseCache int - DatabaseFreezer string - TrieCleanCache int - TrieCleanCacheJournal string `toml:",omitempty"` - TrieCleanCacheRejournal time.Duration `toml:",omitempty"` - TrieDirtyCache int - TrieTimeout time.Duration - SnapshotCache int - Preimages bool - Miner miner.Config - Ethash ethash.Config - TxPool core.TxPoolConfig - GPO gasprice.Config - EnablePreimageRecording bool - DocRoot string `toml:"-"` - RPCGasCap uint64 - RPCEVMTimeout time.Duration - RPCTxFeeCap float64 - Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` - CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` - OverrideArrowGlacier *big.Int `toml:",omitempty"` + Genesis *core.Genesis `toml:",omitempty"` + NetworkId uint64 + SyncMode downloader.SyncMode + EthDiscoveryURLs []string + SnapDiscoveryURLs []string + NoPruning bool + NoPrefetch bool + TxLookupLimit uint64 `toml:",omitempty"` + Whitelist map[uint64]common.Hash `toml:"-"` + LightServ int `toml:",omitempty"` + LightIngress int `toml:",omitempty"` + LightEgress int `toml:",omitempty"` + LightPeers int `toml:",omitempty"` + LightNoPrune bool `toml:",omitempty"` + LightNoSyncServe bool `toml:",omitempty"` + SyncFromCheckpoint bool `toml:",omitempty"` + UltraLightServers []string `toml:",omitempty"` + UltraLightFraction int `toml:",omitempty"` + UltraLightOnlyAnnounce bool `toml:",omitempty"` + SkipBcVersionCheck bool `toml:"-"` + DatabaseHandles int `toml:"-"` + DatabaseCache int + DatabaseFreezer string + TrieCleanCache int + TrieCleanCacheJournal string `toml:",omitempty"` + TrieCleanCacheRejournal time.Duration `toml:",omitempty"` + TrieDirtyCache int + TrieTimeout time.Duration + SnapshotCache int + Preimages bool + Miner miner.Config + Ethash ethash.Config + TxPool core.TxPoolConfig + GPO gasprice.Config + EnablePreimageRecording bool + DocRoot string `toml:"-"` + RPCGasCap uint64 + RPCEVMTimeout time.Duration + RPCTxFeeCap float64 + Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` + CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` + OverrideArrowGlacier *big.Int `toml:",omitempty"` + OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` } var enc Config enc.Genesis = c.Genesis @@ -104,54 +105,56 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.Checkpoint = c.Checkpoint enc.CheckpointOracle = c.CheckpointOracle enc.OverrideArrowGlacier = c.OverrideArrowGlacier + enc.OverrideTerminalTotalDifficulty = c.OverrideTerminalTotalDifficulty return &enc, nil } // UnmarshalTOML unmarshals from TOML. func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { type Config struct { - Genesis *core.Genesis `toml:",omitempty"` - NetworkId *uint64 - SyncMode *downloader.SyncMode - EthDiscoveryURLs []string - SnapDiscoveryURLs []string - NoPruning *bool - NoPrefetch *bool - TxLookupLimit *uint64 `toml:",omitempty"` - Whitelist map[uint64]common.Hash `toml:"-"` - LightServ *int `toml:",omitempty"` - LightIngress *int `toml:",omitempty"` - LightEgress *int `toml:",omitempty"` - LightPeers *int `toml:",omitempty"` - LightNoPrune *bool `toml:",omitempty"` - LightNoSyncServe *bool `toml:",omitempty"` - SyncFromCheckpoint *bool `toml:",omitempty"` - UltraLightServers []string `toml:",omitempty"` - UltraLightFraction *int `toml:",omitempty"` - UltraLightOnlyAnnounce *bool `toml:",omitempty"` - SkipBcVersionCheck *bool `toml:"-"` - DatabaseHandles *int `toml:"-"` - DatabaseCache *int - DatabaseFreezer *string - TrieCleanCache *int - TrieCleanCacheJournal *string `toml:",omitempty"` - TrieCleanCacheRejournal *time.Duration `toml:",omitempty"` - TrieDirtyCache *int - TrieTimeout *time.Duration - SnapshotCache *int - Preimages *bool - Miner *miner.Config - Ethash *ethash.Config - TxPool *core.TxPoolConfig - GPO *gasprice.Config - EnablePreimageRecording *bool - DocRoot *string `toml:"-"` - RPCGasCap *uint64 - RPCEVMTimeout *time.Duration - RPCTxFeeCap *float64 - Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` - CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` - OverrideArrowGlacier *big.Int `toml:",omitempty"` + Genesis *core.Genesis `toml:",omitempty"` + NetworkId *uint64 + SyncMode *downloader.SyncMode + EthDiscoveryURLs []string + SnapDiscoveryURLs []string + NoPruning *bool + NoPrefetch *bool + TxLookupLimit *uint64 `toml:",omitempty"` + Whitelist map[uint64]common.Hash `toml:"-"` + LightServ *int `toml:",omitempty"` + LightIngress *int `toml:",omitempty"` + LightEgress *int `toml:",omitempty"` + LightPeers *int `toml:",omitempty"` + LightNoPrune *bool `toml:",omitempty"` + LightNoSyncServe *bool `toml:",omitempty"` + SyncFromCheckpoint *bool `toml:",omitempty"` + UltraLightServers []string `toml:",omitempty"` + UltraLightFraction *int `toml:",omitempty"` + UltraLightOnlyAnnounce *bool `toml:",omitempty"` + SkipBcVersionCheck *bool `toml:"-"` + DatabaseHandles *int `toml:"-"` + DatabaseCache *int + DatabaseFreezer *string + TrieCleanCache *int + TrieCleanCacheJournal *string `toml:",omitempty"` + TrieCleanCacheRejournal *time.Duration `toml:",omitempty"` + TrieDirtyCache *int + TrieTimeout *time.Duration + SnapshotCache *int + Preimages *bool + Miner *miner.Config + Ethash *ethash.Config + TxPool *core.TxPoolConfig + GPO *gasprice.Config + EnablePreimageRecording *bool + DocRoot *string `toml:"-"` + RPCGasCap *uint64 + RPCEVMTimeout *time.Duration + RPCTxFeeCap *float64 + Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` + CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` + OverrideArrowGlacier *big.Int `toml:",omitempty"` + OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` } var dec Config if err := unmarshal(&dec); err != nil { @@ -283,5 +286,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.OverrideArrowGlacier != nil { c.OverrideArrowGlacier = dec.OverrideArrowGlacier } + if dec.OverrideTerminalTotalDifficulty != nil { + c.OverrideTerminalTotalDifficulty = dec.OverrideTerminalTotalDifficulty + } return nil } diff --git a/eth/gasprice/gasprice_test.go b/eth/gasprice/gasprice_test.go index 0eaa41dd9d6b..c0d3c6b6038e 100644 --- a/eth/gasprice/gasprice_test.go +++ b/eth/gasprice/gasprice_test.go @@ -144,7 +144,7 @@ func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBacke // Construct testing chain diskdb := rawdb.NewMemoryDatabase() gspec.Commit(diskdb) - chain, err := core.NewBlockChain(diskdb, &core.CacheConfig{TrieCleanNoPrefetch: true}, gspec.Config, engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) + chain, err := core.NewBlockChain(diskdb, &core.CacheConfig{TrieCleanNoPrefetch: true}, gspec.Config, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create local chain, %v", err) } diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index 01e7762897e1..085812997b5a 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -105,8 +105,8 @@ func testForkIDSplit(t *testing.T, protocol uint) { genesisNoFork = gspecNoFork.MustCommit(dbNoFork) genesisProFork = gspecProFork.MustCommit(dbProFork) - chainNoFork, _ = core.NewBlockChain(dbNoFork, nil, configNoFork, engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) - chainProFork, _ = core.NewBlockChain(dbProFork, nil, configProFork, engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) + chainNoFork, _ = core.NewBlockChain(dbNoFork, nil, configNoFork, engine, vm.Config{}, nil, nil) + chainProFork, _ = core.NewBlockChain(dbProFork, nil, configProFork, engine, vm.Config{}, nil, nil) blocksNoFork, _ = core.GenerateChain(configNoFork, genesisNoFork, engine, dbNoFork, 2, nil) blocksProFork, _ = core.GenerateChain(configProFork, genesisProFork, engine, dbProFork, 2, nil) diff --git a/eth/handler_test.go b/eth/handler_test.go index 56313d7bb146..4a45e81ed64a 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -137,7 +137,7 @@ func newTestHandlerWithBlocks(blocks int) *testHandler { Alloc: core.GenesisAlloc{testAddr: {Balance: big.NewInt(1000000)}}, }).MustCommit(db) - chain, _ := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) + chain, _ := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) bs, _ := core.GenerateChain(params.TestChainConfig, chain.Genesis(), ethash.NewFaker(), db, blocks, nil) if _, err := chain.InsertChain(bs); err != nil { diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index 0a4f1cbd84b1..66f013409694 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -69,7 +69,7 @@ func newTestBackendWithGenerator(blocks int, generator func(int, *core.BlockGen) Alloc: core.GenesisAlloc{testAddr: {Balance: big.NewInt(100_000_000_000_000_000)}}, }).MustCommit(db) - chain, _ := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) + chain, _ := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) bs, _ := core.GenerateChain(params.TestChainConfig, chain.Genesis(), ethash.NewFaker(), db, blocks, generator) if _, err := chain.InsertChain(bs); err != nil { diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index 3f3a460e03b3..9afd59d596bc 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -81,7 +81,7 @@ func newTestBackend(t *testing.T, n int, gspec *core.Genesis, generator func(i i SnapshotLimit: 0, TrieDirtyDisabled: true, // Archive mode } - chain, err := core.NewBlockChain(backend.chaindb, cacheConfig, backend.chainConfig, backend.engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) + chain, err := core.NewBlockChain(backend.chaindb, cacheConfig, backend.chainConfig, backend.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } diff --git a/les/client.go b/les/client.go index 12f4174f86d1..4fecc970af3e 100644 --- a/les/client.go +++ b/les/client.go @@ -89,7 +89,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideArrowGlacier) + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideArrowGlacier, config.OverrideTerminalTotalDifficulty) if _, isCompat := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !isCompat { return nil, genesisErr } @@ -141,7 +141,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { } // Note: NewLightChain adds the trusted checkpoint so it needs an ODR with // indexers already set but not started yet - if leth.blockchain, err = light.NewLightChain(leth.odr, leth.chainConfig, leth.engine, checkpoint, merger); err != nil { + if leth.blockchain, err = light.NewLightChainWithMerger(leth.odr, leth.chainConfig, leth.engine, checkpoint, merger); err != nil { return nil, err } leth.chainReader = leth.blockchain diff --git a/les/test_helper.go b/les/test_helper.go index ea5502395174..2c5a56669b42 100644 --- a/les/test_helper.go +++ b/les/test_helper.go @@ -202,7 +202,7 @@ func newTestClientHandler(backend *backends.SimulatedBackend, odr *LesOdr, index oracle *checkpointoracle.CheckpointOracle ) genesis := gspec.MustCommit(db) - chain, _ := light.NewLightChain(odr, gspec.Config, engine, nil, core.NewMerger(rawdb.NewMemoryDatabase())) + chain, _ := light.NewLightChain(odr, gspec.Config, engine, nil) if indexers != nil { checkpointConfig := ¶ms.CheckpointOracleConfig{ Address: crypto.CreateAddress(bankAddr, 0), diff --git a/light/lightchain.go b/light/lightchain.go index 0244d295229c..72119f8755a6 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -78,7 +78,14 @@ type LightChain struct { // NewLightChain returns a fully initialised light chain using information // available in the database. It initialises the default Ethereum header // validator. -func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus.Engine, checkpoint *params.TrustedCheckpoint, merger *core.Merger) (*LightChain, error) { +func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus.Engine, checkpoint *params.TrustedCheckpoint) (*LightChain, error) { + return NewLightChainWithMerger(odr, config, engine, checkpoint, core.NewMerger(rawdb.NewMemoryDatabase())) +} + +// NewLightChain returns a fully initialised light chain using information +// available in the database. It initialises the default Ethereum header +// validator. +func NewLightChainWithMerger(odr OdrBackend, config *params.ChainConfig, engine consensus.Engine, checkpoint *params.TrustedCheckpoint, merger *core.Merger) (*LightChain, error) { bodyCache, _ := lru.New(bodyCacheLimit) bodyRLPCache, _ := lru.New(bodyCacheLimit) blockCache, _ := lru.New(blockCacheLimit) @@ -93,10 +100,7 @@ func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus. blockCache: blockCache, engine: engine, } - bc.forker = core.NewForkChoice(bc, merger.LeftPoW(), nil) - merger.SubscribeLeavePoW(func() { - bc.forker.MarkTransitioned() - }) + bc.forker = core.NewForkChoice(bc, nil) var err error bc.hc, err = core.NewHeaderChain(odr.Database(), config, bc.engine, bc.getProcInterrupt) if err != nil { diff --git a/light/lightchain_test.go b/light/lightchain_test.go index d6a20823436f..8600e56345f6 100644 --- a/light/lightchain_test.go +++ b/light/lightchain_test.go @@ -56,7 +56,7 @@ func newCanonical(n int) (ethdb.Database, *LightChain, error) { db := rawdb.NewMemoryDatabase() gspec := core.Genesis{Config: params.TestChainConfig} genesis := gspec.MustCommit(db) - blockchain, _ := NewLightChain(&dummyOdr{db: db, indexerConfig: TestClientIndexerConfig}, gspec.Config, ethash.NewFaker(), nil, core.NewMerger(rawdb.NewMemoryDatabase())) + blockchain, _ := NewLightChain(&dummyOdr{db: db, indexerConfig: TestClientIndexerConfig}, gspec.Config, ethash.NewFaker(), nil) // Create and inject the requested chain if n == 0 { @@ -76,7 +76,7 @@ func newTestLightChain() *LightChain { Config: params.TestChainConfig, } gspec.MustCommit(db) - lc, err := NewLightChain(&dummyOdr{db: db}, gspec.Config, ethash.NewFullFaker(), nil, core.NewMerger(rawdb.NewMemoryDatabase())) + lc, err := NewLightChain(&dummyOdr{db: db}, gspec.Config, ethash.NewFullFaker(), nil) if err != nil { panic(err) } @@ -347,7 +347,7 @@ func TestReorgBadHeaderHashes(t *testing.T) { defer func() { delete(core.BadHashes, headers[3].Hash()) }() // Create a new LightChain and check that it rolled back the state. - ncm, err := NewLightChain(&dummyOdr{db: bc.chainDb}, params.TestChainConfig, ethash.NewFaker(), nil, core.NewMerger(rawdb.NewMemoryDatabase())) + ncm, err := NewLightChain(&dummyOdr{db: bc.chainDb}, params.TestChainConfig, ethash.NewFaker(), nil) if err != nil { t.Fatalf("failed to create new chain manager: %v", err) } diff --git a/light/odr_test.go b/light/odr_test.go index 85c2ba86ba0a..fdf657a82ec5 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -261,14 +261,14 @@ func testChainOdr(t *testing.T, protocol int, fn odrTestFn) { ) gspec.MustCommit(ldb) // Assemble the test environment - blockchain, _ := core.NewBlockChain(sdb, nil, params.TestChainConfig, ethash.NewFullFaker(), vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) + blockchain, _ := core.NewBlockChain(sdb, nil, params.TestChainConfig, ethash.NewFullFaker(), vm.Config{}, nil, nil) gchain, _ := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), sdb, 4, testChainGen) if _, err := blockchain.InsertChain(gchain); err != nil { t.Fatal(err) } odr := &testOdr{sdb: sdb, ldb: ldb, indexerConfig: TestClientIndexerConfig} - lightchain, err := NewLightChain(odr, params.TestChainConfig, ethash.NewFullFaker(), nil, core.NewMerger(rawdb.NewMemoryDatabase())) + lightchain, err := NewLightChain(odr, params.TestChainConfig, ethash.NewFullFaker(), nil) if err != nil { t.Fatal(err) } diff --git a/light/trie_test.go b/light/trie_test.go index 1b1d49bcd22f..e8294cc2a235 100644 --- a/light/trie_test.go +++ b/light/trie_test.go @@ -44,7 +44,7 @@ func TestNodeIterator(t *testing.T) { genesis = gspec.MustCommit(fulldb) ) gspec.MustCommit(lightdb) - blockchain, _ := core.NewBlockChain(fulldb, nil, params.TestChainConfig, ethash.NewFullFaker(), vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) + blockchain, _ := core.NewBlockChain(fulldb, nil, params.TestChainConfig, ethash.NewFullFaker(), vm.Config{}, nil, nil) gchain, _ := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), fulldb, 4, testChainGen) if _, err := blockchain.InsertChain(gchain); err != nil { panic(err) diff --git a/light/txpool_test.go b/light/txpool_test.go index ee1a26df0d2f..cc2651d29ae5 100644 --- a/light/txpool_test.go +++ b/light/txpool_test.go @@ -91,7 +91,7 @@ func TestTxPool(t *testing.T) { ) gspec.MustCommit(ldb) // Assemble the test environment - blockchain, _ := core.NewBlockChain(sdb, nil, params.TestChainConfig, ethash.NewFullFaker(), vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) + blockchain, _ := core.NewBlockChain(sdb, nil, params.TestChainConfig, ethash.NewFullFaker(), vm.Config{}, nil, nil) gchain, _ := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), sdb, poolTestBlocks, txPoolTestChainGen) if _, err := blockchain.InsertChain(gchain); err != nil { panic(err) @@ -103,7 +103,7 @@ func TestTxPool(t *testing.T) { discard: make(chan int, 1), mined: make(chan int, 1), } - lightchain, _ := NewLightChain(odr, params.TestChainConfig, ethash.NewFullFaker(), nil, core.NewMerger(rawdb.NewMemoryDatabase())) + lightchain, _ := NewLightChain(odr, params.TestChainConfig, ethash.NewFullFaker(), nil) txPermanent = 50 pool := NewTxPool(params.TestChainConfig, lightchain, relay) ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) diff --git a/miner/miner_test.go b/miner/miner_test.go index 69d5c551fa05..a4324ee67e61 100644 --- a/miner/miner_test.go +++ b/miner/miner_test.go @@ -245,7 +245,8 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux) { // Create consensus engine engine := clique.New(chainConfig.Clique, chainDB) // Create Ethereum backend - bc, err := core.NewBlockChain(chainDB, nil, chainConfig, engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) + merger := core.NewMerger(rawdb.NewMemoryDatabase()) + bc, err := core.NewBlockChainWithMerger(chainDB, nil, chainConfig, engine, vm.Config{}, nil, nil, merger) if err != nil { t.Fatalf("can't create new chain %v", err) } @@ -257,5 +258,5 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux) { // Create event Mux mux := new(event.TypeMux) // Create Miner - return New(backend, &config, chainConfig, mux, engine, nil, core.NewMerger(rawdb.NewMemoryDatabase())), mux + return New(backend, &config, chainConfig, mux, engine, nil, merger), mux } diff --git a/miner/stress/beacon/main.go b/miner/stress/beacon/main.go index 46bf6d7e1476..dc5c229410cf 100644 --- a/miner/stress/beacon/main.go +++ b/miner/stress/beacon/main.go @@ -172,13 +172,7 @@ func (n *ethNode) insertBlockAndSetHead(parent *types.Header, ed catalyst.Execut if err := n.insertBlock(ed); err != nil { return err } - var config *params.ChainConfig - if n.typ == eth2LightClient { - config = n.lesBackend.BlockChain().Config() - } else { - config = n.ethBackend.BlockChain().Config() - } - block, err := catalyst.ExecutableDataToBlock(config, parent, ed) + block, err := catalyst.ExecutableDataToBlock(ed) if err != nil { return err } @@ -319,7 +313,7 @@ func (mgr *nodeManager) run() { log.Error("Failed to assemble the block", "err", err) continue } - block, _ := catalyst.ExecutableDataToBlock(chain.Config(), parentBlock.Header(), *ed) + block, _ := catalyst.ExecutableDataToBlock(*ed) nodes := mgr.getNodes(eth2MiningNode) nodes = append(nodes, mgr.getNodes(eth2NormalNode)...) diff --git a/miner/worker.go b/miner/worker.go index b7b0ba14cc97..71a4127c4d20 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -19,7 +19,6 @@ package miner import ( "bytes" "errors" - "fmt" "math/big" "sync" "sync/atomic" @@ -1038,17 +1037,6 @@ func (w *worker) commit(uncles []*types.Header, interval func(), update bool, st return err } - // TODO (MariusVanDerWijden) remove this after eth2 interop - // --- - parent := w.chain.GetBlockByHash(block.ParentHash()) - if parent == nil { - return fmt.Errorf("parent unavailable: %v", block.ParentHash()) - } - td := w.chain.GetTdByHash(parent.Hash()) - if td != nil && td.Cmp(w.chain.Config().TerminalTotalDifficulty) > 0 { - log.Warn("Total terminal difficulty reached, keeping on mining to test eth2 clients") - } - // --- if w.isRunning() && !w.merger.LeftPoW() { if interval != nil { interval() diff --git a/miner/worker_test.go b/miner/worker_test.go index 3b4484adfd2a..cbb4703d372f 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -135,7 +135,7 @@ func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine } genesis := gspec.MustCommit(db) - chain, _ := core.NewBlockChain(db, &core.CacheConfig{TrieDirtyDisabled: true}, gspec.Config, engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) + chain, _ := core.NewBlockChain(db, &core.CacheConfig{TrieDirtyDisabled: true}, gspec.Config, engine, vm.Config{}, nil, nil) txpool := core.NewTxPool(testTxPoolConfig, chainConfig, chain) // Generate a small n-block chain and an uncle block for it @@ -232,7 +232,7 @@ func testGenerateBlockAndImport(t *testing.T, isClique bool) { // This test chain imports the mined blocks. db2 := rawdb.NewMemoryDatabase() b.genesis.MustCommit(db2) - chain, _ := core.NewBlockChain(db2, nil, b.chain.Config(), engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) + chain, _ := core.NewBlockChain(db2, nil, b.chain.Config(), engine, vm.Config{}, nil, nil) defer chain.Stop() // Ignore empty commit here for less noise. diff --git a/tests/block_test_util.go b/tests/block_test_util.go index e3daab9b5d22..bcf861e09b9c 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -127,7 +127,7 @@ func (t *BlockTest) Run(snapshotter bool) error { cache.SnapshotLimit = 1 cache.SnapshotWait = true } - chain, err := core.NewBlockChain(db, cache, config, engine, vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) + chain, err := core.NewBlockChain(db, cache, config, engine, vm.Config{}, nil, nil) if err != nil { return err } diff --git a/tests/fuzzers/les/les-fuzzer.go b/tests/fuzzers/les/les-fuzzer.go index 4aeee43a6656..3e1017187345 100644 --- a/tests/fuzzers/les/les-fuzzer.go +++ b/tests/fuzzers/les/les-fuzzer.go @@ -80,7 +80,7 @@ func makechain() (bc *core.BlockChain, addrHashes, txHashes []common.Hash) { addrHashes = append(addrHashes, crypto.Keccak256Hash(addr[:])) txHashes = append(txHashes, tx.Hash()) }) - bc, _ = core.NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, core.NewMerger(rawdb.NewMemoryDatabase())) + bc, _ = core.NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) if _, err := bc.InsertChain(blocks); err != nil { panic(err) } From 7727d0e1b46b2f72ad7d964dd11bed9767f5c2b7 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Wed, 20 Oct 2021 12:48:47 +0200 Subject: [PATCH 08/32] core, eth: correct semantics for LeavePoW, EnterPoS --- consensus/beacon/consensus.go | 28 +++++++----- core/blockchain.go | 10 ++--- core/forkchoice.go | 2 +- core/merger.go | 13 ------ eth/catalyst/api.go | 82 ++++------------------------------- eth/catalyst/api_test.go | 49 --------------------- 6 files changed, 31 insertions(+), 153 deletions(-) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index c8e49341074f..043c53463d5c 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -24,9 +24,9 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/consensus/misc" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/trie" ) @@ -55,12 +55,12 @@ var ( // // The beacon here is a half-functional consensus engine with partial functions which // is only used for necessary consensus checks. The legacy consensus engine can be any -// engine implements the consensus interface(except the beacon itself). +// engine implements the consensus interface (except the beacon itself). type Beacon struct { ethone consensus.Engine // Classic consensus engine used in eth1, e.g. ethash or clique // transitioned is the flag whether the transition has been triggered. - // It's triggered by receiving the first "POS_CHAINHEAD_SET" message + // It's triggered by receiving the first "ENGINE_FORKCHOICEUPDATED" message // from the external consensus engine. transitioned bool lock sync.RWMutex @@ -107,6 +107,7 @@ func (beacon *Beacon) VerifyHeader(chain consensus.ChainHeaderReader, header *ty // VerifyHeaders is similar to VerifyHeader, but verifies a batch of headers // concurrently. The method returns a quit channel to abort the operations and // a results channel to retrieve the async verifications. +// VerifyHeaders expect the headers to be ordered (not necessary continuous). func (beacon *Beacon) VerifyHeaders(chain consensus.ChainHeaderReader, headers []*types.Header, seals []bool) (chan<- struct{}, <-chan error) { if !beacon.IsPoSHeader(headers[len(headers)-1]) { return beacon.ethone.VerifyHeaders(chain, headers, seals) @@ -181,9 +182,10 @@ func (beacon *Beacon) VerifyUncles(chain consensus.ChainReader, block *types.Blo // verifyHeader checks whether a header conforms to the consensus rules of the // stock Ethereum consensus engine. The difference between the beacon and classic is -// (a) the difficulty, mixhash, nonce, extradata and unclehash are expected +// (a) the difficulty, mixhash, nonce and unclehash are expected // to be the desired constants // (b) the timestamp is not verified anymore +// (c) the extradata is limited to 32 bytes func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, parent *types.Header) error { // Ensure that the header's extra-data section is of a reasonable size if len(header.Extra) > 32 { @@ -203,13 +205,17 @@ func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, pa return fmt.Errorf("invalid gasUsed: have %d, gasLimit %d", header.GasUsed, header.GasLimit) } // Verify that the gas limit remains within allowed bounds - diff := int64(parent.GasLimit) - int64(header.GasLimit) - if diff < 0 { - diff *= -1 - } - limit := parent.GasLimit / params.GasLimitBoundDivisor - if uint64(diff) >= limit || header.GasLimit < params.MinGasLimit { - return fmt.Errorf("invalid gas limit: have %d, want %d += %d", header.GasLimit, parent.GasLimit, limit) + if !chain.Config().IsLondon(header.Number) { + // Verify BaseFee not present before EIP-1559 fork. + if header.BaseFee != nil { + return fmt.Errorf("invalid baseFee before fork: have %d, expected 'nil'", header.BaseFee) + } + if err := misc.VerifyGaslimit(parent.GasLimit, header.GasLimit); err != nil { + return err + } + } else if err := misc.VerifyEip1559Header(chain.Config(), parent, header); err != nil { + // Verify the header's EIP-1559 attributes. + return err } // Verify that the block number is parent's +1 if diff := new(big.Int).Sub(header.Number, parent.Number); diff.Cmp(big.NewInt(1)) != 0 { diff --git a/core/blockchain.go b/core/blockchain.go index 5ef19a3c1fc1..851ffbeee9c2 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -928,7 +928,10 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ // Rewind may have occurred, skip in that case. if bc.CurrentHeader().Number.Cmp(head.Number()) >= 0 { reorg, err := bc.forker.Reorg(bc.CurrentFastBlock().Header(), head.Header()) - if err != nil || !reorg { + if err != nil { + log.Warn("Reorg failed", "err", err) + return false + } else if !reorg { return false } rawdb.WriteHeadFastBlockHash(bc.db, head.Hash()) @@ -1184,9 +1187,6 @@ func (bc *BlockChain) writeKnownBlock(block *types.Block) error { // writeBlockWithState writes block, metadata and corresponding state data to the // database. func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.Receipt, logs []*types.Log, state *state.StateDB) error { - bc.wg.Add(1) - defer bc.wg.Done() - // Calculate the total difficulty of the block ptd := bc.GetTd(block.ParentHash(), block.NumberU64()-1) if ptd == nil { @@ -1436,7 +1436,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er if reorg { // Switch to import mode if the forker says the reorg is necessary // and also the block is not on the canonical chain. - // In eth2 the forker always returns True for reorg desision(blindly trust + // In eth2 the forker always returns true for reorg decision (blindly trusting // the external consensus engine), but in order to prevent the unnecessary // reorgs when importing known blocks, the special case is handled here. if bc.GetCanonicalHash(block.NumberU64()) != block.Hash() || block.NumberU64() > current.NumberU64() { diff --git a/core/forkchoice.go b/core/forkchoice.go index e3b8e591201f..a7b75a91386f 100644 --- a/core/forkchoice.go +++ b/core/forkchoice.go @@ -53,7 +53,7 @@ type ForkChoice struct { // preserve is a helper function used in td fork choice. // Miners will prefer to choose the local mined block if the - // local td is equal to the extern one. It can nil for light + // local td is equal to the extern one. It can be nil for light // client preserve func(header *types.Header) bool } diff --git a/core/merger.go b/core/merger.go index c6e2f0eceb89..6734167d7486 100644 --- a/core/merger.go +++ b/core/merger.go @@ -39,7 +39,6 @@ type Merger struct { db ethdb.KeyValueStore status transitionStatus leavePoWCalls []func() - enterPoSCalls []func() lock sync.Mutex } @@ -66,15 +65,6 @@ func (m *Merger) SubscribeLeavePoW(callback func()) { m.leavePoWCalls = append(m.leavePoWCalls, callback) } -// SubscribeEnterPoS registers callback so that if the chain leaves -// from the 'transition' stage and enters the PoS stage it can be invoked. -func (m *Merger) SubscribeEnterPoS(callback func()) { - m.lock.Lock() - defer m.lock.Unlock() - - m.enterPoSCalls = append(m.enterPoSCalls, callback) -} - // LeavePoW is called whenever the first NewHead message received // from the consensus-layer. func (m *Merger) LeavePoW() { @@ -111,9 +101,6 @@ func (m *Merger) EnterPoS() { log.Crit("Failed to encode the transition status", "err", err) } rawdb.WriteTransitionStatus(m.db, blob) - for _, call := range m.enterPoSCalls { - call() - } log.Info("Entered PoS stage") } diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 9925dea5f332..a60fb599fce8 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -22,7 +22,6 @@ import ( "errors" "fmt" "math/big" - "os" "time" "github.com/ethereum/go-ethereum/common" @@ -38,7 +37,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" chainParams "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/trie" ) @@ -196,15 +194,7 @@ func (api *ConsensusAPI) GetPayload(PayloadID hexutil.Uint64) (*ExecutableData, func (api *ConsensusAPI) ConsensusValidated(params ConsensusValidatedParams) error { switch params.Status { case VALID.Status: - // Finalize the transition if it's the first `FinalisedBlock` event. - merger := api.merger() - if !merger.EnteredPoS() { - if err := api.checkTerminalTotalDifficulty(params.BlockHash); err != nil { - return err - } - merger.EnterPoS() - } - return nil //api.setHead(params.BlockHash) + return nil case INVALID.Status: // TODO (MariusVanDerWijden) delete the block from the bc return nil @@ -258,6 +248,10 @@ func (api *ConsensusAPI) ExecutePayload(params ExecutableData) (GenericStringRes if err := api.eth.BlockChain().InsertBlock(block); err != nil { return INVALID, err } + merger := api.merger() + if !merger.LeftPoW() { + merger.LeavePoW() + } return VALID, nil } @@ -277,7 +271,7 @@ func (api *ConsensusAPI) assembleBlock(params AssembleBlockParams) (*ExecutableD } if params.Timestamp < parent.Time() { - return nil, fmt.Errorf("child timestamp lower than parent's: %d >= %d", parent.Time(), params.Timestamp) + return nil, fmt.Errorf("child timestamp lower than parent's: %d < %d", params.Timestamp, parent.Time()) } if now := uint64(time.Now().Unix()); params.Timestamp > now+1 { diff := time.Duration(params.Timestamp-now) * time.Second @@ -468,8 +462,8 @@ func (api *ConsensusAPI) checkTerminalTotalDifficulty(head common.Hash) error { func (api *ConsensusAPI) setHead(newHead common.Hash) error { // Trigger the transition if it's the first `NewHead` event. merger := api.merger() - if !merger.LeftPoW() { - merger.LeavePoW() + if !merger.EnteredPoS() { + merger.EnterPoS() } log.Info("Setting head", "head", newHead) if api.light { @@ -508,63 +502,3 @@ func (api *ConsensusAPI) merger() *core.Merger { } return api.eth.Merger() } - -// Helper API for the merge f2f - -func (api *ConsensusAPI) ExportChain(path string) error { - if api.light { - return errors.New("cannot export chain in light mode") - } - f, err := os.Create(path) - if err != nil { - return err - } - return api.eth.BlockChain().Export(f) -} - -func (api *ConsensusAPI) ImportChain(path string) error { - if api.light { - return errors.New("cannot import chain in light mode") - } - f, err := os.Open(path) - if err != nil { - return err - } - for { - var block types.Block - if err := block.DecodeRLP(rlp.NewStream(f, 0)); err != nil { - break - } - if err := api.eth.BlockChain().InsertBlock(&block); err != nil { - return err - } - } - return nil -} - -func (api *ConsensusAPI) ExportExecutableData(path string) error { - if api.light { - return errors.New("cannot export chain in light mode") - } - f, err := os.Create(path) - if err != nil { - return err - } - for i := uint64(0); i < api.eth.BlockChain().CurrentBlock().NumberU64(); i++ { - block := api.eth.BlockChain().GetBlockByNumber(i) - exec := BlockToExecutableData(block, common.Hash{}) - b, err := exec.MarshalJSON() - if err != nil { - return err - } - if _, err := f.Write(b); err != nil { - return err - } - f.Write([]byte("\n")) - } - return nil -} - -func (api *ConsensusAPI) GetHead() (common.Hash, error) { - return api.eth.BlockChain().CurrentBlock().Hash(), nil -} diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index c3871b45617a..ca53cec8ffcc 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -73,52 +73,6 @@ func generatePreMergeChain(n int) (*core.Genesis, []*types.Block) { return genesis, blocks } -// TODO (MariusVanDerWijden) reenable once engine api is updated to the latest spec -/* -func generateTestChainWithFork(n int, fork int) (*core.Genesis, []*types.Block, []*types.Block) { - if fork >= n { - fork = n - 1 - } - db := rawdb.NewMemoryDatabase() - config := ¶ms.ChainConfig{ - ChainID: big.NewInt(1337), - HomesteadBlock: big.NewInt(0), - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - ByzantiumBlock: big.NewInt(0), - ConstantinopleBlock: big.NewInt(0), - PetersburgBlock: big.NewInt(0), - IstanbulBlock: big.NewInt(0), - MuirGlacierBlock: big.NewInt(0), - BerlinBlock: big.NewInt(0), - LondonBlock: big.NewInt(0), - Ethash: new(params.EthashConfig), - } - genesis := &core.Genesis{ - Config: config, - Alloc: core.GenesisAlloc{testAddr: {Balance: testBalance}}, - ExtraData: []byte("test genesis"), - Timestamp: 9000, - BaseFee: big.NewInt(params.InitialBaseFee), - } - generate := func(i int, g *core.BlockGen) { - g.OffsetTime(5) - g.SetExtra([]byte("test")) - } - generateFork := func(i int, g *core.BlockGen) { - g.OffsetTime(5) - g.SetExtra([]byte("testF")) - } - gblock := genesis.ToBlock(db) - engine := ethash.NewFaker() - blocks, _ := core.GenerateChain(config, gblock, engine, db, n, generate) - blocks = append([]*types.Block{gblock}, blocks...) - forkedBlocks, _ := core.GenerateChain(config, blocks[fork], engine, db, n-fork, generateFork) - return genesis, blocks, forkedBlocks -} -*/ - func TestEth2AssembleBlock(t *testing.T) { genesis, blocks := generatePreMergeChain(10) n, ethservice := startEthService(t, genesis, blocks) @@ -172,9 +126,6 @@ func TestSetHeadBeforeTotalDifficulty(t *testing.T) { defer n.Close() api := NewConsensusAPI(ethservice, nil) - if err := api.ConsensusValidated(ConsensusValidatedParams{BlockHash: blocks[5].Hash(), Status: VALID.Status}); err == nil { - t.Errorf("consensus validated before total terminal difficulty should fail") - } if err := api.ForkchoiceUpdated(ForkChoiceParams{HeadBlockHash: blocks[5].Hash()}); err == nil { t.Errorf("fork choice updated before total terminal difficulty should fail") From 0bc0bb23ef12799836049a1b13925b7fcb0cc7c5 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 2 Nov 2021 15:33:22 +0100 Subject: [PATCH 09/32] core: fixed rebasing artifacts --- consensus/beacon/consensus.go | 2 +- core/blockchain.go | 21 ++++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index 043c53463d5c..448f679d409e 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -107,7 +107,7 @@ func (beacon *Beacon) VerifyHeader(chain consensus.ChainHeaderReader, header *ty // VerifyHeaders is similar to VerifyHeader, but verifies a batch of headers // concurrently. The method returns a quit channel to abort the operations and // a results channel to retrieve the async verifications. -// VerifyHeaders expect the headers to be ordered (not necessary continuous). +// VerifyHeaders expect the headers to be ordered and continuous. func (beacon *Beacon) VerifyHeaders(chain consensus.ChainHeaderReader, headers []*types.Header, seals []bool) (chan<- struct{}, <-chan error) { if !beacon.IsPoSHeader(headers[len(headers)-1]) { return beacon.ethone.VerifyHeaders(chain, headers, seals) diff --git a/core/blockchain.go b/core/blockchain.go index 851ffbeee9c2..890b332bd0b8 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -743,24 +743,31 @@ func (bc *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error { // // Note, this function assumes that the `mu` mutex is held! func (bc *BlockChain) writeHeadBlock(block *types.Block) { + // If the block is on a side chain or an unknown one, force other heads onto it too + updateHeads := rawdb.ReadCanonicalHash(bc.db, block.NumberU64()) != block.Hash() + // Add the block to the canonical chain number scheme and mark as the head batch := bc.db.NewBatch() - rawdb.WriteHeadHeaderHash(batch, block.Hash()) - rawdb.WriteHeadFastBlockHash(batch, block.Hash()) rawdb.WriteCanonicalHash(batch, block.Hash(), block.NumberU64()) rawdb.WriteTxLookupEntriesByBlock(batch, block) rawdb.WriteHeadBlockHash(batch, block.Hash()) + // If the block is better than our head or is on a different chain, force update heads + if updateHeads { + rawdb.WriteHeadHeaderHash(batch, block.Hash()) + rawdb.WriteHeadFastBlockHash(batch, block.Hash()) + } + // Flush the whole batch into the disk, exit the node if failed if err := batch.Write(); err != nil { log.Crit("Failed to update chain indexes and markers", "err", err) } // Update all in-memory chain markers in the last step - bc.hc.SetCurrentHeader(block.Header()) - - bc.currentFastBlock.Store(block) - headFastBlockGauge.Update(int64(block.NumberU64())) - + if updateHeads { + bc.hc.SetCurrentHeader(block.Header()) + bc.currentFastBlock.Store(block) + headFastBlockGauge.Update(int64(block.NumberU64())) + } bc.currentBlock.Store(block) headBlockGauge.Update(int64(block.NumberU64())) } From 16be60f8b178dd82182dd45581c07815eec9138a Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 2 Nov 2021 16:50:59 +0100 Subject: [PATCH 10/32] core: light: performance improvements --- consensus/beacon/consensus.go | 28 ++++++++++++++-------------- core/blockchain.go | 16 +++++----------- core/forkchoice.go | 9 ++------- core/headerchain.go | 34 ++++++++++++++++------------------ light/lightchain.go | 2 +- 5 files changed, 38 insertions(+), 51 deletions(-) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index 448f679d409e..d1fee6b5c074 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -191,6 +191,16 @@ func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, pa if len(header.Extra) > 32 { return fmt.Errorf("extra-data longer than 32 bytes (%d)", len(header.Extra)) } + // Verify the seal parts. Ensure the mixhash, nonce and uncle hash are the expected value. + if header.MixDigest != (common.Hash{}) { + return errInvalidMixDigest + } + if header.Nonce != beaconNonce { + return errInvalidNonce + } + if header.UncleHash != types.EmptyUncleHash { + return errInvalidUncleHash + } // Verify the block's difficulty to ensure it's the default constant if beaconDifficulty.Cmp(header.Difficulty) != 0 { return fmt.Errorf("invalid difficulty: have %v, want %v", header.Difficulty, beaconDifficulty) @@ -204,6 +214,10 @@ func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, pa if header.GasUsed > header.GasLimit { return fmt.Errorf("invalid gasUsed: have %d, gasLimit %d", header.GasUsed, header.GasLimit) } + // Verify that the block number is parent's +1 + if diff := new(big.Int).Sub(header.Number, parent.Number); diff.Cmp(big.NewInt(1)) != 0 { + return consensus.ErrInvalidNumber + } // Verify that the gas limit remains within allowed bounds if !chain.Config().IsLondon(header.Number) { // Verify BaseFee not present before EIP-1559 fork. @@ -217,20 +231,6 @@ func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, pa // Verify the header's EIP-1559 attributes. return err } - // Verify that the block number is parent's +1 - if diff := new(big.Int).Sub(header.Number, parent.Number); diff.Cmp(big.NewInt(1)) != 0 { - return consensus.ErrInvalidNumber - } - // Verify the seal parts. Ensure the mixhash, nonce and uncle hash are the expected value. - if header.MixDigest != (common.Hash{}) { - return errInvalidMixDigest - } - if header.Nonce != beaconNonce { - return errInvalidNonce - } - if header.UncleHash != types.EmptyUncleHash { - return errInvalidUncleHash - } return nil } diff --git a/core/blockchain.go b/core/blockchain.go index 890b332bd0b8..9fe4289972a3 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -883,12 +883,6 @@ const ( SideStatTy ) -// numberHash is just a container for a number and a hash, to represent a block -type numberHash struct { - number uint64 - hash common.Hash -} - // InsertReceiptChain attempts to complete an already existing header chain with // transaction and receipt data. func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain []types.Receipts, ancientLimit uint64) (int, error) { @@ -934,7 +928,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ // Rewind may have occurred, skip in that case. if bc.CurrentHeader().Number.Cmp(head.Number()) >= 0 { - reorg, err := bc.forker.Reorg(bc.CurrentFastBlock().Header(), head.Header()) + reorg, err := bc.forker.ReorgNeeded(bc.CurrentFastBlock().Header(), head.Header()) if err != nil { log.Warn("Reorg failed", "err", err) return false @@ -1292,7 +1286,7 @@ func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types return NonStatTy, err } currentBlock := bc.CurrentBlock() - reorg, err := bc.forker.Reorg(currentBlock.Header(), block.Header()) + reorg, err := bc.forker.ReorgNeeded(currentBlock.Header(), block.Header()) if err != nil { return NonStatTy, err } @@ -1436,7 +1430,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er current = bc.CurrentBlock() ) for block != nil && bc.skipBlock(err, it) { - reorg, err = bc.forker.Reorg(current.Header(), block.Header()) + reorg, err = bc.forker.ReorgNeeded(current.Header(), block.Header()) if err != nil { return it.index, err } @@ -1446,7 +1440,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er // In eth2 the forker always returns true for reorg decision (blindly trusting // the external consensus engine), but in order to prevent the unnecessary // reorgs when importing known blocks, the special case is handled here. - if bc.GetCanonicalHash(block.NumberU64()) != block.Hash() || block.NumberU64() > current.NumberU64() { + if block.NumberU64() > current.NumberU64() || bc.GetCanonicalHash(block.NumberU64()) != block.Hash() { break } } @@ -1773,7 +1767,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i // // If the externTd was larger than our local TD, we now need to reimport the previous // blocks to regenerate the required state - reorg, err := bc.forker.Reorg(current.Header(), lastBlock.Header()) + reorg, err := bc.forker.ReorgNeeded(current.Header(), lastBlock.Header()) if err != nil { return it.index, err } diff --git a/core/forkchoice.go b/core/forkchoice.go index a7b75a91386f..b0dbb200ecc7 100644 --- a/core/forkchoice.go +++ b/core/forkchoice.go @@ -21,7 +21,6 @@ import ( "errors" "math/big" mrand "math/rand" - "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" @@ -49,7 +48,6 @@ type ChainReader interface { type ForkChoice struct { chain ChainReader rand *mrand.Rand - lock sync.RWMutex // preserve is a helper function used in td fork choice. // Miners will prefer to choose the local mined block if the @@ -71,15 +69,12 @@ func NewForkChoice(chainReader ChainReader, preserve func(header *types.Header) } } -// Reorg returns the result whether the reorg should be applied +// ReorgNeeded returns whether the reorg should be applied // based on the given external header and local canonical chain. // In the td mode, the new head is chosen if the corresponding // total difficulty is higher. In the extern mode, the trusted // header is always selected as the head. -func (f *ForkChoice) Reorg(current *types.Header, header *types.Header) (bool, error) { - f.lock.RLock() - defer f.lock.RUnlock() - +func (f *ForkChoice) ReorgNeeded(current *types.Header, header *types.Header) (bool, error) { var ( localTD = f.chain.GetTd(current.Hash(), current.Number.Uint64()) externTd = f.chain.GetTd(header.Hash(), header.Number.Uint64()) diff --git a/core/headerchain.go b/core/headerchain.go index 4c01afda7410..4f79ce62d575 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -135,7 +135,7 @@ type headerWriteResult struct { // Reorg reorgs the local canonical chain into the specified chain. The reorg // can be classified into two cases: (a) extend the local chain (b) switch the // head to the given header. -func (hc *HeaderChain) Reorg(headers []*types.Header) error { +func (hc *HeaderChain) Reorg(headers []*types.Header, inserted []rawdb.NumberHash) error { // Short circuit if nothing to reorg. if len(headers) == 0 { return nil @@ -179,11 +179,9 @@ func (hc *HeaderChain) Reorg(headers []*types.Header) error { } } // Extend the canonical chain with the new headers - for i := 0; i < len(headers); i++ { - hash := headers[i].Hash() - num := headers[i].Number.Uint64() - rawdb.WriteCanonicalHash(batch, hash, num) - rawdb.WriteHeadHeaderHash(batch, hash) + for _, hn := range inserted { + rawdb.WriteCanonicalHash(batch, hn.Hash, hn.Number) + rawdb.WriteHeadHeaderHash(batch, hn.Hash) } if err := batch.Write(); err != nil { return err @@ -199,17 +197,17 @@ func (hc *HeaderChain) Reorg(headers []*types.Header) error { // parents are already known. The chain head header won't be updated in this // function, the additional setChainHead is expected in order to finish the entire // procedure. -func (hc *HeaderChain) WriteHeaders(headers []*types.Header) (int, error) { +func (hc *HeaderChain) WriteHeaders(headers []*types.Header) ([]rawdb.NumberHash, error) { if len(headers) == 0 { - return 0, nil + return []rawdb.NumberHash{}, nil } ptd := hc.GetTd(headers[0].ParentHash, headers[0].Number.Uint64()-1) if ptd == nil { - return 0, consensus.ErrUnknownAncestor + return []rawdb.NumberHash{}, consensus.ErrUnknownAncestor } var ( newTD = new(big.Int).Set(ptd) // Total difficulty of inserted chain - inserted []numberHash // Ephemeral lookup of number/hash for the chain + inserted []rawdb.NumberHash // Ephemeral lookup of number/hash for the chain parentKnown = true // Set to true to force hc.HasHeader check the first iteration batch = hc.chainDb.NewBatch() ) @@ -235,7 +233,7 @@ func (hc *HeaderChain) WriteHeaders(headers []*types.Header) (int, error) { hc.tdCache.Add(hash, new(big.Int).Set(newTD)) rawdb.WriteHeader(batch, header) - inserted = append(inserted, numberHash{number, hash}) + inserted = append(inserted, rawdb.NumberHash{number, hash}) hc.headerCache.Add(hash, header) hc.numberCache.Add(hash, number) } @@ -244,13 +242,13 @@ func (hc *HeaderChain) WriteHeaders(headers []*types.Header) (int, error) { // Skip the slow disk write of all headers if interrupted. if hc.procInterrupt() { log.Debug("Premature abort during headers import") - return 0, errors.New("aborted") + return []rawdb.NumberHash{}, errors.New("aborted") } // Commit to disk! if err := batch.Write(); err != nil { log.Crit("Failed to write headers", "error", err) } - return len(inserted), nil + return inserted, nil } // writeHeadersAndSetHead writes a batch of block headers and applies the last @@ -270,17 +268,17 @@ func (hc *HeaderChain) writeHeadersAndSetHead(headers []*types.Header, forker *F lastHash = headers[len(headers)-1].Hash() result = &headerWriteResult{ status: NonStatTy, - ignored: len(headers) - inserted, - imported: inserted, + ignored: len(headers) - len(inserted), + imported: len(inserted), lastHash: lastHash, lastHeader: lastHeader, } ) // Ask the fork choicer if the reorg is necessary - if reorg, err := forker.Reorg(hc.CurrentHeader(), lastHeader); err != nil { + if reorg, err := forker.ReorgNeeded(hc.CurrentHeader(), lastHeader); err != nil { return nil, err } else if !reorg { - if inserted != 0 { + if len(inserted) != 0 { result.status = SideStatTy } return result, nil @@ -291,7 +289,7 @@ func (hc *HeaderChain) writeHeadersAndSetHead(headers []*types.Header, forker *F return result, nil } // Apply the reorg operation - if err := hc.Reorg(headers); err != nil { + if err := hc.Reorg(headers, inserted); err != nil { return nil, err } result.status = CanonStatTy diff --git a/light/lightchain.go b/light/lightchain.go index 72119f8755a6..4ed20c8cf0f3 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -403,7 +403,7 @@ func (lc *LightChain) SetChainHead(header *types.Header) error { lc.wg.Add(1) defer lc.wg.Done() - if err := lc.hc.Reorg([]*types.Header{header}); err != nil { + if err := lc.hc.Reorg([]*types.Header{header}, []rawdb.NumberHash{{Number: header.Number.Uint64(), Hash: header.Hash()}}); err != nil { return err } // Emit events From 54c752abdbdac8de64b20234a7349fc4ed68c36a Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Wed, 3 Nov 2021 17:32:58 +0100 Subject: [PATCH 11/32] core: use keyed field (f) --- core/headerchain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/headerchain.go b/core/headerchain.go index 4f79ce62d575..d369d21b5ea1 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -233,7 +233,7 @@ func (hc *HeaderChain) WriteHeaders(headers []*types.Header) ([]rawdb.NumberHash hc.tdCache.Add(hash, new(big.Int).Set(newTD)) rawdb.WriteHeader(batch, header) - inserted = append(inserted, rawdb.NumberHash{number, hash}) + inserted = append(inserted, rawdb.NumberHash{Number: number, Hash: hash}) hc.headerCache.Add(hash, header) hc.numberCache.Add(hash, number) } From 6963517082527b976bad09fd80e113b055ee19da Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 8 Nov 2021 13:44:07 +0100 Subject: [PATCH 12/32] core: eth: fix compilation issues + tests --- core/headerchain.go | 36 ++++++++++++++++++++++-------------- eth/catalyst/api.go | 2 +- light/lightchain.go | 2 +- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/core/headerchain.go b/core/headerchain.go index d369d21b5ea1..335945d48f08 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -135,7 +135,7 @@ type headerWriteResult struct { // Reorg reorgs the local canonical chain into the specified chain. The reorg // can be classified into two cases: (a) extend the local chain (b) switch the // head to the given header. -func (hc *HeaderChain) Reorg(headers []*types.Header, inserted []rawdb.NumberHash) error { +func (hc *HeaderChain) Reorg(headers []*types.Header) error { // Short circuit if nothing to reorg. if len(headers) == 0 { return nil @@ -179,10 +179,18 @@ func (hc *HeaderChain) Reorg(headers []*types.Header, inserted []rawdb.NumberHas } } // Extend the canonical chain with the new headers - for _, hn := range inserted { - rawdb.WriteCanonicalHash(batch, hn.Hash, hn.Number) - rawdb.WriteHeadHeaderHash(batch, hn.Hash) - } + for i := 0; i < len(headers)-1; i++ { + hash := headers[i+1].ParentHash // Save some extra hashing + num := headers[i].Number.Uint64() + rawdb.WriteCanonicalHash(batch, hash, num) + rawdb.WriteHeadHeaderHash(batch, hash) + } + // Write the last header + hash := headers[len(headers)-1].Hash() + num := headers[len(headers)-1].Number.Uint64() + rawdb.WriteCanonicalHash(batch, hash, num) + rawdb.WriteHeadHeaderHash(batch, hash) + if err := batch.Write(); err != nil { return err } @@ -197,13 +205,13 @@ func (hc *HeaderChain) Reorg(headers []*types.Header, inserted []rawdb.NumberHas // parents are already known. The chain head header won't be updated in this // function, the additional setChainHead is expected in order to finish the entire // procedure. -func (hc *HeaderChain) WriteHeaders(headers []*types.Header) ([]rawdb.NumberHash, error) { +func (hc *HeaderChain) WriteHeaders(headers []*types.Header) (int, error) { if len(headers) == 0 { - return []rawdb.NumberHash{}, nil + return 0, nil } ptd := hc.GetTd(headers[0].ParentHash, headers[0].Number.Uint64()-1) if ptd == nil { - return []rawdb.NumberHash{}, consensus.ErrUnknownAncestor + return 0, consensus.ErrUnknownAncestor } var ( newTD = new(big.Int).Set(ptd) // Total difficulty of inserted chain @@ -242,13 +250,13 @@ func (hc *HeaderChain) WriteHeaders(headers []*types.Header) ([]rawdb.NumberHash // Skip the slow disk write of all headers if interrupted. if hc.procInterrupt() { log.Debug("Premature abort during headers import") - return []rawdb.NumberHash{}, errors.New("aborted") + return 0, errors.New("aborted") } // Commit to disk! if err := batch.Write(); err != nil { log.Crit("Failed to write headers", "error", err) } - return inserted, nil + return len(inserted), nil } // writeHeadersAndSetHead writes a batch of block headers and applies the last @@ -268,8 +276,8 @@ func (hc *HeaderChain) writeHeadersAndSetHead(headers []*types.Header, forker *F lastHash = headers[len(headers)-1].Hash() result = &headerWriteResult{ status: NonStatTy, - ignored: len(headers) - len(inserted), - imported: len(inserted), + ignored: len(headers) - inserted, + imported: inserted, lastHash: lastHash, lastHeader: lastHeader, } @@ -278,7 +286,7 @@ func (hc *HeaderChain) writeHeadersAndSetHead(headers []*types.Header, forker *F if reorg, err := forker.ReorgNeeded(hc.CurrentHeader(), lastHeader); err != nil { return nil, err } else if !reorg { - if len(inserted) != 0 { + if inserted != 0 { result.status = SideStatTy } return result, nil @@ -289,7 +297,7 @@ func (hc *HeaderChain) writeHeadersAndSetHead(headers []*types.Header, forker *F return result, nil } // Apply the reorg operation - if err := hc.Reorg(headers, inserted); err != nil { + if err := hc.Reorg(headers); err != nil { return nil, err } result.status = CanonStatTy diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index a60fb599fce8..c689bdfeaca6 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -157,7 +157,7 @@ func (api *ConsensusAPI) makeEnv(parent *types.Block, header *types.Header) (*bl // The maximum acceptable reorg depth can be limited by the // finalised block somehow. TODO(rjl493456442) fix the hard- // coded number here later. - state, err = api.eth.StateAtBlock(parent, 1000, nil, false) + state, err = api.eth.StateAtBlock(parent, 1000, nil, false, false) } if err != nil { return nil, err diff --git a/light/lightchain.go b/light/lightchain.go index 4ed20c8cf0f3..72119f8755a6 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -403,7 +403,7 @@ func (lc *LightChain) SetChainHead(header *types.Header) error { lc.wg.Add(1) defer lc.wg.Done() - if err := lc.hc.Reorg([]*types.Header{header}, []rawdb.NumberHash{{Number: header.Number.Uint64(), Hash: header.Hash()}}); err != nil { + if err := lc.hc.Reorg([]*types.Header{header}); err != nil { return err } // Emit events From c0216eb378be5ba0915c0abe9fd5ec132d06f366 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 9 Nov 2021 15:17:25 +0100 Subject: [PATCH 13/32] eth/catalyst: dbetter error codes --- eth/catalyst/api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index c689bdfeaca6..27b484162350 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -45,8 +45,8 @@ var ( VALID = GenericStringResponse{"VALID"} INVALID = GenericStringResponse{"INVALID"} SYNCING = GenericStringResponse{"SYNCING"} - UnknownHeader = rpc.CustomError{Code: 4, Message: "unknown header"} - UnknownPayload = rpc.CustomError{Code: 5, Message: "unknown payload"} + UnknownHeader = rpc.CustomError{Code: -32000, Message: "unknown header"} + UnknownPayload = rpc.CustomError{Code: -32001, Message: "unknown payload"} ) // Register adds catalyst APIs to the full node. From 92fdcc469d76bb1faf1852af843b97960482ec6d Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Thu, 11 Nov 2021 11:05:37 +0100 Subject: [PATCH 14/32] all: move Merger to consensus/, remove reliance on it in bc --- accounts/abi/bind/backends/simulated.go | 2 +- cmd/utils/flags.go | 2 +- consensus/beacon/consensus.go | 26 ++--- core/block_validator_test.go | 27 +++--- core/blockchain.go | 7 -- core/blockchain_test.go | 20 ++-- core/merger.go | 121 ------------------------ eth/backend.go | 8 +- eth/catalyst/api.go | 10 +- eth/catalyst/api_test.go | 2 + eth/ethconfig/config.go | 7 +- eth/handler.go | 5 +- eth/handler_eth_test.go | 5 +- eth/handler_test.go | 3 +- les/client.go | 6 +- les/test_helper.go | 3 +- light/lightchain.go | 4 +- miner/miner.go | 2 +- miner/miner_test.go | 5 +- miner/worker.go | 4 +- miner/worker_test.go | 2 +- 21 files changed, 70 insertions(+), 201 deletions(-) delete mode 100644 core/merger.go diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 78122ecd6f0d..6854c9624e35 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -78,7 +78,7 @@ type SimulatedBackend struct { func NewSimulatedBackendWithDatabase(database ethdb.Database, alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend { genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc} genesis.MustCommit(database) - blockchain, _ := core.NewBlockChainWithMerger(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}, nil, nil, core.NewMerger(database)) + blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}, nil, nil) backend := &SimulatedBackend{ database: database, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 0dee29b3537e..9dcd775a282f 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1925,7 +1925,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai // TODO(rjl493456442) disable snapshot generation/wiping if the chain is read only. // Disable transaction indexing/unindexing by default. - chain, err = core.NewBlockChainWithMerger(chainDb, cache, config, engine, vmcfg, nil, nil, core.NewMerger(chainDb)) + chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil, nil) if err != nil { Fatalf("Can't create BlockChain: %v", err) } diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index d1fee6b5c074..f79a42aafe3f 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -59,21 +59,19 @@ var ( type Beacon struct { ethone consensus.Engine // Classic consensus engine used in eth1, e.g. ethash or clique - // transitioned is the flag whether the transition has been triggered. - // It's triggered by receiving the first "ENGINE_FORKCHOICEUPDATED" message - // from the external consensus engine. - transitioned bool - lock sync.RWMutex + // merger allows us to query whether the transition has been triggered. + merger *consensus.Merger + lock sync.RWMutex } // New creates a consensus engine with the given embedded eth1 engine. -func New(ethone consensus.Engine, transitioned bool) *Beacon { +func New(ethone consensus.Engine, merger *consensus.Merger) *Beacon { if _, ok := ethone.(*Beacon); ok { panic("nested consensus engine") } return &Beacon{ - ethone: ethone, - transitioned: transitioned, + ethone: ethone, + merger: merger, } } @@ -281,7 +279,7 @@ func (beacon *Beacon) Prepare(chain consensus.ChainHeaderReader, header *types.H defer beacon.lock.RUnlock() // Transition isn't triggered yet, use the legacy rules for preparation. - if !beacon.transitioned { + if !beacon.merger.LeftPoW() { return beacon.ethone.Prepare(chain, header) } parent := chain.GetHeader(header.ParentHash, header.Number.Uint64()-1) @@ -356,7 +354,7 @@ func (beacon *Beacon) CalcDifficulty(chain consensus.ChainHeaderReader, time uin defer beacon.lock.RUnlock() // Transition isn't triggered yet, use the legacy rules for calculation - if !beacon.transitioned { + if !beacon.merger.LeftPoW() { return beacon.ethone.CalcDifficulty(chain, time, parent) } return beaconDifficulty @@ -382,14 +380,6 @@ func (beacon *Beacon) IsPoSHeader(header *types.Header) bool { return header.Difficulty.Cmp(beaconDifficulty) == 0 } -// MarkTransitioned sets the transitioned flag. -func (beacon *Beacon) MarkTransitioned() { - beacon.lock.Lock() - defer beacon.lock.Unlock() - - beacon.transitioned = true -} - // InnerEngine returns the embedded eth1 consensus engine. func (beacon *Beacon) InnerEngine() consensus.Engine { return beacon.ethone diff --git a/core/block_validator_test.go b/core/block_validator_test.go index ad177257a474..bbe92f266a86 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -94,6 +94,7 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { postBlocks []*types.Block runEngine consensus.Engine chainConfig *params.ChainConfig + merger = consensus.NewMerger(rawdb.NewMemoryDatabase()) ) if isClique { var ( @@ -111,7 +112,8 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { copy(genspec.ExtraData[32:], addr[:]) genesis := genspec.MustCommit(testdb) - genEngine := beacon.New(engine, false) + genMerger := consensus.NewMerger(rawdb.NewMemoryDatabase()) + genEngine := beacon.New(engine, genMerger) preBlocks, _ = GenerateChain(params.AllCliqueProtocolChanges, genesis, genEngine, testdb, 8, nil) for i, block := range preBlocks { header := block.Header() @@ -125,21 +127,24 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { copy(header.Extra[len(header.Extra)-crypto.SignatureLength:], sig) preBlocks[i] = block.WithSeal(header) } - genEngine.MarkTransitioned() + genMerger.LeavePoW() + genMerger.EnterPoS() postBlocks, _ = GenerateChain(params.AllCliqueProtocolChanges, preBlocks[len(preBlocks)-1], genEngine, testdb, 8, nil) chainConfig = params.AllCliqueProtocolChanges - runEngine = beacon.New(engine, false) + runEngine = beacon.New(engine, merger) } else { gspec := &Genesis{Config: params.TestChainConfig} genesis := gspec.MustCommit(testdb) - engine := beacon.New(ethash.NewFaker(), false) + genMerger := consensus.NewMerger(rawdb.NewMemoryDatabase()) + genEngine := beacon.New(ethash.NewFaker(), genMerger) - preBlocks, _ = GenerateChain(params.TestChainConfig, genesis, engine, testdb, 8, nil) - engine.MarkTransitioned() - postBlocks, _ = GenerateChain(params.TestChainConfig, preBlocks[len(preBlocks)-1], engine, testdb, 8, nil) + preBlocks, _ = GenerateChain(params.TestChainConfig, genesis, genEngine, testdb, 8, nil) + genMerger.LeavePoW() + genMerger.EnterPoS() + postBlocks, _ = GenerateChain(params.TestChainConfig, preBlocks[len(preBlocks)-1], genEngine, testdb, 8, nil) chainConfig = params.TestChainConfig - runEngine = beacon.New(ethash.NewFaker(), false) + runEngine = beacon.New(ethash.NewFaker(), merger) } preHeaders := make([]*types.Header, len(preBlocks)) @@ -158,11 +163,7 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { } // Run the header checker for blocks one-by-one, checking for both valid and invalid nonces - merger := NewMerger(rawdb.NewMemoryDatabase()) - merger.SubscribeLeavePoW(func() { - runEngine.(*beacon.Beacon).MarkTransitioned() - }) - chain, _ := NewBlockChainWithMerger(testdb, nil, chainConfig, runEngine, vm.Config{}, nil, nil, merger) + chain, _ := NewBlockChain(testdb, nil, chainConfig, runEngine, vm.Config{}, nil, nil) defer chain.Stop() // Verify the blocks before the merging diff --git a/core/blockchain.go b/core/blockchain.go index 9fe4289972a3..b960a37fea1d 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -215,13 +215,6 @@ type BlockChain struct { // available in the database. It initialises the default Ethereum Validator // and Processor. func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(header *types.Header) bool, txLookupLimit *uint64) (*BlockChain, error) { - return NewBlockChainWithMerger(db, cacheConfig, chainConfig, engine, vmConfig, shouldPreserve, txLookupLimit, NewMerger(rawdb.NewMemoryDatabase())) -} - -// NewBlockChainWithMerger returns a fully initialised block chain using information -// available in the database. It initialises the default Ethereum Validator -// and Processor. -func NewBlockChainWithMerger(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(header *types.Header) bool, txLookupLimit *uint64, merger *Merger) (*BlockChain, error) { if cacheConfig == nil { cacheConfig = defaultCacheConfig } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index dfd373c7d3d8..b85a83ef8dd5 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -1942,8 +1942,9 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon chainConfig := *params.TestChainConfig // Generate a canonical chain to act as the main dataset var ( - genEngine = beacon.New(ethash.NewFaker(), false) - runEngine = beacon.New(ethash.NewFaker(), false) + merger = consensus.NewMerger(rawdb.NewMemoryDatabase()) + genEngine = beacon.New(ethash.NewFaker(), merger) + runEngine = beacon.New(ethash.NewFaker(), merger) db = rawdb.NewMemoryDatabase() key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -1967,8 +1968,8 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon } // Activate the transition since genesis if required if mergePoint == 0 { - genEngine.MarkTransitioned() - runEngine.MarkTransitioned() + merger.LeavePoW() + merger.EnterPoS() // Set the terminal total difficulty in the config gspec.Config.TerminalTotalDifficulty = big.NewInt(0) @@ -2000,8 +2001,8 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon // Activate the transition in the middle of the chain if mergePoint == 1 { - genEngine.MarkTransitioned() - runEngine.MarkTransitioned() + merger.LeavePoW() + merger.EnterPoS() // Set the terminal total difficulty in the config gspec.Config.TerminalTotalDifficulty = big.NewInt(int64(len(blocks))) } @@ -2222,12 +2223,13 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i var ( db = rawdb.NewMemoryDatabase() genesis = (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee), Config: &chainConfig}).MustCommit(db) - runEngine = beacon.New(ethash.NewFaker(), false) - genEngine = beacon.New(ethash.NewFaker(), false) + runMerger = consensus.NewMerger(db) + runEngine = beacon.New(ethash.NewFaker(), runMerger) + genEngine = beacon.New(ethash.NewFaker(), runMerger) ) applyMerge := func(engine *beacon.Beacon, height int) { if engine != nil { - engine.MarkTransitioned() + runMerger.EnterPoS() // Set the terminal total difficulty in the config chainConfig.TerminalTotalDifficulty = big.NewInt(int64(height)) } diff --git a/core/merger.go b/core/merger.go deleted file mode 100644 index 6734167d7486..000000000000 --- a/core/merger.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2021 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package core - -import ( - "sync" - - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" -) - -// transitionStatus describes the status of eth1/2 transition. This switch -// between modes is a one-way action which is triggered by corresponding -// consensus-layer message. -type transitionStatus struct { - LeftPoW bool // The flag is set when the first NewHead message received - EnteredPoS bool // The flag is set when the first FinalisedBlock message received -} - -// Merger is an internal help structure used to track the eth1/2 transition status. -// It's a common structure can be used in both full node and light client. -type Merger struct { - db ethdb.KeyValueStore - status transitionStatus - leavePoWCalls []func() - lock sync.Mutex -} - -func NewMerger(db ethdb.KeyValueStore) *Merger { - var status transitionStatus - blob := rawdb.ReadTransitionStatus(db) - if len(blob) != 0 { - if err := rlp.DecodeBytes(blob, &status); err != nil { - log.Crit("Failed to decode the transition status", "err", err) - } - } - return &Merger{ - db: db, - status: status, - } -} - -// SubscribeLeavePoW registers callback so that if the chain leaves -// from the PoW stage and enters to 'transition' stage it can be invoked. -func (m *Merger) SubscribeLeavePoW(callback func()) { - m.lock.Lock() - defer m.lock.Unlock() - - m.leavePoWCalls = append(m.leavePoWCalls, callback) -} - -// LeavePoW is called whenever the first NewHead message received -// from the consensus-layer. -func (m *Merger) LeavePoW() { - m.lock.Lock() - defer m.lock.Unlock() - - if m.status.LeftPoW { - return - } - m.status = transitionStatus{LeftPoW: true} - blob, err := rlp.EncodeToBytes(m.status) - if err != nil { - log.Crit("Failed to encode the transition status", "err", err) - } - rawdb.WriteTransitionStatus(m.db, blob) - for _, call := range m.leavePoWCalls { - call() - } - log.Info("Left PoW stage") -} - -// EnterPoS is called whenever the first FinalisedBlock message received -// from the consensus-layer. -func (m *Merger) EnterPoS() { - m.lock.Lock() - defer m.lock.Unlock() - - if m.status.EnteredPoS { - return - } - m.status = transitionStatus{LeftPoW: true, EnteredPoS: true} - blob, err := rlp.EncodeToBytes(m.status) - if err != nil { - log.Crit("Failed to encode the transition status", "err", err) - } - rawdb.WriteTransitionStatus(m.db, blob) - log.Info("Entered PoS stage") -} - -// LeftPoW reports whether the chain has left the PoW stage. -func (m *Merger) LeftPoW() bool { - m.lock.Lock() - defer m.lock.Unlock() - - return m.status.LeftPoW -} - -// EnteredPoS reports whether the chain has entered the PoS stage. -func (m *Merger) EnteredPoS() bool { - m.lock.Lock() - defer m.lock.Unlock() - - return m.status.EnteredPoS -} diff --git a/eth/backend.go b/eth/backend.go index c4fa7b596b42..62be14539a6b 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -72,7 +72,7 @@ type Ethereum struct { handler *handler ethDialCandidates enode.Iterator snapDialCandidates enode.Iterator - merger *core.Merger + merger *consensus.Merger // DB interfaces chainDb ethdb.Database // Block chain database @@ -142,7 +142,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb, stack.ResolvePath(config.TrieCleanCacheJournal)); err != nil { log.Error("Failed to recover state", "error", err) } - merger := core.NewMerger(chainDb) + merger := consensus.NewMerger(chainDb) eth := &Ethereum{ config: config, merger: merger, @@ -192,7 +192,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { Preimages: config.Preimages, } ) - eth.blockchain, err = core.NewBlockChainWithMerger(chainDb, cacheConfig, chainConfig, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit, merger) + eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, chainConfig, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit) if err != nil { return nil, err } @@ -528,7 +528,7 @@ func (s *Ethereum) Synced() bool { return atomic.LoadUint3 func (s *Ethereum) SetSynced() { atomic.StoreUint32(&s.handler.acceptTxs, 1) } func (s *Ethereum) ArchiveMode() bool { return s.config.NoPruning } func (s *Ethereum) BloomIndexer() *core.ChainIndexer { return s.bloomIndexer } -func (s *Ethereum) Merger() *core.Merger { return s.merger } +func (s *Ethereum) Merger() *consensus.Merger { return s.merger } func (s *Ethereum) SyncMode() downloader.SyncMode { mode, _ := s.handler.chainSync.modeAndLocalHead() return mode diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 27b484162350..9b501f167dc1 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -92,18 +92,18 @@ func NewConsensusAPI(eth *eth.Ethereum, les *les.LightEthereum) *ConsensusAPI { panic("Catalyst started without valid total difficulty") } if b, ok := les.Engine().(*beacon.Beacon); ok { - engine = beacon.New(b.InnerEngine(), true) + engine = beacon.New(b.InnerEngine(), les.Merger()) } else { - engine = beacon.New(les.Engine(), true) + engine = beacon.New(les.Engine(), les.Merger()) } } else { if eth.BlockChain().Config().TerminalTotalDifficulty == nil { panic("Catalyst started without valid total difficulty") } if b, ok := eth.Engine().(*beacon.Beacon); ok { - engine = beacon.New(b.InnerEngine(), true) + engine = beacon.New(b.InnerEngine(), eth.Merger()) } else { - engine = beacon.New(eth.Engine(), true) + engine = beacon.New(eth.Engine(), eth.Merger()) } } return &ConsensusAPI{ @@ -496,7 +496,7 @@ func (api *ConsensusAPI) setHead(newHead common.Hash) error { } // Helper function, return the merger instance. -func (api *ConsensusAPI) merger() *core.Merger { +func (api *ConsensusAPI) merger() *consensus.Merger { if api.light { return api.les.Merger() } diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index ca53cec8ffcc..67ce0b2ecf82 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -179,6 +179,7 @@ func checkLogEvents(t *testing.T, logsCh <-chan []*types.Log, rmLogsCh <-chan co func TestEth2NewBlock(t *testing.T) { genesis, preMergeBlocks := generatePreMergeChain(10) n, ethservice := startEthService(t, genesis, preMergeBlocks) + ethservice.Merger().LeavePoW() defer n.Close() var ( @@ -345,6 +346,7 @@ func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block) func TestFullAPI(t *testing.T) { genesis, preMergeBlocks := generatePreMergeChain(10) n, ethservice := startEthService(t, genesis, preMergeBlocks) + ethservice.Merger().LeavePoW() defer n.Close() var ( api = NewConsensusAPI(ethservice, nil) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 121750ada730..9eabf1e2c67f 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -211,7 +211,7 @@ type Config struct { } // CreateConsensusEngine creates a consensus engine for the given chain configuration. -func CreateConsensusEngine(stack *node.Node, chainConfig *params.ChainConfig, config *ethash.Config, notify []string, noverify bool, db ethdb.Database, merger *core.Merger) consensus.Engine { +func CreateConsensusEngine(stack *node.Node, chainConfig *params.ChainConfig, config *ethash.Config, notify []string, noverify bool, db ethdb.Database, merger *consensus.Merger) consensus.Engine { // If proof-of-authority is requested, set it up var engine consensus.Engine if chainConfig.Clique != nil { @@ -242,9 +242,6 @@ func CreateConsensusEngine(stack *node.Node, chainConfig *params.ChainConfig, co if merger == nil { return engine } - engine = beacon.New(engine, merger.LeftPoW()) - merger.SubscribeLeavePoW(func() { - engine.(*beacon.Beacon).MarkTransitioned() - }) + engine = beacon.New(engine, merger) return engine } diff --git a/eth/handler.go b/eth/handler.go index 96fe3ab872cc..e37a5936e2fb 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -25,6 +25,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" @@ -80,7 +81,7 @@ type handlerConfig struct { Database ethdb.Database // Database for direct sync insertions Chain *core.BlockChain // Blockchain to serve data from TxPool txPool // Transaction pool to propagate from - Merger *core.Merger // The manager for eth1/2 transition + Merger *consensus.Merger // The manager for eth1/2 transition Network uint64 // Network identifier to adfvertise Sync downloader.SyncMode // Whether to fast or full sync BloomCache uint64 // Megabytes to alloc for fast sync bloom @@ -110,7 +111,7 @@ type handler struct { blockFetcher *fetcher.BlockFetcher txFetcher *fetcher.TxFetcher peers *peerSet - merger *core.Merger + merger *consensus.Merger eventMux *event.TypeMux txsCh chan core.NewTxsEvent diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index 085812997b5a..38f0427d0367 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -25,6 +25,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" @@ -115,7 +116,7 @@ func testForkIDSplit(t *testing.T, protocol uint) { Database: dbNoFork, Chain: chainNoFork, TxPool: newTestTxPool(), - Merger: core.NewMerger(rawdb.NewMemoryDatabase()), + Merger: consensus.NewMerger(rawdb.NewMemoryDatabase()), Network: 1, Sync: downloader.FullSync, BloomCache: 1, @@ -124,7 +125,7 @@ func testForkIDSplit(t *testing.T, protocol uint) { Database: dbProFork, Chain: chainProFork, TxPool: newTestTxPool(), - Merger: core.NewMerger(rawdb.NewMemoryDatabase()), + Merger: consensus.NewMerger(rawdb.NewMemoryDatabase()), Network: 1, Sync: downloader.FullSync, BloomCache: 1, diff --git a/eth/handler_test.go b/eth/handler_test.go index 4a45e81ed64a..040bfe627d97 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -22,6 +22,7 @@ import ( "sync" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" @@ -149,7 +150,7 @@ func newTestHandlerWithBlocks(blocks int) *testHandler { Database: db, Chain: chain, TxPool: txpool, - Merger: core.NewMerger(rawdb.NewMemoryDatabase()), + Merger: consensus.NewMerger(rawdb.NewMemoryDatabase()), Network: 1, Sync: downloader.FastSync, BloomCache: 1, diff --git a/les/client.go b/les/client.go index 4fecc970af3e..2c0378782bc1 100644 --- a/les/client.go +++ b/les/client.go @@ -63,7 +63,7 @@ type LightEthereum struct { serverPool *vfc.ServerPool serverPoolIterator enode.Iterator pruner *pruner - merger *core.Merger + merger *consensus.Merger bloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests bloomIndexer *core.ChainIndexer // Bloom indexer operating during block imports @@ -96,7 +96,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { log.Info("Initialised chain configuration", "config", chainConfig) peers := newServerPeerSet() - merger := core.NewMerger(chainDb) + merger := consensus.NewMerger(chainDb) leth := &LightEthereum{ lesCommons: lesCommons{ genesis: genesisHash, @@ -335,7 +335,7 @@ func (s *LightEthereum) Engine() consensus.Engine { return s.engine } func (s *LightEthereum) LesVersion() int { return int(ClientProtocolVersions[0]) } func (s *LightEthereum) Downloader() *downloader.Downloader { return s.handler.downloader } func (s *LightEthereum) EventMux() *event.TypeMux { return s.eventMux } -func (s *LightEthereum) Merger() *core.Merger { return s.merger } +func (s *LightEthereum) Merger() *consensus.Merger { return s.merger } // Protocols returns all the currently configured network protocols to start. func (s *LightEthereum) Protocols() []p2p.Protocol { diff --git a/les/test_helper.go b/les/test_helper.go index 2c5a56669b42..10367ea800c4 100644 --- a/les/test_helper.go +++ b/les/test_helper.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/contracts/checkpointoracle/contract" "github.com/ethereum/go-ethereum/core" @@ -239,7 +240,7 @@ func newTestClientHandler(backend *backends.SimulatedBackend, odr *LesOdr, index engine: engine, blockchain: chain, eventMux: evmux, - merger: core.NewMerger(rawdb.NewMemoryDatabase()), + merger: consensus.NewMerger(rawdb.NewMemoryDatabase()), } client.handler = newClientHandler(ulcServers, ulcFraction, nil, client) diff --git a/light/lightchain.go b/light/lightchain.go index 72119f8755a6..d3613f388cce 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -79,13 +79,13 @@ type LightChain struct { // available in the database. It initialises the default Ethereum header // validator. func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus.Engine, checkpoint *params.TrustedCheckpoint) (*LightChain, error) { - return NewLightChainWithMerger(odr, config, engine, checkpoint, core.NewMerger(rawdb.NewMemoryDatabase())) + return NewLightChainWithMerger(odr, config, engine, checkpoint, consensus.NewMerger(rawdb.NewMemoryDatabase())) } // NewLightChain returns a fully initialised light chain using information // available in the database. It initialises the default Ethereum header // validator. -func NewLightChainWithMerger(odr OdrBackend, config *params.ChainConfig, engine consensus.Engine, checkpoint *params.TrustedCheckpoint, merger *core.Merger) (*LightChain, error) { +func NewLightChainWithMerger(odr OdrBackend, config *params.ChainConfig, engine consensus.Engine, checkpoint *params.TrustedCheckpoint, merger *consensus.Merger) (*LightChain, error) { bodyCache, _ := lru.New(bodyCacheLimit) bodyRLPCache, _ := lru.New(bodyCacheLimit) blockCache, _ := lru.New(blockCacheLimit) diff --git a/miner/miner.go b/miner/miner.go index 96bc7215b322..c8aaa5b92842 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -68,7 +68,7 @@ type Miner struct { wg sync.WaitGroup } -func New(eth Backend, config *Config, chainConfig *params.ChainConfig, mux *event.TypeMux, engine consensus.Engine, isLocalBlock func(header *types.Header) bool, merger *core.Merger) *Miner { +func New(eth Backend, config *Config, chainConfig *params.ChainConfig, mux *event.TypeMux, engine consensus.Engine, isLocalBlock func(header *types.Header) bool, merger *consensus.Merger) *Miner { miner := &Miner{ eth: eth, mux: mux, diff --git a/miner/miner_test.go b/miner/miner_test.go index a4324ee67e61..2c4588ac7168 100644 --- a/miner/miner_test.go +++ b/miner/miner_test.go @@ -22,6 +22,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" @@ -245,8 +246,8 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux) { // Create consensus engine engine := clique.New(chainConfig.Clique, chainDB) // Create Ethereum backend - merger := core.NewMerger(rawdb.NewMemoryDatabase()) - bc, err := core.NewBlockChainWithMerger(chainDB, nil, chainConfig, engine, vm.Config{}, nil, nil, merger) + merger := consensus.NewMerger(rawdb.NewMemoryDatabase()) + bc, err := core.NewBlockChain(chainDB, nil, chainConfig, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("can't create new chain %v", err) } diff --git a/miner/worker.go b/miner/worker.go index 0705a329e25d..0266fd44ec91 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -128,7 +128,7 @@ type worker struct { engine consensus.Engine eth Backend chain *core.BlockChain - merger *core.Merger + merger *consensus.Merger // Feeds pendingLogsFeed event.Feed @@ -191,7 +191,7 @@ type worker struct { resubmitHook func(time.Duration, time.Duration) // Method to call upon updating resubmitting interval. } -func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux, isLocalBlock func(header *types.Header) bool, init bool, merger *core.Merger) *worker { +func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux, isLocalBlock func(header *types.Header) bool, init bool, merger *consensus.Merger) *worker { worker := &worker{ config: config, chainConfig: chainConfig, diff --git a/miner/worker_test.go b/miner/worker_test.go index cbb4703d372f..c8ddd2c320b8 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -197,7 +197,7 @@ func (b *testWorkerBackend) newRandomTx(creation bool) *types.Transaction { func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, db ethdb.Database, blocks int) (*worker, *testWorkerBackend) { backend := newTestWorkerBackend(t, chainConfig, engine, db, blocks) backend.txPool.AddLocals(pendingTxs) - w := newWorker(testConfig, chainConfig, engine, backend, new(event.TypeMux), nil, false, core.NewMerger(rawdb.NewMemoryDatabase())) + w := newWorker(testConfig, chainConfig, engine, backend, new(event.TypeMux), nil, false, consensus.NewMerger(rawdb.NewMemoryDatabase())) w.setEtherbase(testBankAddress) return w, backend } From 148f1fd088c4840d02ac8c5a6d41c1647cabc4aa Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Thu, 11 Nov 2021 11:10:15 +0100 Subject: [PATCH 15/32] all: renamed EnterPoS and LeavePoW to ReachTDD and FinalizePoS --- consensus/beacon/consensus.go | 4 ++-- core/block_validator_test.go | 12 ++++++------ core/blockchain_test.go | 10 +++++----- eth/catalyst/api.go | 10 +++++----- eth/catalyst/api_test.go | 4 ++-- eth/handler.go | 8 ++++---- eth/handler_eth.go | 4 ++-- eth/sync.go | 2 +- les/client_handler.go | 4 ++-- miner/worker.go | 2 +- 10 files changed, 30 insertions(+), 30 deletions(-) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index f79a42aafe3f..d56575478dea 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -279,7 +279,7 @@ func (beacon *Beacon) Prepare(chain consensus.ChainHeaderReader, header *types.H defer beacon.lock.RUnlock() // Transition isn't triggered yet, use the legacy rules for preparation. - if !beacon.merger.LeftPoW() { + if !beacon.merger.TDDReached() { return beacon.ethone.Prepare(chain, header) } parent := chain.GetHeader(header.ParentHash, header.Number.Uint64()-1) @@ -354,7 +354,7 @@ func (beacon *Beacon) CalcDifficulty(chain consensus.ChainHeaderReader, time uin defer beacon.lock.RUnlock() // Transition isn't triggered yet, use the legacy rules for calculation - if !beacon.merger.LeftPoW() { + if !beacon.merger.TDDReached() { return beacon.ethone.CalcDifficulty(chain, time, parent) } return beaconDifficulty diff --git a/core/block_validator_test.go b/core/block_validator_test.go index bbe92f266a86..8fd3ae585d12 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -127,8 +127,8 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { copy(header.Extra[len(header.Extra)-crypto.SignatureLength:], sig) preBlocks[i] = block.WithSeal(header) } - genMerger.LeavePoW() - genMerger.EnterPoS() + genMerger.ReachTTD() + genMerger.FinalizePoS() postBlocks, _ = GenerateChain(params.AllCliqueProtocolChanges, preBlocks[len(preBlocks)-1], genEngine, testdb, 8, nil) chainConfig = params.AllCliqueProtocolChanges runEngine = beacon.New(engine, merger) @@ -139,8 +139,8 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { genEngine := beacon.New(ethash.NewFaker(), genMerger) preBlocks, _ = GenerateChain(params.TestChainConfig, genesis, genEngine, testdb, 8, nil) - genMerger.LeavePoW() - genMerger.EnterPoS() + genMerger.ReachTTD() + genMerger.FinalizePoS() postBlocks, _ = GenerateChain(params.TestChainConfig, preBlocks[len(preBlocks)-1], genEngine, testdb, 8, nil) chainConfig = params.TestChainConfig @@ -188,8 +188,8 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { } // Make the transition - merger.LeavePoW() - merger.EnterPoS() + merger.ReachTTD() + merger.FinalizePoS() // Verify the blocks after the merging for i := 0; i < len(postBlocks); i++ { diff --git a/core/blockchain_test.go b/core/blockchain_test.go index b85a83ef8dd5..1b81ad6b206b 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -1968,8 +1968,8 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon } // Activate the transition since genesis if required if mergePoint == 0 { - merger.LeavePoW() - merger.EnterPoS() + merger.ReachTTD() + merger.FinalizePoS() // Set the terminal total difficulty in the config gspec.Config.TerminalTotalDifficulty = big.NewInt(0) @@ -2001,8 +2001,8 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon // Activate the transition in the middle of the chain if mergePoint == 1 { - merger.LeavePoW() - merger.EnterPoS() + merger.ReachTTD() + merger.FinalizePoS() // Set the terminal total difficulty in the config gspec.Config.TerminalTotalDifficulty = big.NewInt(int64(len(blocks))) } @@ -2229,7 +2229,7 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i ) applyMerge := func(engine *beacon.Beacon, height int) { if engine != nil { - runMerger.EnterPoS() + runMerger.FinalizePoS() // Set the terminal total difficulty in the config chainConfig.TerminalTotalDifficulty = big.NewInt(int64(height)) } diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 9b501f167dc1..e99e6047e9d1 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -249,8 +249,8 @@ func (api *ConsensusAPI) ExecutePayload(params ExecutableData) (GenericStringRes return INVALID, err } merger := api.merger() - if !merger.LeftPoW() { - merger.LeavePoW() + if !merger.TDDReached() { + merger.ReachTTD() } return VALID, nil } @@ -439,7 +439,7 @@ func (api *ConsensusAPI) insertTransactions(txs types.Transactions) error { func (api *ConsensusAPI) checkTerminalTotalDifficulty(head common.Hash) error { // shortcut if we entered PoS already - if api.merger().EnteredPoS() { + if api.merger().PoSFinalized() { return nil } // make sure the parent has enough terminal total difficulty @@ -462,8 +462,8 @@ func (api *ConsensusAPI) checkTerminalTotalDifficulty(head common.Hash) error { func (api *ConsensusAPI) setHead(newHead common.Hash) error { // Trigger the transition if it's the first `NewHead` event. merger := api.merger() - if !merger.EnteredPoS() { - merger.EnterPoS() + if !merger.PoSFinalized() { + merger.FinalizePoS() } log.Info("Setting head", "head", newHead) if api.light { diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 67ce0b2ecf82..98130673eb74 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -179,7 +179,7 @@ func checkLogEvents(t *testing.T, logsCh <-chan []*types.Log, rmLogsCh <-chan co func TestEth2NewBlock(t *testing.T) { genesis, preMergeBlocks := generatePreMergeChain(10) n, ethservice := startEthService(t, genesis, preMergeBlocks) - ethservice.Merger().LeavePoW() + ethservice.Merger().ReachTTD() defer n.Close() var ( @@ -346,7 +346,7 @@ func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block) func TestFullAPI(t *testing.T) { genesis, preMergeBlocks := generatePreMergeChain(10) n, ethservice := startEthService(t, genesis, preMergeBlocks) - ethservice.Merger().LeavePoW() + ethservice.Merger().ReachTTD() defer n.Close() var ( api = NewConsensusAPI(ethservice, nil) diff --git a/eth/handler.go b/eth/handler.go index e37a5936e2fb..d9529dd69ac9 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -193,7 +193,7 @@ func newHandler(config *handlerConfig) (*handler, error) { validator := func(header *types.Header) error { // All the block fetcher activities should be disabled // after the transition. Print the warning log. - if h.merger.EnteredPoS() { + if h.merger.PoSFinalized() { log.Warn("Unexpected validation activity", "hash", header.Hash(), "number", header.Number) return errors.New("unexpected behavior after transition") } @@ -214,7 +214,7 @@ func newHandler(config *handlerConfig) (*handler, error) { inserter := func(blocks types.Blocks) (int, error) { // All the block fetcher activities should be disabled // after the transition. Print the warning log. - if h.merger.EnteredPoS() { + if h.merger.PoSFinalized() { var ctx []interface{} ctx = append(ctx, "blocks", len(blocks)) if len(blocks) > 0 { @@ -245,7 +245,7 @@ func newHandler(config *handlerConfig) (*handler, error) { log.Warn("Fast syncing, discarded propagated block", "number", blocks[0].Number(), "hash", blocks[0].Hash()) return 0, nil } - if h.merger.LeftPoW() { + if h.merger.TDDReached() { // The blocks from the p2p network is regarded as untrusted // after the transition. In theory block gossip should be disabled // entirely whenever the transition is started. But in order to @@ -491,7 +491,7 @@ func (h *handler) Stop() { func (h *handler) BroadcastBlock(block *types.Block, propagate bool) { // Disable the block propagation if the chain has already entered the PoS // stage. The block propagation is delegated to the consensus layer. - if h.merger.EnteredPoS() { + if h.merger.PoSFinalized() { return } // Disable the block propagation if it's the post-merge block. diff --git a/eth/handler_eth.go b/eth/handler_eth.go index 1788a5849c1b..294e27296dee 100644 --- a/eth/handler_eth.go +++ b/eth/handler_eth.go @@ -183,7 +183,7 @@ func (h *ethHandler) handleBlockAnnounces(peer *eth.Peer, hashes []common.Hash, // Drop all incoming block announces from the p2p network if // the chain already entered the pos stage and disconnect the // remote peer. - if h.merger.EnteredPoS() { + if h.merger.PoSFinalized() { return errors.New("unexpected block announces") } // Schedule all the unknown hashes for retrieval @@ -209,7 +209,7 @@ func (h *ethHandler) handleBlockBroadcast(peer *eth.Peer, block *types.Block, td // Drop all incoming block announces from the p2p network if // the chain already entered the pos stage and disconnect the // remote peer. - if h.merger.EnteredPoS() { + if h.merger.PoSFinalized() { return errors.New("unexpected block announces") } // Schedule the block for import diff --git a/eth/sync.go b/eth/sync.go index 8902ba3d041f..61a6981fbb5f 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -146,7 +146,7 @@ func (cs *chainSyncer) nextSyncOp() *chainSyncOp { return nil // Sync already running. } // Disable the td based sync trigger after the transition - if cs.handler.merger.LeftPoW() { + if cs.handler.merger.TDDReached() { return nil } // Ensure we're at minimum peer count. diff --git a/les/client_handler.go b/les/client_handler.go index fd78ab34877f..db5eb8a640d0 100644 --- a/les/client_handler.go +++ b/les/client_handler.go @@ -146,7 +146,7 @@ func (h *clientHandler) handle(p *serverPeer, noInitAnnounce bool) error { // Discard all the announces after the transition // Also discarding initial signal to prevent syncing during testing. - if !(noInitAnnounce || h.backend.merger.LeftPoW()) { + if !(noInitAnnounce || h.backend.merger.TDDReached()) { h.fetcher.announce(p, &announceData{Hash: p.headInfo.Hash, Number: p.headInfo.Number, Td: p.headInfo.Td}) } @@ -216,7 +216,7 @@ func (h *clientHandler) handleMsg(p *serverPeer) error { p.updateHead(req.Hash, req.Number, req.Td) // Discard all the announces after the transition - if !h.backend.merger.LeftPoW() { + if !h.backend.merger.TDDReached() { h.fetcher.announce(p, &req) } } diff --git a/miner/worker.go b/miner/worker.go index 0266fd44ec91..54932a474deb 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -1040,7 +1040,7 @@ func (w *worker) commit(uncles []*types.Header, interval func(), update bool, st return err } - if w.isRunning() && !w.merger.LeftPoW() { + if w.isRunning() && !w.merger.TDDReached() { if interval != nil { interval() } From c9a852d05f4305569da8d089efe837141fcdc0e6 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Thu, 11 Nov 2021 11:12:26 +0100 Subject: [PATCH 16/32] core: make mergelogs a function --- core/blockchain.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index b960a37fea1d..196c09dfcb2d 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1887,7 +1887,7 @@ func (bc *BlockChain) collectLogs(hash common.Hash, removed bool) []*types.Log { } // mergeLogs returns a merged log slice with specified sort order. -func (bc *BlockChain) mergeLogs(logs [][]*types.Log, reverse bool) []*types.Log { +func mergeLogs(logs [][]*types.Log, reverse bool) []*types.Log { var ret []*types.Log if reverse { for i := len(logs) - 1; i >= 0; i-- { @@ -2032,10 +2032,10 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // ever happens if we're reorging empty blocks, which will only happen on idle // networks where performance is not an issue either way. if len(deletedLogs) > 0 { - bc.rmLogsFeed.Send(RemovedLogsEvent{bc.mergeLogs(deletedLogs, true)}) + bc.rmLogsFeed.Send(RemovedLogsEvent{mergeLogs(deletedLogs, true)}) } if len(rebirthLogs) > 0 { - bc.logsFeed.Send(bc.mergeLogs(rebirthLogs, false)) + bc.logsFeed.Send(mergeLogs(rebirthLogs, false)) } if len(oldChain) > 0 { for i := len(oldChain) - 1; i >= 0; i-- { From 7f5469047f970a0ae08d0dc922fc2b226a40374f Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Thu, 11 Nov 2021 13:12:22 +0100 Subject: [PATCH 17/32] core: use InsertChain instead of InsertBlock --- core/blockchain.go | 177 +++++++------------------------------------- eth/catalyst/api.go | 2 +- eth/handler.go | 2 +- 3 files changed, 30 insertions(+), 151 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 196c09dfcb2d..f2a1d872c2b7 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1366,7 +1366,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { return 0, errChainStopped } defer bc.chainmu.Unlock() - return bc.insertChain(chain, true) + return bc.insertChain(chain, true, false) } // insertChain is the internal implementation of InsertChain, which assumes that @@ -1377,7 +1377,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { // racey behaviour. If a sidechain import is in progress, and the historic state // is imported, but then new canon-head is added before the actual sidechain // completes, then the historic state could be pruned again -func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, error) { +func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, catalystMode bool) (int, error) { // If the chain is terminating, don't even bother starting up. if bc.insertStopped() { return 0, nil @@ -1468,7 +1468,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er return bc.insertSideChain(block, it) // First block is future, shove it (and all children) to the future queue (unknown ancestor) - case errors.Is(err, consensus.ErrFutureBlock) || (errors.Is(err, consensus.ErrUnknownAncestor) && bc.futureBlocks.Contains(it.first().ParentHash())): + case !catalystMode && (errors.Is(err, consensus.ErrFutureBlock) || (errors.Is(err, consensus.ErrUnknownAncestor) && bc.futureBlocks.Contains(it.first().ParentHash()))): for block != nil && (it.index == 0 || errors.Is(err, consensus.ErrUnknownAncestor)) { log.Debug("Future block, postponing import", "number", block.Number(), "hash", block.Hash()) if err := bc.addFutureBlock(block); err != nil { @@ -1621,15 +1621,24 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er // Update the metrics touched during block validation accountHashTimer.Update(statedb.AccountHashes) // Account hashes are complete, we can mark them storageHashTimer.Update(statedb.StorageHashes) // Storage hashes are complete, we can mark them - blockValidationTimer.Update(time.Since(substart) - (statedb.AccountHashes + statedb.StorageHashes - triehash)) // Write the block to the chain and get the status. substart = time.Now() - status, err := bc.writeBlockAndSetHead(block, receipts, logs, statedb, false) - atomic.StoreUint32(&followupInterrupt, 1) - if err != nil { - return it.index, err + var status WriteStatus + if catalystMode { + // In catalyst mode we don't set the head, only insert the block + err := bc.writeBlockWithState(block, receipts, logs, statedb) + atomic.StoreUint32(&followupInterrupt, 1) + if err != nil { + return it.index, err + } + } else { + status, err = bc.writeBlockAndSetHead(block, receipts, logs, statedb, false) + atomic.StoreUint32(&followupInterrupt, 1) + if err != nil { + return it.index, err + } } // Update the metrics touched during block commit accountCommitTimer.Update(statedb.AccountCommits) // Account commits are complete, we can mark them @@ -1639,6 +1648,11 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er blockWriteTimer.Update(time.Since(substart) - statedb.AccountCommits - statedb.StorageCommits - statedb.SnapshotCommits) blockInsertTimer.UpdateSince(start) + if catalystMode { + log.Info("Inserted block", "number", block.Number(), "hash", block.Hash(), "txs", len(block.Transactions()), "elapsed", common.PrettyDuration(time.Since(start))) + return it.index, nil + } + switch status { case CanonStatTy: log.Debug("Inserted new block", "number", block.Number(), "hash", block.Hash(), @@ -1801,7 +1815,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i // memory here. if len(blocks) >= 2048 || memory > 64*1024*1024 { log.Info("Importing heavy sidechain segment", "blocks", len(blocks), "start", blocks[0].NumberU64(), "end", block.NumberU64()) - if _, err := bc.insertChain(blocks, false); err != nil { + if _, err := bc.insertChain(blocks, false, false); err != nil { return 0, err } blocks, memory = blocks[:0], 0 @@ -1815,54 +1829,11 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i } if len(blocks) > 0 { log.Info("Importing sidechain segment", "start", blocks[0].NumberU64(), "end", blocks[len(blocks)-1].NumberU64()) - return bc.insertChain(blocks, false) + return bc.insertChain(blocks, false, false) } return 0, nil } -// recoverAncestors finds the closest ancestor with available state and re-execute -// all the ancestor blocks since that. -func (bc *BlockChain) recoverAncestors(block *types.Block) error { - // Gather all the sidechain hashes (full blocks may be memory heavy) - var ( - hashes []common.Hash - numbers []uint64 - parent = block - ) - for parent != nil && !bc.HasState(parent.Root()) { - hashes = append(hashes, parent.Hash()) - numbers = append(numbers, parent.NumberU64()) - parent = bc.GetBlock(parent.ParentHash(), parent.NumberU64()-1) - - // If the chain is terminating, stop iteration - if bc.insertStopped() { - log.Debug("Abort during blocks iteration") - return errInsertionInterrupted - } - } - if parent == nil { - return errors.New("missing parent") - } - // Import all the pruned blocks to make the state available - for i := len(hashes) - 1; i >= 0; i-- { - // If the chain is terminating, stop processing blocks - if bc.insertStopped() { - log.Debug("Abort during blocks processing") - return errInsertionInterrupted - } - var b *types.Block - if i == 0 { - b = block - } else { - b = bc.GetBlock(hashes[i], numbers[i]) - } - if err := bc.insertBlock(b); err != nil { - return err - } - } - return nil -} - // collectLogs collects the logs that were generated or removed during // the processing of the block that corresponds with the given hash. // These logs are later announced as deleted or reborn. @@ -2045,111 +2016,19 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { return nil } -// InsertBlock executes the block, runs the necessary verification +// InsertCatalystBlock executes the block, runs the necessary verification // upon it and then persist the block and the associate state into the database. // The key difference between the InsertChain is it won't do the canonical chain // updating. It relies on the additional SetChainHead call to finalize the entire // procedure. -func (bc *BlockChain) InsertBlock(block *types.Block) error { +func (bc *BlockChain) InsertCatalystBlock(block *types.Block) error { if !bc.chainmu.TryLock() { return errChainStopped } defer bc.chainmu.Unlock() - return bc.insertBlock(block) -} - -// insertBlock is the inner version of InsertBlock without holding the lock. -func (bc *BlockChain) insertBlock(block *types.Block) error { - // If the chain is terminating, don't even bother starting up - if bc.insertStopped() { - return errInsertionInterrupted - } - // Start a parallel signature recovery (signer will fluke on fork transition, minimal perf loss) - senderCacher.recoverFromBlocks(types.MakeSigner(bc.chainConfig, block.Number()), []*types.Block{block}) - - // Verify the header with the given consensus engine. - if err := bc.engine.VerifyHeader(bc, block.Header(), true); err != nil { - return err - } - // Verify the block body then. - err := bc.validator.ValidateBody(block) - switch { - // The block is already known, return without any error - // but print a warning log. It's not expected behavior. - case errors.Is(err, ErrKnownBlock): - log.Warn("Received known block", "hash", block.Hash(), "number", block.NumberU64()) - return nil - - // The parent is pruned, try to recover the parent state - case errors.Is(err, consensus.ErrPrunedAncestor): - log.Debug("Pruned ancestor", "number", block.Number(), "hash", block.Hash()) - return bc.recoverAncestors(block) - - // Some other error occurred(include the future block), abort - case err != nil: - return err - } - // If the header is a banned one, straight out abort - if BadHashes[block.Hash()] { - bc.reportBlock(block, nil, ErrBannedHash) - return ErrBannedHash - } - // Retrieve the parent block and it's state to execute on top - start := time.Now() - parent := bc.GetHeader(block.ParentHash(), block.NumberU64()-1) - statedb, err := state.New(parent.Root, bc.stateCache, bc.snaps) - if err != nil { - return err - } - // Enable prefetching to pull in trie node paths while processing transactions - statedb.StartPrefetcher("chain") - defer statedb.StopPrefetcher() - - // Process block using the parent state as reference point - substart := time.Now() - receipts, logs, usedGas, err := bc.processor.Process(block, statedb, bc.vmConfig) - if err != nil { - bc.reportBlock(block, receipts, err) - return err - } - // Update the metrics touched during block processing - accountReadTimer.Update(statedb.AccountReads) // Account reads are complete, we can mark them - storageReadTimer.Update(statedb.StorageReads) // Storage reads are complete, we can mark them - accountUpdateTimer.Update(statedb.AccountUpdates) // Account updates are complete, we can mark them - storageUpdateTimer.Update(statedb.StorageUpdates) // Storage updates are complete, we can mark them - snapshotAccountReadTimer.Update(statedb.SnapshotAccountReads) // Account reads are complete, we can mark them - snapshotStorageReadTimer.Update(statedb.SnapshotStorageReads) // Storage reads are complete, we can mark them - triehash := statedb.AccountHashes + statedb.StorageHashes // Save to not double count in validation - trieproc := statedb.SnapshotAccountReads + statedb.AccountReads + statedb.AccountUpdates - trieproc += statedb.SnapshotStorageReads + statedb.StorageReads + statedb.StorageUpdates - blockExecutionTimer.Update(time.Since(substart) - trieproc - triehash) - - // Validate the state using the default validator - substart = time.Now() - if err := bc.validator.ValidateState(block, statedb, receipts, usedGas); err != nil { - bc.reportBlock(block, receipts, err) - return err - } - // Update the metrics touched during block validation - accountHashTimer.Update(statedb.AccountHashes) // Account hashes are complete, we can mark them - storageHashTimer.Update(statedb.StorageHashes) // Storage hashes are complete, we can mark them - blockValidationTimer.Update(time.Since(substart) - (statedb.AccountHashes + statedb.StorageHashes - triehash)) - - // Write the block to the chain and get the status. - substart = time.Now() - if err := bc.writeBlockWithState(block, receipts, logs, statedb); err != nil { - return err - } - // Update the metrics touched during block commit - accountCommitTimer.Update(statedb.AccountCommits) // Account commits are complete, we can mark them - storageCommitTimer.Update(statedb.StorageCommits) // Storage commits are complete, we can mark them - snapshotCommitTimer.Update(statedb.SnapshotCommits) // Snapshot commits are complete, we can mark them - - blockWriteTimer.Update(time.Since(substart) - statedb.AccountCommits - statedb.StorageCommits - statedb.SnapshotCommits) - blockInsertTimer.UpdateSince(start) - log.Info("Inserted block", "number", block.Number(), "hash", block.Hash(), "txs", len(block.Transactions()), "elapsed", common.PrettyDuration(time.Since(start))) - return nil + _, err := bc.insertChain(types.Blocks{block}, true, true) + return err } // SetChainHead rewinds the chain to set the new head block as the specified diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index e99e6047e9d1..ec43b3700665 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -245,7 +245,7 @@ func (api *ConsensusAPI) ExecutePayload(params ExecutableData) (GenericStringRes if td.Cmp(ttd) < 0 { return INVALID, fmt.Errorf("can not execute payload on top of block with low td got: %v threshold %v", td, ttd) } - if err := api.eth.BlockChain().InsertBlock(block); err != nil { + if err := api.eth.BlockChain().InsertCatalystBlock(block); err != nil { return INVALID, err } merger := api.merger() diff --git a/eth/handler.go b/eth/handler.go index d9529dd69ac9..7500a7eae12a 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -262,7 +262,7 @@ func newHandler(config *handlerConfig) (*handler, error) { log.Info("Filtered out non-termimal pow block", "number", block.NumberU64(), "hash", block.Hash()) return 0, nil } - if err := h.chain.InsertBlock(block); err != nil { + if err := h.chain.InsertCatalystBlock(block); err != nil { return i, err } } From e8ce790002238ec67b8c264b65da47f60a9cd09c Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Thu, 11 Nov 2021 15:34:08 +0100 Subject: [PATCH 18/32] les: drop merger from lightchain object --- les/client.go | 2 +- light/lightchain.go | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/les/client.go b/les/client.go index 2c0378782bc1..05b54fbc4aa5 100644 --- a/les/client.go +++ b/les/client.go @@ -141,7 +141,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { } // Note: NewLightChain adds the trusted checkpoint so it needs an ODR with // indexers already set but not started yet - if leth.blockchain, err = light.NewLightChainWithMerger(leth.odr, leth.chainConfig, leth.engine, checkpoint, merger); err != nil { + if leth.blockchain, err = light.NewLightChain(leth.odr, leth.chainConfig, leth.engine, checkpoint); err != nil { return nil, err } leth.chainReader = leth.blockchain diff --git a/light/lightchain.go b/light/lightchain.go index d3613f388cce..61309ce35601 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -79,13 +79,6 @@ type LightChain struct { // available in the database. It initialises the default Ethereum header // validator. func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus.Engine, checkpoint *params.TrustedCheckpoint) (*LightChain, error) { - return NewLightChainWithMerger(odr, config, engine, checkpoint, consensus.NewMerger(rawdb.NewMemoryDatabase())) -} - -// NewLightChain returns a fully initialised light chain using information -// available in the database. It initialises the default Ethereum header -// validator. -func NewLightChainWithMerger(odr OdrBackend, config *params.ChainConfig, engine consensus.Engine, checkpoint *params.TrustedCheckpoint, merger *consensus.Merger) (*LightChain, error) { bodyCache, _ := lru.New(bodyCacheLimit) bodyRLPCache, _ := lru.New(bodyCacheLimit) blockCache, _ := lru.New(blockCacheLimit) From 9304474b805e12f733568737c0de3e84b697425d Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Thu, 11 Nov 2021 20:03:06 +0100 Subject: [PATCH 19/32] consensus: add merger --- consensus/merger.go | 109 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 consensus/merger.go diff --git a/consensus/merger.go b/consensus/merger.go new file mode 100644 index 000000000000..66cd47f4d311 --- /dev/null +++ b/consensus/merger.go @@ -0,0 +1,109 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package consensus + +import ( + "sync" + + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" +) + +// transitionStatus describes the status of eth1/2 transition. This switch +// between modes is a one-way action which is triggered by corresponding +// consensus-layer message. +type transitionStatus struct { + LeftPoW bool // The flag is set when the first NewHead message received + EnteredPoS bool // The flag is set when the first FinalisedBlock message received +} + +// Merger is an internal help structure used to track the eth1/2 transition status. +// It's a common structure can be used in both full node and light client. +type Merger struct { + db ethdb.KeyValueStore + status transitionStatus + lock sync.Mutex +} + +func NewMerger(db ethdb.KeyValueStore) *Merger { + var status transitionStatus + blob := rawdb.ReadTransitionStatus(db) + if len(blob) != 0 { + if err := rlp.DecodeBytes(blob, &status); err != nil { + log.Crit("Failed to decode the transition status", "err", err) + } + } + return &Merger{ + db: db, + status: status, + lock: sync.Mutex{}, + } +} + +// ReachTTD is called whenever the first NewHead message received +// from the consensus-layer. +func (m *Merger) ReachTTD() { + m.lock.Lock() + defer m.lock.Unlock() + + if m.status.LeftPoW { + return + } + m.status = transitionStatus{LeftPoW: true} + blob, err := rlp.EncodeToBytes(m.status) + if err != nil { + log.Crit("Failed to encode the transition status", "err", err) + } + rawdb.WriteTransitionStatus(m.db, blob) + log.Info("Left PoW stage") +} + +// FinalizePoS is called whenever the first FinalisedBlock message received +// from the consensus-layer. +func (m *Merger) FinalizePoS() { + m.lock.Lock() + defer m.lock.Unlock() + + if m.status.EnteredPoS { + return + } + m.status = transitionStatus{LeftPoW: true, EnteredPoS: true} + blob, err := rlp.EncodeToBytes(m.status) + if err != nil { + log.Crit("Failed to encode the transition status", "err", err) + } + rawdb.WriteTransitionStatus(m.db, blob) + log.Info("Entered PoS stage") +} + +// TDDReached reports whether the chain has left the PoW stage. +func (m *Merger) TDDReached() bool { + m.lock.Lock() + defer m.lock.Unlock() + + return m.status.LeftPoW +} + +// PoSFinalized reports whether the chain has entered the PoS stage. +func (m *Merger) PoSFinalized() bool { + m.lock.Lock() + defer m.lock.Unlock() + + return m.status.EnteredPoS +} From 6ccb36ca5261bd25b568e55d935b572ff0242fd3 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 15 Nov 2021 10:35:25 +0100 Subject: [PATCH 20/32] core: recoverAncestors in catalyst mode --- core/blockchain.go | 61 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index f2a1d872c2b7..b8191bc2abd7 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1462,10 +1462,15 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, catalystMode // Falls through to the block import } switch { - // First block is pruned, insert as sidechain and reorg only if TD grows enough - case errors.Is(err, consensus.ErrPrunedAncestor): + // We're in catalyst mode and the parent is pruned, try to recover the parent state + case catalystMode && errors.Is(err, consensus.ErrPrunedAncestor): + log.Debug("Pruned ancestor", "number", block.Number(), "hash", block.Hash()) + return it.index, bc.recoverAncestors(block) + + // First block is pruned, insert as sidechain and reorg only if TD grows enough + case !catalystMode && errors.Is(err, consensus.ErrPrunedAncestor): log.Debug("Pruned ancestor, inserting as sidechain", "number", block.Number(), "hash", block.Hash()) - return bc.insertSideChain(block, it) + return bc.insertSideChain(block, it, catalystMode) // First block is future, shove it (and all children) to the future queue (unknown ancestor) case !catalystMode && (errors.Is(err, consensus.ErrFutureBlock) || (errors.Is(err, consensus.ErrUnknownAncestor) && bc.futureBlocks.Contains(it.first().ParentHash()))): @@ -1711,7 +1716,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, catalystMode // // The method writes all (header-and-body-valid) blocks to disk, then tries to // switch over to the new chain if the TD exceeded the current chain. -func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (int, error) { +func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator, catalystMode bool) (int, error) { var ( externTd *big.Int lastBlock = block @@ -1815,7 +1820,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i // memory here. if len(blocks) >= 2048 || memory > 64*1024*1024 { log.Info("Importing heavy sidechain segment", "blocks", len(blocks), "start", blocks[0].NumberU64(), "end", block.NumberU64()) - if _, err := bc.insertChain(blocks, false, false); err != nil { + if _, err := bc.insertChain(blocks, false, catalystMode); err != nil { return 0, err } blocks, memory = blocks[:0], 0 @@ -1829,11 +1834,55 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i } if len(blocks) > 0 { log.Info("Importing sidechain segment", "start", blocks[0].NumberU64(), "end", blocks[len(blocks)-1].NumberU64()) - return bc.insertChain(blocks, false, false) + return bc.insertChain(blocks, false, catalystMode) } return 0, nil } +// recoverAncestors finds the closest ancestor with available state and re-execute +// all the ancestor blocks since that. +// recoverAncestors is only used in catalyst mode. +func (bc *BlockChain) recoverAncestors(block *types.Block) error { + // Gather all the sidechain hashes (full blocks may be memory heavy) + var ( + hashes []common.Hash + numbers []uint64 + parent = block + ) + for parent != nil && !bc.HasState(parent.Root()) { + hashes = append(hashes, parent.Hash()) + numbers = append(numbers, parent.NumberU64()) + parent = bc.GetBlock(parent.ParentHash(), parent.NumberU64()-1) + + // If the chain is terminating, stop iteration + if bc.insertStopped() { + log.Debug("Abort during blocks iteration") + return errInsertionInterrupted + } + } + if parent == nil { + return errors.New("missing parent") + } + // Import all the pruned blocks to make the state available + for i := len(hashes) - 1; i >= 0; i-- { + // If the chain is terminating, stop processing blocks + if bc.insertStopped() { + log.Debug("Abort during blocks processing") + return errInsertionInterrupted + } + var b *types.Block + if i == 0 { + b = block + } else { + b = bc.GetBlock(hashes[i], numbers[i]) + } + if _, err := bc.insertChain(types.Blocks{b}, false, true); err != nil { + return err + } + } + return nil +} + // collectLogs collects the logs that were generated or removed during // the processing of the block that corresponds with the given hash. // These logs are later announced as deleted or reborn. From 5b1c3cc4f99c2a450a1d0e6012362c4329f954da Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 15 Nov 2021 10:51:40 +0100 Subject: [PATCH 21/32] core: fix nitpick --- core/blockchain.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index b8191bc2abd7..5428140cd906 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1470,7 +1470,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, catalystMode // First block is pruned, insert as sidechain and reorg only if TD grows enough case !catalystMode && errors.Is(err, consensus.ErrPrunedAncestor): log.Debug("Pruned ancestor, inserting as sidechain", "number", block.Number(), "hash", block.Hash()) - return bc.insertSideChain(block, it, catalystMode) + return bc.insertSideChain(block, it) // First block is future, shove it (and all children) to the future queue (unknown ancestor) case !catalystMode && (errors.Is(err, consensus.ErrFutureBlock) || (errors.Is(err, consensus.ErrUnknownAncestor) && bc.futureBlocks.Contains(it.first().ParentHash()))): @@ -1716,7 +1716,8 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, catalystMode // // The method writes all (header-and-body-valid) blocks to disk, then tries to // switch over to the new chain if the TD exceeded the current chain. -func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator, catalystMode bool) (int, error) { +// insertSideChain is only used in non-catalyst mode. +func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (int, error) { var ( externTd *big.Int lastBlock = block @@ -1820,7 +1821,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator, ca // memory here. if len(blocks) >= 2048 || memory > 64*1024*1024 { log.Info("Importing heavy sidechain segment", "blocks", len(blocks), "start", blocks[0].NumberU64(), "end", block.NumberU64()) - if _, err := bc.insertChain(blocks, false, catalystMode); err != nil { + if _, err := bc.insertChain(blocks, false, false); err != nil { return 0, err } blocks, memory = blocks[:0], 0 @@ -1834,7 +1835,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator, ca } if len(blocks) > 0 { log.Info("Importing sidechain segment", "start", blocks[0].NumberU64(), "end", blocks[len(blocks)-1].NumberU64()) - return bc.insertChain(blocks, false, catalystMode) + return bc.insertChain(blocks, false, false) } return 0, nil } From 75bc816869a193792a494dec0ecb12eb0011c6bc Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 16 Nov 2021 13:21:36 +0100 Subject: [PATCH 22/32] all: removed merger from beacon, use TTD, nitpicks --- consensus/beacon/consensus.go | 82 ++++++++++++++--------------------- consensus/consensus.go | 3 ++ core/block_validator_test.go | 35 ++++++++------- core/blockchain_test.go | 8 ++-- core/chain_makers.go | 3 +- eth/backend.go | 2 +- eth/catalyst/api.go | 8 ++-- eth/ethconfig/config.go | 8 +--- les/client.go | 2 +- 9 files changed, 69 insertions(+), 82 deletions(-) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index d56575478dea..a9eba271147c 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -20,7 +20,6 @@ import ( "errors" "fmt" "math/big" - "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" @@ -48,7 +47,7 @@ var ( errInvalidUncleHash = errors.New("invalid uncle hash") ) -// Beacon is a consensus engine combines the eth1 consensus and proof-of-stake +// Beacon is a consensus engine that combines the eth1 consensus and proof-of-stake // algorithm. There is a special flag inside to decide whether to use legacy consensus // rules or new rules. The transition rule is described in the eth1/2 merge spec. // https://github.com/ethereum/EIPs/blob/master/EIPS/eip-3675.md @@ -57,22 +56,15 @@ var ( // is only used for necessary consensus checks. The legacy consensus engine can be any // engine implements the consensus interface (except the beacon itself). type Beacon struct { - ethone consensus.Engine // Classic consensus engine used in eth1, e.g. ethash or clique - - // merger allows us to query whether the transition has been triggered. - merger *consensus.Merger - lock sync.RWMutex + ethone consensus.Engine // Original consensus engine used in eth1, e.g. ethash or clique } // New creates a consensus engine with the given embedded eth1 engine. -func New(ethone consensus.Engine, merger *consensus.Merger) *Beacon { +func New(ethone consensus.Engine) *Beacon { if _, ok := ethone.(*Beacon); ok { panic("nested consensus engine") } - return &Beacon{ - ethone: ethone, - merger: merger, - } + return &Beacon{ethone: ethone} } // Author implements consensus.Engine, returning the verified author of the block. @@ -86,7 +78,8 @@ func (beacon *Beacon) Author(header *types.Header) (common.Address, error) { // VerifyHeader checks whether a header conforms to the consensus rules of the // stock Ethereum consensus engine. func (beacon *Beacon) VerifyHeader(chain consensus.ChainHeaderReader, header *types.Header, seal bool) error { - if !beacon.IsPoSHeader(header) { + reached, _ := IsTTDReached(chain, header.ParentHash, header.Number.Uint64()-1) + if !reached { return beacon.ethone.VerifyHeader(chain, header, seal) } // Short circuit if the header is known, or its parent not @@ -166,7 +159,7 @@ func (beacon *Beacon) VerifyHeaders(chain consensus.ChainHeaderReader, headers [ } // VerifyUncles verifies that the given block's uncles conform to the consensus -// rules of the stock Ethereum consensus engine. +// rules of the Ethereum consensus engine. func (beacon *Beacon) VerifyUncles(chain consensus.ChainReader, block *types.Block) error { if !beacon.IsPoSHeader(block.Header()) { return beacon.ethone.VerifyUncles(chain, block) @@ -180,7 +173,10 @@ func (beacon *Beacon) VerifyUncles(chain consensus.ChainReader, block *types.Blo // verifyHeader checks whether a header conforms to the consensus rules of the // stock Ethereum consensus engine. The difference between the beacon and classic is -// (a) the difficulty, mixhash, nonce and unclehash are expected +// (a) The following fields are expected to be constants: +// - difficulty is expected to be 0 +// - nonce is expected to be 0 +// - unclehash is expected to be Hash(emptyHeader) // to be the desired constants // (b) the timestamp is not verified anymore // (c) the extradata is limited to 32 bytes @@ -216,20 +212,8 @@ func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, pa if diff := new(big.Int).Sub(header.Number, parent.Number); diff.Cmp(big.NewInt(1)) != 0 { return consensus.ErrInvalidNumber } - // Verify that the gas limit remains within allowed bounds - if !chain.Config().IsLondon(header.Number) { - // Verify BaseFee not present before EIP-1559 fork. - if header.BaseFee != nil { - return fmt.Errorf("invalid baseFee before fork: have %d, expected 'nil'", header.BaseFee) - } - if err := misc.VerifyGaslimit(parent.GasLimit, header.GasLimit); err != nil { - return err - } - } else if err := misc.VerifyEip1559Header(chain.Config(), parent, header); err != nil { - // Verify the header's EIP-1559 attributes. - return err - } - return nil + // Verify the header's EIP-1559 attributes. + return misc.VerifyEip1559Header(chain.Config(), parent, header) } // verifyHeaders is similar to verifyHeader, but verifies a batch of headers @@ -275,26 +259,21 @@ func (beacon *Beacon) verifyHeaders(chain consensus.ChainHeaderReader, headers [ // Prepare implements consensus.Engine, initializing the difficulty field of a // header to conform to the beacon protocol. The changes are done inline. func (beacon *Beacon) Prepare(chain consensus.ChainHeaderReader, header *types.Header) error { - beacon.lock.RLock() - defer beacon.lock.RUnlock() // Transition isn't triggered yet, use the legacy rules for preparation. - if !beacon.merger.TDDReached() { - return beacon.ethone.Prepare(chain, header) - } - parent := chain.GetHeader(header.ParentHash, header.Number.Uint64()-1) - if parent == nil { + reached, err := IsTTDReached(chain, header.ParentHash, header.Number.Uint64()-1) + if err != nil { return consensus.ErrUnknownAncestor } + if !reached { + return beacon.ethone.Prepare(chain, header) + } header.Difficulty = beaconDifficulty return nil } // Finalize implements consensus.Engine, setting the final state on the header func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, uncles []*types.Header) { - beacon.lock.RLock() - defer beacon.lock.RUnlock() - // Finalize is different with Prepare, it can be used in both block generation // and verification. So determine the consensus rules by header type. if !beacon.IsPoSHeader(header) { @@ -303,15 +282,12 @@ func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types. } // The block reward is no longer handled here. It's done by the // external consensus engine. - header.Root = state.IntermediateRoot(chain.Config().IsEIP158(header.Number)) + header.Root = state.IntermediateRoot(true) } // FinalizeAndAssemble implements consensus.Engine, setting the final state and // assembling the block. func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, uncles []*types.Header, receipts []*types.Receipt) (*types.Block, error) { - beacon.lock.RLock() - defer beacon.lock.RUnlock() - // FinalizeAndAssemble is different with Prepare, it can be used in both block // generation and verification. So determine the consensus rules by header type. if !beacon.IsPoSHeader(header) { @@ -328,9 +304,6 @@ func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea // Note, the method returns immediately and will send the result async. More // than one result may also be returned depending on the consensus algorithm. func (beacon *Beacon) Seal(chain consensus.ChainHeaderReader, block *types.Block, results chan<- *types.Block, stop <-chan struct{}) error { - beacon.lock.RLock() - defer beacon.lock.RUnlock() - if !beacon.IsPoSHeader(block.Header()) { return beacon.ethone.Seal(chain, block, results, stop) } @@ -350,11 +323,8 @@ func (beacon *Beacon) SealHash(header *types.Header) common.Hash { // the difficulty that a new block should have when created at time // given the parent block's time and difficulty. func (beacon *Beacon) CalcDifficulty(chain consensus.ChainHeaderReader, time uint64, parent *types.Header) *big.Int { - beacon.lock.RLock() - defer beacon.lock.RUnlock() - // Transition isn't triggered yet, use the legacy rules for calculation - if !beacon.merger.TDDReached() { + if reached, _ := IsTTDReached(chain, parent.Hash(), parent.Number.Uint64()); !reached { return beacon.ethone.CalcDifficulty(chain, time, parent) } return beaconDifficulty @@ -395,3 +365,15 @@ func (beacon *Beacon) SetThreads(threads int) { th.SetThreads(threads) } } + +func IsTTDReached(chain consensus.ChainHeaderReader, header common.Hash, number uint64) (bool, error) { + if chain.Config().TerminalTotalDifficulty == nil { + return false, nil + } + td := chain.GetTd(header, number) + if td == nil { + return false, errors.New("TD not found") + } + // We only care about if the new block is > TTD not if the parent was < TTD + return chain.Config().IsTerminalPoWBlock(big.NewInt(0), td), nil +} diff --git a/consensus/consensus.go b/consensus/consensus.go index 2a5aac945d9d..af8ce98ff3be 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -44,6 +44,9 @@ type ChainHeaderReader interface { // GetHeaderByHash retrieves a block header from the database by its hash. GetHeaderByHash(hash common.Hash) *types.Header + + // GetTd retrieves the total difficulty from the database by hash and number. + GetTd(hash common.Hash, number uint64) *big.Int } // ChainReader defines a small collection of methods needed to access the local diff --git a/core/block_validator_test.go b/core/block_validator_test.go index 8fd3ae585d12..4971c0a998e3 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -112,9 +112,9 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { copy(genspec.ExtraData[32:], addr[:]) genesis := genspec.MustCommit(testdb) - genMerger := consensus.NewMerger(rawdb.NewMemoryDatabase()) - genEngine := beacon.New(engine, genMerger) + genEngine := beacon.New(engine) preBlocks, _ = GenerateChain(params.AllCliqueProtocolChanges, genesis, genEngine, testdb, 8, nil) + td := 0 for i, block := range preBlocks { header := block.Header() if i > 0 { @@ -126,25 +126,31 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { sig, _ := crypto.Sign(genEngine.SealHash(header).Bytes(), key) copy(header.Extra[len(header.Extra)-crypto.SignatureLength:], sig) preBlocks[i] = block.WithSeal(header) + // calculate td + td += int(block.Difficulty().Uint64()) } - genMerger.ReachTTD() - genMerger.FinalizePoS() - postBlocks, _ = GenerateChain(params.AllCliqueProtocolChanges, preBlocks[len(preBlocks)-1], genEngine, testdb, 8, nil) - chainConfig = params.AllCliqueProtocolChanges - runEngine = beacon.New(engine, merger) + config := *params.AllCliqueProtocolChanges + config.TerminalTotalDifficulty = big.NewInt(int64(td)) + postBlocks, _ = GenerateChain(&config, preBlocks[len(preBlocks)-1], genEngine, testdb, 8, nil) + chainConfig = &config + runEngine = beacon.New(engine) } else { gspec := &Genesis{Config: params.TestChainConfig} genesis := gspec.MustCommit(testdb) - genMerger := consensus.NewMerger(rawdb.NewMemoryDatabase()) - genEngine := beacon.New(ethash.NewFaker(), genMerger) + genEngine := beacon.New(ethash.NewFaker()) preBlocks, _ = GenerateChain(params.TestChainConfig, genesis, genEngine, testdb, 8, nil) - genMerger.ReachTTD() - genMerger.FinalizePoS() + td := 0 + for _, block := range preBlocks { + // calculate td + td += int(block.Difficulty().Uint64()) + } + config := *params.TestChainConfig + config.TerminalTotalDifficulty = big.NewInt(int64(td)) postBlocks, _ = GenerateChain(params.TestChainConfig, preBlocks[len(preBlocks)-1], genEngine, testdb, 8, nil) - chainConfig = params.TestChainConfig - runEngine = beacon.New(ethash.NewFaker(), merger) + chainConfig = &config + runEngine = beacon.New(ethash.NewFaker()) } preHeaders := make([]*types.Header, len(preBlocks)) @@ -161,7 +167,6 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { blob, _ := json.Marshal(block.Header()) t.Logf("Log header after the merging %d: %v", block.NumberU64(), string(blob)) } - // Run the header checker for blocks one-by-one, checking for both valid and invalid nonces chain, _ := NewBlockChain(testdb, nil, chainConfig, runEngine, vm.Config{}, nil, nil) defer chain.Stop() @@ -209,7 +214,7 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { t.Fatalf("test %d: unexpected result returned: %v", i, result) case <-time.After(25 * time.Millisecond): } - chain.InsertChain(postBlocks[i : i+1]) + chain.InsertCatalystBlock(postBlocks[i]) } // Verify the blocks with pre-merge blocks and post-merge blocks diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 1b81ad6b206b..f1da5dd68d76 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -1943,8 +1943,8 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon // Generate a canonical chain to act as the main dataset var ( merger = consensus.NewMerger(rawdb.NewMemoryDatabase()) - genEngine = beacon.New(ethash.NewFaker(), merger) - runEngine = beacon.New(ethash.NewFaker(), merger) + genEngine = beacon.New(ethash.NewFaker()) + runEngine = beacon.New(ethash.NewFaker()) db = rawdb.NewMemoryDatabase() key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -2224,8 +2224,8 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i db = rawdb.NewMemoryDatabase() genesis = (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee), Config: &chainConfig}).MustCommit(db) runMerger = consensus.NewMerger(db) - runEngine = beacon.New(ethash.NewFaker(), runMerger) - genEngine = beacon.New(ethash.NewFaker(), runMerger) + runEngine = beacon.New(ethash.NewFaker()) + genEngine = beacon.New(ethash.NewFaker()) ) applyMerge := func(engine *beacon.Beacon, height int) { if engine != nil { diff --git a/core/chain_makers.go b/core/chain_makers.go index 403e64b34c31..c119968270c6 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -208,7 +208,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse // Set the difficulty for clique block. The chain maker doesn't have access // to a chain, so the difficulty will be left unset (nil). Set it here to the // correct value. - if b.header.Difficulty == nil { + if config.TerminalTotalDifficulty == nil && b.header.Difficulty == nil { b.header.Difficulty = big.NewInt(2) } // Mutate the state and block according to any hard-fork specs @@ -319,3 +319,4 @@ func (cr *fakeChainReader) GetHeaderByNumber(number uint64) *types.Header func (cr *fakeChainReader) GetHeaderByHash(hash common.Hash) *types.Header { return nil } func (cr *fakeChainReader) GetHeader(hash common.Hash, number uint64) *types.Header { return nil } func (cr *fakeChainReader) GetBlock(hash common.Hash, number uint64) *types.Block { return nil } +func (cr *fakeChainReader) GetTd(hash common.Hash, number uint64) *big.Int { return nil } diff --git a/eth/backend.go b/eth/backend.go index 62be14539a6b..b466a8463a76 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -149,7 +149,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { chainDb: chainDb, eventMux: stack.EventMux(), accountManager: stack.AccountManager(), - engine: ethconfig.CreateConsensusEngine(stack, chainConfig, ðashConfig, config.Miner.Notify, config.Miner.Noverify, chainDb, merger), + engine: ethconfig.CreateConsensusEngine(stack, chainConfig, ðashConfig, config.Miner.Notify, config.Miner.Noverify, chainDb), closeBloomHandler: make(chan struct{}), networkID: config.NetworkId, gasPrice: config.Miner.GasPrice, diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index ec43b3700665..540a6bf1b756 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -92,18 +92,18 @@ func NewConsensusAPI(eth *eth.Ethereum, les *les.LightEthereum) *ConsensusAPI { panic("Catalyst started without valid total difficulty") } if b, ok := les.Engine().(*beacon.Beacon); ok { - engine = beacon.New(b.InnerEngine(), les.Merger()) + engine = beacon.New(b.InnerEngine()) } else { - engine = beacon.New(les.Engine(), les.Merger()) + engine = beacon.New(les.Engine()) } } else { if eth.BlockChain().Config().TerminalTotalDifficulty == nil { panic("Catalyst started without valid total difficulty") } if b, ok := eth.Engine().(*beacon.Beacon); ok { - engine = beacon.New(b.InnerEngine(), eth.Merger()) + engine = beacon.New(b.InnerEngine()) } else { - engine = beacon.New(eth.Engine(), eth.Merger()) + engine = beacon.New(eth.Engine()) } } return &ConsensusAPI{ diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 9eabf1e2c67f..1dbd5a7f1fd8 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -211,7 +211,7 @@ type Config struct { } // CreateConsensusEngine creates a consensus engine for the given chain configuration. -func CreateConsensusEngine(stack *node.Node, chainConfig *params.ChainConfig, config *ethash.Config, notify []string, noverify bool, db ethdb.Database, merger *consensus.Merger) consensus.Engine { +func CreateConsensusEngine(stack *node.Node, chainConfig *params.ChainConfig, config *ethash.Config, notify []string, noverify bool, db ethdb.Database) consensus.Engine { // If proof-of-authority is requested, set it up var engine consensus.Engine if chainConfig.Clique != nil { @@ -239,9 +239,5 @@ func CreateConsensusEngine(stack *node.Node, chainConfig *params.ChainConfig, co }, notify, noverify) engine.(*ethash.Ethash).SetThreads(-1) // Disable CPU mining } - if merger == nil { - return engine - } - engine = beacon.New(engine, merger) - return engine + return beacon.New(engine) } diff --git a/les/client.go b/les/client.go index 05b54fbc4aa5..c20c3343957c 100644 --- a/les/client.go +++ b/les/client.go @@ -112,7 +112,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { reqDist: newRequestDistributor(peers, &mclock.System{}), accountManager: stack.AccountManager(), merger: merger, - engine: ethconfig.CreateConsensusEngine(stack, chainConfig, &config.Ethash, nil, false, chainDb, merger), + engine: ethconfig.CreateConsensusEngine(stack, chainConfig, &config.Ethash, nil, false, chainDb), bloomRequests: make(chan chan *bloombits.Retrieval), bloomIndexer: core.NewBloomIndexer(chainDb, params.BloomBitsBlocksClient, params.HelperTrieConfirmations), p2pServer: stack.Server(), From fe45669447e33a2a196eff737204d343c2f39ab3 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 16 Nov 2021 13:58:58 +0100 Subject: [PATCH 23/32] consensus: eth: add docstring, removed unnecessary code duplication --- consensus/beacon/consensus.go | 10 ++++++---- eth/backend.go | 23 ++++++++++------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index a9eba271147c..a65c1d10ad7b 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -209,7 +209,7 @@ func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, pa return fmt.Errorf("invalid gasUsed: have %d, gasLimit %d", header.GasUsed, header.GasLimit) } // Verify that the block number is parent's +1 - if diff := new(big.Int).Sub(header.Number, parent.Number); diff.Cmp(big.NewInt(1)) != 0 { + if diff := new(big.Int).Sub(header.Number, parent.Number); diff.Cmp(common.Big1) != 0 { return consensus.ErrInvalidNumber } // Verify the header's EIP-1559 attributes. @@ -366,14 +366,16 @@ func (beacon *Beacon) SetThreads(threads int) { } } -func IsTTDReached(chain consensus.ChainHeaderReader, header common.Hash, number uint64) (bool, error) { +// IsTTDReached checks if the TotalTerminalDifficulty has been reached. +// It depends on the parentHash already being stored in the database. +func IsTTDReached(chain consensus.ChainHeaderReader, parentHash common.Hash, number uint64) (bool, error) { if chain.Config().TerminalTotalDifficulty == nil { return false, nil } - td := chain.GetTd(header, number) + td := chain.GetTd(parentHash, number) if td == nil { return false, errors.New("TD not found") } // We only care about if the new block is > TTD not if the parent was < TTD - return chain.Config().IsTerminalPoWBlock(big.NewInt(0), td), nil + return chain.Config().IsTerminalPoWBlock(common.Big0, td), nil } diff --git a/eth/backend.go b/eth/backend.go index b466a8463a76..8e2bdf3642f7 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -471,24 +471,21 @@ func (s *Ethereum) StartMining(threads int) error { log.Error("Cannot start mining without etherbase", "err", err) return fmt.Errorf("etherbase missing: %v", err) } - if clique, ok := s.engine.(*clique.Clique); ok { + var cli *clique.Clique + if c, ok := s.engine.(*clique.Clique); ok { + cli = c + } else if cl, ok := s.engine.(*beacon.Beacon); ok { + if c, ok := cl.InnerEngine().(*clique.Clique); ok { + cli = c + } + } + if cli != nil { wallet, err := s.accountManager.Find(accounts.Account{Address: eb}) if wallet == nil || err != nil { log.Error("Etherbase account unavailable locally", "err", err) return fmt.Errorf("signer missing: %v", err) } - clique.Authorize(eb, wallet.SignData) - } - // Authorize nested engines too - if cl, ok := s.engine.(*beacon.Beacon); ok { - if clique, ok := cl.InnerEngine().(*clique.Clique); ok { - wallet, err := s.accountManager.Find(accounts.Account{Address: eb}) - if wallet == nil || err != nil { - log.Error("Etherbase account unavailable locally", "err", err) - return fmt.Errorf("signer missing: %v", err) - } - clique.Authorize(eb, wallet.SignData) - } + cli.Authorize(eb, wallet.SignData) } // If mining is started, we can disable the transaction rejection mechanism // introduced to speed sync times. From 9b78e1ea5cb43732c4de5766b584c8f89ae33ce8 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 16 Nov 2021 15:18:37 +0100 Subject: [PATCH 24/32] consensus/beacon: better comment --- consensus/beacon/consensus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index a65c1d10ad7b..d9a69c20a40a 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -366,7 +366,7 @@ func (beacon *Beacon) SetThreads(threads int) { } } -// IsTTDReached checks if the TotalTerminalDifficulty has been reached. +// IsTTDReached checks if the TotalTerminalDifficulty has been surpassed on the `parentHash` block. // It depends on the parentHash already being stored in the database. func IsTTDReached(chain consensus.ChainHeaderReader, parentHash common.Hash, number uint64) (bool, error) { if chain.Config().TerminalTotalDifficulty == nil { From 153d85f000a43c4eb8f87ab6a2471acd24bf652a Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Wed, 17 Nov 2021 17:04:56 +0100 Subject: [PATCH 25/32] all: easy to fix nitpicks by karalabe --- consensus/beacon/consensus.go | 8 ++++---- consensus/merger.go | 25 +++++++++++++------------ core/block_validator_test.go | 2 +- core/blockchain.go | 33 +++++++++++++++++---------------- core/chain_makers.go | 10 ++++++++-- eth/catalyst/api.go | 2 +- eth/handler.go | 2 +- 7 files changed, 45 insertions(+), 37 deletions(-) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index d9a69c20a40a..413d2009bc8e 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -259,11 +259,10 @@ func (beacon *Beacon) verifyHeaders(chain consensus.ChainHeaderReader, headers [ // Prepare implements consensus.Engine, initializing the difficulty field of a // header to conform to the beacon protocol. The changes are done inline. func (beacon *Beacon) Prepare(chain consensus.ChainHeaderReader, header *types.Header) error { - // Transition isn't triggered yet, use the legacy rules for preparation. reached, err := IsTTDReached(chain, header.ParentHash, header.Number.Uint64()-1) if err != nil { - return consensus.ErrUnknownAncestor + return err } if !reached { return beacon.ethone.Prepare(chain, header) @@ -345,7 +344,7 @@ func (beacon *Beacon) Close() error { // because the header difficulty is not set yet. func (beacon *Beacon) IsPoSHeader(header *types.Header) bool { if header.Difficulty == nil { - return false // we should never enter here. + panic("IsPoSHeader called with invalid difficulty") } return header.Difficulty.Cmp(beaconDifficulty) == 0 } @@ -368,13 +367,14 @@ func (beacon *Beacon) SetThreads(threads int) { // IsTTDReached checks if the TotalTerminalDifficulty has been surpassed on the `parentHash` block. // It depends on the parentHash already being stored in the database. +// If the parentHash is not stored in the database a UnknownAncestor error is returned. func IsTTDReached(chain consensus.ChainHeaderReader, parentHash common.Hash, number uint64) (bool, error) { if chain.Config().TerminalTotalDifficulty == nil { return false, nil } td := chain.GetTd(parentHash, number) if td == nil { - return false, errors.New("TD not found") + return false, consensus.ErrUnknownAncestor } // We only care about if the new block is > TTD not if the parent was < TTD return chain.Config().IsTerminalPoWBlock(common.Big0, td), nil diff --git a/consensus/merger.go b/consensus/merger.go index 66cd47f4d311..ffbcbf2b8569 100644 --- a/consensus/merger.go +++ b/consensus/merger.go @@ -17,6 +17,7 @@ package consensus import ( + "fmt" "sync" "github.com/ethereum/go-ethereum/core/rawdb" @@ -38,9 +39,10 @@ type transitionStatus struct { type Merger struct { db ethdb.KeyValueStore status transitionStatus - lock sync.Mutex + mu sync.RWMutex } +// NewMerger creates a new Merger which stores its transition status in the provided db. func NewMerger(db ethdb.KeyValueStore) *Merger { var status transitionStatus blob := rawdb.ReadTransitionStatus(db) @@ -52,15 +54,14 @@ func NewMerger(db ethdb.KeyValueStore) *Merger { return &Merger{ db: db, status: status, - lock: sync.Mutex{}, } } // ReachTTD is called whenever the first NewHead message received // from the consensus-layer. func (m *Merger) ReachTTD() { - m.lock.Lock() - defer m.lock.Unlock() + m.mu.Lock() + defer m.mu.Unlock() if m.status.LeftPoW { return @@ -68,7 +69,7 @@ func (m *Merger) ReachTTD() { m.status = transitionStatus{LeftPoW: true} blob, err := rlp.EncodeToBytes(m.status) if err != nil { - log.Crit("Failed to encode the transition status", "err", err) + panic(fmt.Sprintf("Failed to encode the transition status: %v", err)) } rawdb.WriteTransitionStatus(m.db, blob) log.Info("Left PoW stage") @@ -77,8 +78,8 @@ func (m *Merger) ReachTTD() { // FinalizePoS is called whenever the first FinalisedBlock message received // from the consensus-layer. func (m *Merger) FinalizePoS() { - m.lock.Lock() - defer m.lock.Unlock() + m.mu.Lock() + defer m.mu.Unlock() if m.status.EnteredPoS { return @@ -86,7 +87,7 @@ func (m *Merger) FinalizePoS() { m.status = transitionStatus{LeftPoW: true, EnteredPoS: true} blob, err := rlp.EncodeToBytes(m.status) if err != nil { - log.Crit("Failed to encode the transition status", "err", err) + panic(fmt.Sprintf("Failed to encode the transition status: %v", err)) } rawdb.WriteTransitionStatus(m.db, blob) log.Info("Entered PoS stage") @@ -94,16 +95,16 @@ func (m *Merger) FinalizePoS() { // TDDReached reports whether the chain has left the PoW stage. func (m *Merger) TDDReached() bool { - m.lock.Lock() - defer m.lock.Unlock() + m.mu.RLock() + defer m.mu.RUnlock() return m.status.LeftPoW } // PoSFinalized reports whether the chain has entered the PoS stage. func (m *Merger) PoSFinalized() bool { - m.lock.Lock() - defer m.lock.Unlock() + m.mu.RLock() + defer m.mu.RUnlock() return m.status.EnteredPoS } diff --git a/core/block_validator_test.go b/core/block_validator_test.go index 4971c0a998e3..0f183ba52778 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -214,7 +214,7 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { t.Fatalf("test %d: unexpected result returned: %v", i, result) case <-time.After(25 * time.Millisecond): } - chain.InsertCatalystBlock(postBlocks[i]) + chain.InsertBlockWithoutSetHead(postBlocks[i]) } // Verify the blocks with pre-merge blocks and post-merge blocks diff --git a/core/blockchain.go b/core/blockchain.go index 5428140cd906..9bffd0ece53c 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1366,7 +1366,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { return 0, errChainStopped } defer bc.chainmu.Unlock() - return bc.insertChain(chain, true, false) + return bc.insertChain(chain, true, true) } // insertChain is the internal implementation of InsertChain, which assumes that @@ -1377,7 +1377,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { // racey behaviour. If a sidechain import is in progress, and the historic state // is imported, but then new canon-head is added before the actual sidechain // completes, then the historic state could be pruned again -func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, catalystMode bool) (int, error) { +func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, setHead bool) (int, error) { // If the chain is terminating, don't even bother starting up. if bc.insertStopped() { return 0, nil @@ -1462,18 +1462,18 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, catalystMode // Falls through to the block import } switch { - // We're in catalyst mode and the parent is pruned, try to recover the parent state - case catalystMode && errors.Is(err, consensus.ErrPrunedAncestor): + // We're post-merge and the parent is pruned, try to recover the parent state + case !setHead && errors.Is(err, consensus.ErrPrunedAncestor): log.Debug("Pruned ancestor", "number", block.Number(), "hash", block.Hash()) return it.index, bc.recoverAncestors(block) - // First block is pruned, insert as sidechain and reorg only if TD grows enough - case !catalystMode && errors.Is(err, consensus.ErrPrunedAncestor): + // First block is pruned, insert as sidechain and reorg only if TD grows enough + case setHead && errors.Is(err, consensus.ErrPrunedAncestor): log.Debug("Pruned ancestor, inserting as sidechain", "number", block.Number(), "hash", block.Hash()) return bc.insertSideChain(block, it) // First block is future, shove it (and all children) to the future queue (unknown ancestor) - case !catalystMode && (errors.Is(err, consensus.ErrFutureBlock) || (errors.Is(err, consensus.ErrUnknownAncestor) && bc.futureBlocks.Contains(it.first().ParentHash()))): + case setHead && (errors.Is(err, consensus.ErrFutureBlock) || (errors.Is(err, consensus.ErrUnknownAncestor) && bc.futureBlocks.Contains(it.first().ParentHash()))): for block != nil && (it.index == 0 || errors.Is(err, consensus.ErrUnknownAncestor)) { log.Debug("Future block, postponing import", "number", block.Number(), "hash", block.Hash()) if err := bc.addFutureBlock(block); err != nil { @@ -1631,8 +1631,8 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, catalystMode // Write the block to the chain and get the status. substart = time.Now() var status WriteStatus - if catalystMode { - // In catalyst mode we don't set the head, only insert the block + if !setHead { + // Don't set the head, only insert the block err := bc.writeBlockWithState(block, receipts, logs, statedb) atomic.StoreUint32(&followupInterrupt, 1) if err != nil { @@ -1653,7 +1653,8 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, catalystMode blockWriteTimer.Update(time.Since(substart) - statedb.AccountCommits - statedb.StorageCommits - statedb.SnapshotCommits) blockInsertTimer.UpdateSince(start) - if catalystMode { + if !setHead { + // We did not setHead, so we don't have any stats to update log.Info("Inserted block", "number", block.Number(), "hash", block.Hash(), "txs", len(block.Transactions()), "elapsed", common.PrettyDuration(time.Since(start))) return it.index, nil } @@ -1821,7 +1822,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i // memory here. if len(blocks) >= 2048 || memory > 64*1024*1024 { log.Info("Importing heavy sidechain segment", "blocks", len(blocks), "start", blocks[0].NumberU64(), "end", block.NumberU64()) - if _, err := bc.insertChain(blocks, false, false); err != nil { + if _, err := bc.insertChain(blocks, false, true); err != nil { return 0, err } blocks, memory = blocks[:0], 0 @@ -1835,7 +1836,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i } if len(blocks) > 0 { log.Info("Importing sidechain segment", "start", blocks[0].NumberU64(), "end", blocks[len(blocks)-1].NumberU64()) - return bc.insertChain(blocks, false, false) + return bc.insertChain(blocks, false, true) } return 0, nil } @@ -1877,7 +1878,7 @@ func (bc *BlockChain) recoverAncestors(block *types.Block) error { } else { b = bc.GetBlock(hashes[i], numbers[i]) } - if _, err := bc.insertChain(types.Blocks{b}, false, true); err != nil { + if _, err := bc.insertChain(types.Blocks{b}, false, false); err != nil { return err } } @@ -2066,18 +2067,18 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { return nil } -// InsertCatalystBlock executes the block, runs the necessary verification +// InsertBlockWithoutSetHead executes the block, runs the necessary verification // upon it and then persist the block and the associate state into the database. // The key difference between the InsertChain is it won't do the canonical chain // updating. It relies on the additional SetChainHead call to finalize the entire // procedure. -func (bc *BlockChain) InsertCatalystBlock(block *types.Block) error { +func (bc *BlockChain) InsertBlockWithoutSetHead(block *types.Block) error { if !bc.chainmu.TryLock() { return errChainStopped } defer bc.chainmu.Unlock() - _, err := bc.insertChain(types.Blocks{block}, true, true) + _, err := bc.insertChain(types.Blocks{block}, true, false) return err } diff --git a/core/chain_makers.go b/core/chain_makers.go index c119968270c6..e1d9e6f78310 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -208,8 +208,14 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse // Set the difficulty for clique block. The chain maker doesn't have access // to a chain, so the difficulty will be left unset (nil). Set it here to the // correct value. - if config.TerminalTotalDifficulty == nil && b.header.Difficulty == nil { - b.header.Difficulty = big.NewInt(2) + if b.header.Difficulty == nil { + if config.TerminalTotalDifficulty == nil { + // Clique chain + b.header.Difficulty = big.NewInt(2) + } else { + // Post-merge chain + b.header.Difficulty = big.NewInt(0) + } } // Mutate the state and block according to any hard-fork specs if daoBlock := config.DAOForkBlock; daoBlock != nil { diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 540a6bf1b756..059df3670286 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -245,7 +245,7 @@ func (api *ConsensusAPI) ExecutePayload(params ExecutableData) (GenericStringRes if td.Cmp(ttd) < 0 { return INVALID, fmt.Errorf("can not execute payload on top of block with low td got: %v threshold %v", td, ttd) } - if err := api.eth.BlockChain().InsertCatalystBlock(block); err != nil { + if err := api.eth.BlockChain().InsertBlockWithoutSetHead(block); err != nil { return INVALID, err } merger := api.merger() diff --git a/eth/handler.go b/eth/handler.go index 7500a7eae12a..20ad8692ce78 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -262,7 +262,7 @@ func newHandler(config *handlerConfig) (*handler, error) { log.Info("Filtered out non-termimal pow block", "number", block.NumberU64(), "hash", block.Hash()) return 0, nil } - if err := h.chain.InsertCatalystBlock(block); err != nil { + if err := h.chain.InsertBlockWithoutSetHead(block); err != nil { return i, err } } From 87686a1965aa0b7145eb8cf64fa0a4a5d96d4116 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Wed, 17 Nov 2021 17:07:32 +0100 Subject: [PATCH 26/32] consensus/beacon: verify known headers to be sure --- consensus/beacon/consensus.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index 413d2009bc8e..4d1eba52204f 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -82,12 +82,8 @@ func (beacon *Beacon) VerifyHeader(chain consensus.ChainHeaderReader, header *ty if !reached { return beacon.ethone.VerifyHeader(chain, header, seal) } - // Short circuit if the header is known, or its parent not - number := header.Number.Uint64() - if chain.GetHeader(header.Hash(), number) != nil { - return nil - } - parent := chain.GetHeader(header.ParentHash, number-1) + // Short circuit if the parent is not known + parent := chain.GetHeader(header.ParentHash, header.Number.Uint64()-1) if parent == nil { return consensus.ErrUnknownAncestor } From 082d6a892d334321fc47a61a3e72844c1ea77db5 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Wed, 17 Nov 2021 18:38:13 +0100 Subject: [PATCH 27/32] core: comments --- core/blockchain.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 9bffd0ece53c..f165006e93fc 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1717,7 +1717,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, setHead bool) // // The method writes all (header-and-body-valid) blocks to disk, then tries to // switch over to the new chain if the TD exceeded the current chain. -// insertSideChain is only used in non-catalyst mode. +// insertSideChain is only used pre-merge. func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (int, error) { var ( externTd *big.Int @@ -1843,7 +1843,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i // recoverAncestors finds the closest ancestor with available state and re-execute // all the ancestor blocks since that. -// recoverAncestors is only used in catalyst mode. +// recoverAncestors is only used post-merge. func (bc *BlockChain) recoverAncestors(block *types.Block) error { // Gather all the sidechain hashes (full blocks may be memory heavy) var ( From 3440375c5e3a80b8e419a97e1e55c937bb877685 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Fri, 19 Nov 2021 10:59:44 +0100 Subject: [PATCH 28/32] core: eth: don't drop peers who advertise blocks, nitpicks --- core/blockchain.go | 35 ++++++++++++++++------------------- eth/handler_eth.go | 8 ++++++-- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index f165006e93fc..89451cc8813e 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1462,16 +1462,17 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, setHead bool) // Falls through to the block import } switch { - // We're post-merge and the parent is pruned, try to recover the parent state - case !setHead && errors.Is(err, consensus.ErrPrunedAncestor): - log.Debug("Pruned ancestor", "number", block.Number(), "hash", block.Hash()) - return it.index, bc.recoverAncestors(block) - - // First block is pruned, insert as sidechain and reorg only if TD grows enough - case setHead && errors.Is(err, consensus.ErrPrunedAncestor): - log.Debug("Pruned ancestor, inserting as sidechain", "number", block.Number(), "hash", block.Hash()) - return bc.insertSideChain(block, it) - + // First block is pruned + case errors.Is(err, consensus.ErrPrunedAncestor): + if setHead { + // First block is pruned, insert as sidechain and reorg only if TD grows enough + log.Debug("Pruned ancestor, inserting as sidechain", "number", block.Number(), "hash", block.Hash()) + return bc.insertSideChain(block, it) + } else { + // We're post-merge and the parent is pruned, try to recover the parent state + log.Debug("Pruned ancestor", "number", block.Number(), "hash", block.Hash()) + return it.index, bc.recoverAncestors(block) + } // First block is future, shove it (and all children) to the future queue (unknown ancestor) case setHead && (errors.Is(err, consensus.ErrFutureBlock) || (errors.Is(err, consensus.ErrUnknownAncestor) && bc.futureBlocks.Contains(it.first().ParentHash()))): for block != nil && (it.index == 0 || errors.Is(err, consensus.ErrUnknownAncestor)) { @@ -1633,17 +1634,13 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, setHead bool) var status WriteStatus if !setHead { // Don't set the head, only insert the block - err := bc.writeBlockWithState(block, receipts, logs, statedb) - atomic.StoreUint32(&followupInterrupt, 1) - if err != nil { - return it.index, err - } + err = bc.writeBlockWithState(block, receipts, logs, statedb) } else { status, err = bc.writeBlockAndSetHead(block, receipts, logs, statedb, false) - atomic.StoreUint32(&followupInterrupt, 1) - if err != nil { - return it.index, err - } + } + atomic.StoreUint32(&followupInterrupt, 1) + if err != nil { + return it.index, err } // Update the metrics touched during block commit accountCommitTimer.Update(statedb.AccountCommits) // Account commits are complete, we can mark them diff --git a/eth/handler_eth.go b/eth/handler_eth.go index 294e27296dee..3c22dce0ae9d 100644 --- a/eth/handler_eth.go +++ b/eth/handler_eth.go @@ -184,7 +184,9 @@ func (h *ethHandler) handleBlockAnnounces(peer *eth.Peer, hashes []common.Hash, // the chain already entered the pos stage and disconnect the // remote peer. if h.merger.PoSFinalized() { - return errors.New("unexpected block announces") + // TODO (MariusVanDerWijden) drop non-updated peers after the merge + return nil + // return errors.New("unexpected block announces") } // Schedule all the unknown hashes for retrieval var ( @@ -210,7 +212,9 @@ func (h *ethHandler) handleBlockBroadcast(peer *eth.Peer, block *types.Block, td // the chain already entered the pos stage and disconnect the // remote peer. if h.merger.PoSFinalized() { - return errors.New("unexpected block announces") + // TODO (MariusVanDerWijden) drop non-updated peers after the merge + return nil + // return errors.New("unexpected block announces") } // Schedule the block for import h.blockFetcher.Enqueue(peer.ID(), block) From d4212bedf3eb517a472c580e2e05f6ec27e9854d Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Fri, 19 Nov 2021 13:14:21 +0100 Subject: [PATCH 29/32] core: never add beacon blocks to the future queue --- core/blockchain.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/blockchain.go b/core/blockchain.go index 89451cc8813e..bf10b13954ea 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1330,6 +1330,10 @@ func (bc *BlockChain) addFutureBlock(block *types.Block) error { if block.Time() > max { return fmt.Errorf("future block timestamp %v > allowed %v", block.Time(), max) } + if block.Difficulty().Cmp(common.Big0) == 0 { + // Never add PoS blocks into the future queue + return nil + } bc.futureBlocks.Add(block.Hash(), block) return nil } @@ -1474,7 +1478,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, setHead bool) return it.index, bc.recoverAncestors(block) } // First block is future, shove it (and all children) to the future queue (unknown ancestor) - case setHead && (errors.Is(err, consensus.ErrFutureBlock) || (errors.Is(err, consensus.ErrUnknownAncestor) && bc.futureBlocks.Contains(it.first().ParentHash()))): + case errors.Is(err, consensus.ErrFutureBlock) || (errors.Is(err, consensus.ErrUnknownAncestor) && bc.futureBlocks.Contains(it.first().ParentHash())): for block != nil && (it.index == 0 || errors.Is(err, consensus.ErrUnknownAncestor)) { log.Debug("Future block, postponing import", "number", block.Number(), "hash", block.Hash()) if err := bc.addFutureBlock(block); err != nil { From 237fadce720e817b2770541f7c6677f563caebd9 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 22 Nov 2021 10:37:37 +0100 Subject: [PATCH 30/32] core: fixed nitpicks --- core/blockchain.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index bf10b13954ea..a50e93b5f4ea 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -380,7 +380,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par // Start future block processor. bc.wg.Add(1) - go bc.update() + go bc.updateFutureBlocks() // Start tx indexer/unindexer. if txLookupLimit != nil { @@ -750,7 +750,6 @@ func (bc *BlockChain) writeHeadBlock(block *types.Block) { rawdb.WriteHeadHeaderHash(batch, block.Hash()) rawdb.WriteHeadFastBlockHash(batch, block.Hash()) } - // Flush the whole batch into the disk, exit the node if failed if err := batch.Write(); err != nil { log.Crit("Failed to update chain indexes and markers", "err", err) @@ -2112,7 +2111,7 @@ func (bc *BlockChain) SetChainHead(newBlock *types.Block) error { return nil } -func (bc *BlockChain) update() { +func (bc *BlockChain) updateFutureBlocks() { futureTimer := time.NewTicker(5 * time.Second) defer futureTimer.Stop() defer bc.wg.Done() From 1a92d4a1ed6913433b48291041472a599737386e Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 23 Nov 2021 12:43:52 +0100 Subject: [PATCH 31/32] consensus/beacon: simplify IsTTDReached check --- consensus/beacon/consensus.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index 4d1eba52204f..e0a985686367 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -372,6 +372,5 @@ func IsTTDReached(chain consensus.ChainHeaderReader, parentHash common.Hash, num if td == nil { return false, consensus.ErrUnknownAncestor } - // We only care about if the new block is > TTD not if the parent was < TTD - return chain.Config().IsTerminalPoWBlock(common.Big0, td), nil + return td.Cmp(chain.Config().TerminalTotalDifficulty) > 0, nil } From dd22e9a9a0ed548fddca60dc3bdababf95c6097e Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 23 Nov 2021 12:58:47 +0100 Subject: [PATCH 32/32] consensus/beacon: correct IsTTDReached check --- consensus/beacon/consensus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index e0a985686367..d90406438558 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -372,5 +372,5 @@ func IsTTDReached(chain consensus.ChainHeaderReader, parentHash common.Hash, num if td == nil { return false, consensus.ErrUnknownAncestor } - return td.Cmp(chain.Config().TerminalTotalDifficulty) > 0, nil + return td.Cmp(chain.Config().TerminalTotalDifficulty) >= 0, nil }