From 088cc588f6042361fbb5ddda7ca0ceb66b90603c Mon Sep 17 00:00:00 2001 From: ssd04 Date: Mon, 21 Nov 2022 22:42:46 +0200 Subject: [PATCH 01/33] add custom error for trie get node --- process/sync/baseSync.go | 9 ++------- process/sync/metablock.go | 12 +++++++++--- process/sync/metablock_test.go | 3 ++- process/sync/shardblock.go | 8 +++++++- process/sync/shardblock_test.go | 3 ++- trie/errors.go | 28 ++++++++++++++++++++++++++++ trie/node.go | 4 +--- 7 files changed, 51 insertions(+), 16 deletions(-) diff --git a/process/sync/baseSync.go b/process/sync/baseSync.go index 2bb613d2d8c..1b07c400124 100644 --- a/process/sync/baseSync.go +++ b/process/sync/baseSync.go @@ -693,14 +693,9 @@ func (boot *baseBootstrap) handleTrieSyncError(err error, ctx context.Context) { } } -func (boot *baseBootstrap) syncUserAccountsState() error { - rootHash, err := boot.accounts.RootHash() - if err != nil { - return err - } - +func (boot *baseBootstrap) syncUserAccountsState(key []byte) error { log.Warn("base sync: started syncUserAccountsState") - return boot.accountsDBSyncer.SyncAccounts(rootHash) + return boot.accountsDBSyncer.SyncAccounts(key) } func (boot *baseBootstrap) cleanNoncesSyncedWithErrorsBehindFinal() { diff --git a/process/sync/metablock.go b/process/sync/metablock.go index 95d06f22563..c881dbc5c19 100644 --- a/process/sync/metablock.go +++ b/process/sync/metablock.go @@ -12,6 +12,7 @@ import ( "github.com/ElrondNetwork/elrond-go/process" "github.com/ElrondNetwork/elrond-go/state" "github.com/ElrondNetwork/elrond-go/storage" + "github.com/ElrondNetwork/elrond-go/trie" ) // MetaBootstrap implements the bootstrap mechanism @@ -180,14 +181,19 @@ func (boot *MetaBootstrap) setLastEpochStartRound() { func (boot *MetaBootstrap) SyncBlock(ctx context.Context) error { err := boot.syncBlock() if errors.IsGetNodeFromDBError(err) { - errSync := boot.syncAccountsDBs() + getNodeErr, ok := err.(*trie.GetErr) + if !ok { + return err + } + + errSync := boot.syncAccountsDBs(getNodeErr.GetKey()) boot.handleTrieSyncError(errSync, ctx) } return err } -func (boot *MetaBootstrap) syncAccountsDBs() error { +func (boot *MetaBootstrap) syncAccountsDBs(key []byte) error { var err error err = boot.syncValidatorAccountsState() @@ -195,7 +201,7 @@ func (boot *MetaBootstrap) syncAccountsDBs() error { return err } - err = boot.syncUserAccountsState() + err = boot.syncUserAccountsState(key) if err != nil { return err } diff --git a/process/sync/metablock_test.go b/process/sync/metablock_test.go index 43c5b71a0a7..f3b2c1bc361 100644 --- a/process/sync/metablock_test.go +++ b/process/sync/metablock_test.go @@ -30,6 +30,7 @@ import ( stateMock "github.com/ElrondNetwork/elrond-go/testscommon/state" statusHandlerMock "github.com/ElrondNetwork/elrond-go/testscommon/statusHandler" storageStubs "github.com/ElrondNetwork/elrond-go/testscommon/storage" + "github.com/ElrondNetwork/elrond-go/trie" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -1621,7 +1622,7 @@ func TestMetaBootstrap_SyncBlockErrGetNodeDBShouldSyncAccounts(t *testing.T) { } args.ChainHandler = blkc - errGetNodeFromDB := errors.New(common.GetNodeFromDBErrorString) + errGetNodeFromDB := trie.NewGetErr([]byte("key")) blockProcessor := createMetaBlockProcessor(args.ChainHandler) blockProcessor.ProcessBlockCalled = func(header data.HeaderHandler, body data.BodyHandler, haveTime func() time.Duration) error { return errGetNodeFromDB diff --git a/process/sync/shardblock.go b/process/sync/shardblock.go index 785a33044ec..a975769d96f 100644 --- a/process/sync/shardblock.go +++ b/process/sync/shardblock.go @@ -12,6 +12,7 @@ import ( "github.com/ElrondNetwork/elrond-go/errors" "github.com/ElrondNetwork/elrond-go/process" "github.com/ElrondNetwork/elrond-go/storage" + "github.com/ElrondNetwork/elrond-go/trie" ) // ShardBootstrap implements the bootstrap mechanism @@ -144,7 +145,12 @@ func (boot *ShardBootstrap) StartSyncingBlocks() { func (boot *ShardBootstrap) SyncBlock(ctx context.Context) error { err := boot.syncBlock() if errors.IsGetNodeFromDBError(err) { - errSync := boot.syncUserAccountsState() + getNodeErr, ok := err.(*trie.GetErr) + if !ok { + return err + } + + errSync := boot.syncUserAccountsState(getNodeErr.GetKey()) boot.handleTrieSyncError(errSync, ctx) } diff --git a/process/sync/shardblock_test.go b/process/sync/shardblock_test.go index 58b1fbd13de..782577af3ab 100644 --- a/process/sync/shardblock_test.go +++ b/process/sync/shardblock_test.go @@ -34,6 +34,7 @@ import ( stateMock "github.com/ElrondNetwork/elrond-go/testscommon/state" statusHandlerMock "github.com/ElrondNetwork/elrond-go/testscommon/statusHandler" storageStubs "github.com/ElrondNetwork/elrond-go/testscommon/storage" + "github.com/ElrondNetwork/elrond-go/trie" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -2061,7 +2062,7 @@ func TestShardBootstrap_SyncBlockGetNodeDBErrorShouldSync(t *testing.T) { } args.ChainHandler = blkc - errGetNodeFromDB := errors.New(common.GetNodeFromDBErrorString) + errGetNodeFromDB := trie.NewGetErr([]byte("key")) blockProcessor := createBlockProcessor(args.ChainHandler) blockProcessor.ProcessBlockCalled = func(header data.HeaderHandler, body data.BodyHandler, haveTime func() time.Duration) error { return errGetNodeFromDB diff --git a/trie/errors.go b/trie/errors.go index a225a84c00c..03142a6821c 100644 --- a/trie/errors.go +++ b/trie/errors.go @@ -1,9 +1,37 @@ package trie import ( + "encoding/hex" "errors" + "fmt" + + "github.com/ElrondNetwork/elrond-go/common" ) +// GetErr defines a custom error for trie get node +type GetErr struct { + key []byte +} + +// NewGetErr will create a new instance of GetErr +func NewGetErr(key []byte) *GetErr { + return &GetErr{key: key} +} + +// Error returns the error as string +func (e *GetErr) Error() string { + return fmt.Sprintf( + "%s for key %v", + common.GetNodeFromDBErrorString, + hex.EncodeToString(e.key), + ) +} + +// GetKey will return the key that generated the error +func (e *GetErr) GetKey() []byte { + return e.key +} + // ErrInvalidNode is raised when we reach an invalid node var ErrInvalidNode = errors.New("invalid node") diff --git a/trie/node.go b/trie/node.go index ac03e15d422..5fe87fccaf7 100644 --- a/trie/node.go +++ b/trie/node.go @@ -3,8 +3,6 @@ package trie import ( "context" - "encoding/hex" - "fmt" "time" "github.com/ElrondNetwork/elrond-go-core/hashing" @@ -120,7 +118,7 @@ func getNodeFromDBAndDecode(n []byte, db common.DBWriteCacher, marshalizer marsh encChild, err := db.Get(n) if err != nil { log.Trace(common.GetNodeFromDBErrorString, "error", err, "key", n) - return nil, fmt.Errorf(common.GetNodeFromDBErrorString+" %w for key %v", err, hex.EncodeToString(n)) + return nil, NewGetErr(n) } return decodeNode(encChild, marshalizer, hasher) From e0dbd57c87a0daa49f4e52d56998497063b37eeb Mon Sep 17 00:00:00 2001 From: ssd04 Date: Mon, 21 Nov 2022 22:58:24 +0200 Subject: [PATCH 02/33] move get node from db custom error to common errors --- errors/missingTrieNodeError.go | 26 ++++++++++++++++++++++++++ process/sync/metablock.go | 3 +-- process/sync/metablock_test.go | 4 ++-- process/sync/shardblock.go | 3 +-- process/sync/shardblock_test.go | 4 ++-- trie/errors.go | 28 ---------------------------- trie/node.go | 2 +- 7 files changed, 33 insertions(+), 37 deletions(-) diff --git a/errors/missingTrieNodeError.go b/errors/missingTrieNodeError.go index 029cc45069f..3cf5961029f 100644 --- a/errors/missingTrieNodeError.go +++ b/errors/missingTrieNodeError.go @@ -1,6 +1,8 @@ package errors import ( + "encoding/hex" + "fmt" "strings" "github.com/ElrondNetwork/elrond-go/common" @@ -22,3 +24,27 @@ func IsGetNodeFromDBError(err error) bool { return false } + +// GetNodeFromDBErr defines a custom error for trie get node +type GetNodeFromDBErr struct { + key []byte +} + +// NewGetNodeFromDBErr will create a new instance of GetNodeFromDBErr +func NewGetNodeFromDBErr(key []byte) *GetNodeFromDBErr { + return &GetNodeFromDBErr{key: key} +} + +// Error returns the error as string +func (e *GetNodeFromDBErr) Error() string { + return fmt.Sprintf( + "%s for key %v", + common.GetNodeFromDBErrorString, + hex.EncodeToString(e.key), + ) +} + +// GetKey will return the key that generated the error +func (e *GetNodeFromDBErr) GetKey() []byte { + return e.key +} diff --git a/process/sync/metablock.go b/process/sync/metablock.go index c881dbc5c19..5021ba9b428 100644 --- a/process/sync/metablock.go +++ b/process/sync/metablock.go @@ -12,7 +12,6 @@ import ( "github.com/ElrondNetwork/elrond-go/process" "github.com/ElrondNetwork/elrond-go/state" "github.com/ElrondNetwork/elrond-go/storage" - "github.com/ElrondNetwork/elrond-go/trie" ) // MetaBootstrap implements the bootstrap mechanism @@ -181,7 +180,7 @@ func (boot *MetaBootstrap) setLastEpochStartRound() { func (boot *MetaBootstrap) SyncBlock(ctx context.Context) error { err := boot.syncBlock() if errors.IsGetNodeFromDBError(err) { - getNodeErr, ok := err.(*trie.GetErr) + getNodeErr, ok := err.(*errors.GetNodeFromDBErr) if !ok { return err } diff --git a/process/sync/metablock_test.go b/process/sync/metablock_test.go index f3b2c1bc361..efc230dedf9 100644 --- a/process/sync/metablock_test.go +++ b/process/sync/metablock_test.go @@ -19,6 +19,7 @@ import ( "github.com/ElrondNetwork/elrond-go/consensus/round" "github.com/ElrondNetwork/elrond-go/dataRetriever" "github.com/ElrondNetwork/elrond-go/dataRetriever/blockchain" + commonErrors "github.com/ElrondNetwork/elrond-go/errors" "github.com/ElrondNetwork/elrond-go/process" "github.com/ElrondNetwork/elrond-go/process/mock" "github.com/ElrondNetwork/elrond-go/process/sync" @@ -30,7 +31,6 @@ import ( stateMock "github.com/ElrondNetwork/elrond-go/testscommon/state" statusHandlerMock "github.com/ElrondNetwork/elrond-go/testscommon/statusHandler" storageStubs "github.com/ElrondNetwork/elrond-go/testscommon/storage" - "github.com/ElrondNetwork/elrond-go/trie" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -1622,7 +1622,7 @@ func TestMetaBootstrap_SyncBlockErrGetNodeDBShouldSyncAccounts(t *testing.T) { } args.ChainHandler = blkc - errGetNodeFromDB := trie.NewGetErr([]byte("key")) + errGetNodeFromDB := commonErrors.NewGetNodeFromDBErr([]byte("key")) blockProcessor := createMetaBlockProcessor(args.ChainHandler) blockProcessor.ProcessBlockCalled = func(header data.HeaderHandler, body data.BodyHandler, haveTime func() time.Duration) error { return errGetNodeFromDB diff --git a/process/sync/shardblock.go b/process/sync/shardblock.go index a975769d96f..204718a6b9c 100644 --- a/process/sync/shardblock.go +++ b/process/sync/shardblock.go @@ -12,7 +12,6 @@ import ( "github.com/ElrondNetwork/elrond-go/errors" "github.com/ElrondNetwork/elrond-go/process" "github.com/ElrondNetwork/elrond-go/storage" - "github.com/ElrondNetwork/elrond-go/trie" ) // ShardBootstrap implements the bootstrap mechanism @@ -145,7 +144,7 @@ func (boot *ShardBootstrap) StartSyncingBlocks() { func (boot *ShardBootstrap) SyncBlock(ctx context.Context) error { err := boot.syncBlock() if errors.IsGetNodeFromDBError(err) { - getNodeErr, ok := err.(*trie.GetErr) + getNodeErr, ok := err.(*errors.GetNodeFromDBErr) if !ok { return err } diff --git a/process/sync/shardblock_test.go b/process/sync/shardblock_test.go index 782577af3ab..26d4b469f2b 100644 --- a/process/sync/shardblock_test.go +++ b/process/sync/shardblock_test.go @@ -20,6 +20,7 @@ import ( "github.com/ElrondNetwork/elrond-go/consensus/round" "github.com/ElrondNetwork/elrond-go/dataRetriever" "github.com/ElrondNetwork/elrond-go/dataRetriever/blockchain" + commonErrors "github.com/ElrondNetwork/elrond-go/errors" "github.com/ElrondNetwork/elrond-go/process" "github.com/ElrondNetwork/elrond-go/process/mock" "github.com/ElrondNetwork/elrond-go/process/sync" @@ -34,7 +35,6 @@ import ( stateMock "github.com/ElrondNetwork/elrond-go/testscommon/state" statusHandlerMock "github.com/ElrondNetwork/elrond-go/testscommon/statusHandler" storageStubs "github.com/ElrondNetwork/elrond-go/testscommon/storage" - "github.com/ElrondNetwork/elrond-go/trie" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -2062,7 +2062,7 @@ func TestShardBootstrap_SyncBlockGetNodeDBErrorShouldSync(t *testing.T) { } args.ChainHandler = blkc - errGetNodeFromDB := trie.NewGetErr([]byte("key")) + errGetNodeFromDB := commonErrors.NewGetNodeFromDBErr([]byte("key")) blockProcessor := createBlockProcessor(args.ChainHandler) blockProcessor.ProcessBlockCalled = func(header data.HeaderHandler, body data.BodyHandler, haveTime func() time.Duration) error { return errGetNodeFromDB diff --git a/trie/errors.go b/trie/errors.go index 03142a6821c..a225a84c00c 100644 --- a/trie/errors.go +++ b/trie/errors.go @@ -1,37 +1,9 @@ package trie import ( - "encoding/hex" "errors" - "fmt" - - "github.com/ElrondNetwork/elrond-go/common" ) -// GetErr defines a custom error for trie get node -type GetErr struct { - key []byte -} - -// NewGetErr will create a new instance of GetErr -func NewGetErr(key []byte) *GetErr { - return &GetErr{key: key} -} - -// Error returns the error as string -func (e *GetErr) Error() string { - return fmt.Sprintf( - "%s for key %v", - common.GetNodeFromDBErrorString, - hex.EncodeToString(e.key), - ) -} - -// GetKey will return the key that generated the error -func (e *GetErr) GetKey() []byte { - return e.key -} - // ErrInvalidNode is raised when we reach an invalid node var ErrInvalidNode = errors.New("invalid node") diff --git a/trie/node.go b/trie/node.go index 5fe87fccaf7..45cff10ef97 100644 --- a/trie/node.go +++ b/trie/node.go @@ -118,7 +118,7 @@ func getNodeFromDBAndDecode(n []byte, db common.DBWriteCacher, marshalizer marsh encChild, err := db.Get(n) if err != nil { log.Trace(common.GetNodeFromDBErrorString, "error", err, "key", n) - return nil, NewGetErr(n) + return nil, errors.NewGetNodeFromDBErr(n) } return decodeNode(encChild, marshalizer, hasher) From 004264ca9183d88e879dc962fa48f05d4c80bc32 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Mon, 21 Nov 2022 23:23:31 +0200 Subject: [PATCH 03/33] fix unit test on context closing --- errors/missingTrieNodeError.go | 10 ++++++---- trie/node.go | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/errors/missingTrieNodeError.go b/errors/missingTrieNodeError.go index 3cf5961029f..2de83156eac 100644 --- a/errors/missingTrieNodeError.go +++ b/errors/missingTrieNodeError.go @@ -27,19 +27,21 @@ func IsGetNodeFromDBError(err error) bool { // GetNodeFromDBErr defines a custom error for trie get node type GetNodeFromDBErr struct { - key []byte + getErr error + key []byte } // NewGetNodeFromDBErr will create a new instance of GetNodeFromDBErr -func NewGetNodeFromDBErr(key []byte) *GetNodeFromDBErr { - return &GetNodeFromDBErr{key: key} +func NewGetNodeFromDBErr(key []byte, err error) *GetNodeFromDBErr { + return &GetNodeFromDBErr{getErr: err, key: key} } // Error returns the error as string func (e *GetNodeFromDBErr) Error() string { return fmt.Sprintf( - "%s for key %v", + "%s: %s for key %v", common.GetNodeFromDBErrorString, + e.getErr.Error(), hex.EncodeToString(e.key), ) } diff --git a/trie/node.go b/trie/node.go index 45cff10ef97..f6ee33bfe2e 100644 --- a/trie/node.go +++ b/trie/node.go @@ -118,7 +118,7 @@ func getNodeFromDBAndDecode(n []byte, db common.DBWriteCacher, marshalizer marsh encChild, err := db.Get(n) if err != nil { log.Trace(common.GetNodeFromDBErrorString, "error", err, "key", n) - return nil, errors.NewGetNodeFromDBErr(n) + return nil, errors.NewGetNodeFromDBErr(n, err) } return decodeNode(encChild, marshalizer, hasher) From 0d6e8d6ad507b55da198b4d17a2facfb04fa0595 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Mon, 21 Nov 2022 23:25:10 +0200 Subject: [PATCH 04/33] fix unit tests in process sync --- process/sync/metablock_test.go | 2 +- process/sync/shardblock_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/process/sync/metablock_test.go b/process/sync/metablock_test.go index efc230dedf9..07055519d90 100644 --- a/process/sync/metablock_test.go +++ b/process/sync/metablock_test.go @@ -1622,7 +1622,7 @@ func TestMetaBootstrap_SyncBlockErrGetNodeDBShouldSyncAccounts(t *testing.T) { } args.ChainHandler = blkc - errGetNodeFromDB := commonErrors.NewGetNodeFromDBErr([]byte("key")) + errGetNodeFromDB := commonErrors.NewGetNodeFromDBErr([]byte("key"), errors.New("get error")) blockProcessor := createMetaBlockProcessor(args.ChainHandler) blockProcessor.ProcessBlockCalled = func(header data.HeaderHandler, body data.BodyHandler, haveTime func() time.Duration) error { return errGetNodeFromDB diff --git a/process/sync/shardblock_test.go b/process/sync/shardblock_test.go index 26d4b469f2b..396a7805ff2 100644 --- a/process/sync/shardblock_test.go +++ b/process/sync/shardblock_test.go @@ -2062,7 +2062,7 @@ func TestShardBootstrap_SyncBlockGetNodeDBErrorShouldSync(t *testing.T) { } args.ChainHandler = blkc - errGetNodeFromDB := commonErrors.NewGetNodeFromDBErr([]byte("key")) + errGetNodeFromDB := commonErrors.NewGetNodeFromDBErr([]byte("key"), errors.New("get error")) blockProcessor := createBlockProcessor(args.ChainHandler) blockProcessor.ProcessBlockCalled = func(header data.HeaderHandler, body data.BodyHandler, haveTime func() time.Duration) error { return errGetNodeFromDB From 0141588292623a51a38e27d1d6ef9cd21231a4c1 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Wed, 23 Nov 2022 10:00:29 +0200 Subject: [PATCH 05/33] add custom error cast check in error check function --- errors/missingTrieNodeError.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/errors/missingTrieNodeError.go b/errors/missingTrieNodeError.go index 2de83156eac..b379c349baa 100644 --- a/errors/missingTrieNodeError.go +++ b/errors/missingTrieNodeError.go @@ -3,7 +3,6 @@ package errors import ( "encoding/hex" "fmt" - "strings" "github.com/ElrondNetwork/elrond-go/common" ) @@ -18,11 +17,12 @@ func IsGetNodeFromDBError(err error) bool { return false } - if strings.Contains(err.Error(), common.GetNodeFromDBErrorString) { - return true + _, ok := err.(*GetNodeFromDBErr) + if !ok { + return false } - return false + return true } // GetNodeFromDBErr defines a custom error for trie get node From 8b2d6e4db684f0a266cbdc6661bf484da4e8343d Mon Sep 17 00:00:00 2001 From: ssd04 Date: Wed, 23 Nov 2022 10:47:37 +0200 Subject: [PATCH 06/33] use new custom error in process unit tests --- process/block/preprocess/rewardTxPreProcessor_test.go | 6 +++--- process/block/preprocess/smartContractResults_test.go | 6 +++--- process/block/preprocess/transactionsV2_test.go | 6 +++--- process/block/preprocess/transactions_test.go | 3 ++- process/rewardTransaction/process_test.go | 5 ++--- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/process/block/preprocess/rewardTxPreProcessor_test.go b/process/block/preprocess/rewardTxPreProcessor_test.go index 35c09c7347b..9629b956ac4 100644 --- a/process/block/preprocess/rewardTxPreProcessor_test.go +++ b/process/block/preprocess/rewardTxPreProcessor_test.go @@ -1,7 +1,7 @@ package preprocess import ( - "fmt" + "errors" "testing" "time" @@ -9,8 +9,8 @@ import ( "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/rewardTx" - "github.com/ElrondNetwork/elrond-go/common" "github.com/ElrondNetwork/elrond-go/dataRetriever" + commonErrors "github.com/ElrondNetwork/elrond-go/errors" "github.com/ElrondNetwork/elrond-go/process" "github.com/ElrondNetwork/elrond-go/process/mock" "github.com/ElrondNetwork/elrond-go/storage" @@ -684,7 +684,7 @@ func TestRewardTxPreprocessor_ProcessBlockTransactions(t *testing.T) { func TestRewardTxPreprocessor_ProcessBlockTransactionsMissingTrieNode(t *testing.T) { t.Parallel() - missingNodeErr := fmt.Errorf(common.GetNodeFromDBErrorString) + missingNodeErr := commonErrors.NewGetNodeFromDBErr([]byte("key"), errors.New("get error")) txHash := testTxHash tdp := initDataPool() rtp, _ := NewRewardTxPreprocessor( diff --git a/process/block/preprocess/smartContractResults_test.go b/process/block/preprocess/smartContractResults_test.go index 5fcad42a95f..66287ae04fa 100644 --- a/process/block/preprocess/smartContractResults_test.go +++ b/process/block/preprocess/smartContractResults_test.go @@ -2,7 +2,7 @@ package preprocess import ( "encoding/json" - "fmt" + "errors" "reflect" "testing" "time" @@ -12,8 +12,8 @@ import ( "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/smartContractResult" - "github.com/ElrondNetwork/elrond-go/common" "github.com/ElrondNetwork/elrond-go/dataRetriever" + commonErrors "github.com/ElrondNetwork/elrond-go/errors" "github.com/ElrondNetwork/elrond-go/process" "github.com/ElrondNetwork/elrond-go/process/mock" "github.com/ElrondNetwork/elrond-go/storage" @@ -1106,7 +1106,7 @@ func TestScrsPreprocessor_ProcessBlockTransactionsShouldWork(t *testing.T) { func TestScrsPreprocessor_ProcessBlockTransactionsMissingTrieNode(t *testing.T) { t.Parallel() - missingNodeErr := fmt.Errorf(common.GetNodeFromDBErrorString) + missingNodeErr := commonErrors.NewGetNodeFromDBErr([]byte("key"), errors.New("get error")) tdp := initDataPool() requestTransaction := func(shardID uint32, txHashes [][]byte) {} scrPreproc, _ := NewSmartContractResultPreprocessor( diff --git a/process/block/preprocess/transactionsV2_test.go b/process/block/preprocess/transactionsV2_test.go index 1fa8a23d33e..5823d5b33ce 100644 --- a/process/block/preprocess/transactionsV2_test.go +++ b/process/block/preprocess/transactionsV2_test.go @@ -2,7 +2,7 @@ package preprocess import ( "bytes" - "fmt" + "errors" "math/big" "testing" @@ -11,7 +11,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/smartContractResult" "github.com/ElrondNetwork/elrond-go-core/data/transaction" - "github.com/ElrondNetwork/elrond-go/common" + commonErrors "github.com/ElrondNetwork/elrond-go/errors" "github.com/ElrondNetwork/elrond-go/process" "github.com/ElrondNetwork/elrond-go/process/mock" "github.com/ElrondNetwork/elrond-go/storage/txcache" @@ -757,7 +757,7 @@ func TestTransactions_CreateAndProcessMiniBlocksFromMeV2ShouldWork(t *testing.T) func TestTransactions_CreateAndProcessMiniBlocksFromMeV2MissingTrieNode(t *testing.T) { t.Parallel() - missingNodeErr := fmt.Errorf(common.GetNodeFromDBErrorString) + missingNodeErr := commonErrors.NewGetNodeFromDBErr([]byte("key"), errors.New("get error")) preprocessor := createTransactionPreprocessor() preprocessor.txProcessor = &testscommon.TxProcessorMock{ ProcessTransactionCalled: func(transaction *transaction.Transaction) (vmcommon.ReturnCode, error) { diff --git a/process/block/preprocess/transactions_test.go b/process/block/preprocess/transactions_test.go index d74bf26646d..2e20593c25b 100644 --- a/process/block/preprocess/transactions_test.go +++ b/process/block/preprocess/transactions_test.go @@ -24,6 +24,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/marshal" "github.com/ElrondNetwork/elrond-go/common" "github.com/ElrondNetwork/elrond-go/dataRetriever" + commonErrors "github.com/ElrondNetwork/elrond-go/errors" "github.com/ElrondNetwork/elrond-go/process" "github.com/ElrondNetwork/elrond-go/process/mock" "github.com/ElrondNetwork/elrond-go/sharding" @@ -1152,7 +1153,7 @@ func TestTransactionPreprocessor_ProcessTxsToMeShouldUseCorrectSenderAndReceiver func TestTransactionPreprocessor_ProcessTxsToMeMissingTrieNode(t *testing.T) { t.Parallel() - missingNodeErr := fmt.Errorf(common.GetNodeFromDBErrorString) + missingNodeErr := commonErrors.NewGetNodeFromDBErr([]byte("key"), errors.New("get error")) args := createDefaultTransactionsProcessorArgs() args.Accounts = &stateMock.AccountsStub{ diff --git a/process/rewardTransaction/process_test.go b/process/rewardTransaction/process_test.go index 427daf567b4..e560d5f6f47 100644 --- a/process/rewardTransaction/process_test.go +++ b/process/rewardTransaction/process_test.go @@ -2,13 +2,12 @@ package rewardTransaction_test import ( "errors" - "fmt" "math/big" "testing" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/data/rewardTx" - "github.com/ElrondNetwork/elrond-go/common" + commonErrors "github.com/ElrondNetwork/elrond-go/errors" "github.com/ElrondNetwork/elrond-go/process" "github.com/ElrondNetwork/elrond-go/process/mock" "github.com/ElrondNetwork/elrond-go/process/rewardTransaction" @@ -218,7 +217,7 @@ func TestRewardTxProcessor_ProcessRewardTransactionShouldWork(t *testing.T) { func TestRewardTxProcessor_ProcessRewardTransactionMissingTrieNode(t *testing.T) { t.Parallel() - missingNodeErr := fmt.Errorf(common.GetNodeFromDBErrorString) + missingNodeErr := commonErrors.NewGetNodeFromDBErr([]byte("key"), errors.New("get error")) accountsDb := &stateMock.AccountsStub{ LoadAccountCalled: func(address []byte) (vmcommon.AccountHandler, error) { acc, _ := state.NewUserAccount(address) From 8d6ea224fdc8d8ed0290239e058620831e8ef171 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Wed, 23 Nov 2022 11:00:47 +0200 Subject: [PATCH 07/33] fix linter issue --- errors/missingTrieNodeError.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/errors/missingTrieNodeError.go b/errors/missingTrieNodeError.go index b379c349baa..4e0774b0c54 100644 --- a/errors/missingTrieNodeError.go +++ b/errors/missingTrieNodeError.go @@ -18,11 +18,7 @@ func IsGetNodeFromDBError(err error) bool { } _, ok := err.(*GetNodeFromDBErr) - if !ok { - return false - } - - return true + return ok } // GetNodeFromDBErr defines a custom error for trie get node From 3bcb4a334134418e28c26b792b4894a836cdce57 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Wed, 23 Nov 2022 12:30:46 +0200 Subject: [PATCH 08/33] revert get node from db error additional check --- errors/missingTrieNodeError.go | 8 ++++++-- process/block/preprocess/rewardTxPreProcessor_test.go | 6 +++--- process/block/preprocess/smartContractResults_test.go | 6 +++--- process/block/preprocess/transactionsV2_test.go | 6 +++--- process/block/preprocess/transactions_test.go | 3 +-- process/rewardTransaction/process_test.go | 5 +++-- 6 files changed, 19 insertions(+), 15 deletions(-) diff --git a/errors/missingTrieNodeError.go b/errors/missingTrieNodeError.go index 4e0774b0c54..2de83156eac 100644 --- a/errors/missingTrieNodeError.go +++ b/errors/missingTrieNodeError.go @@ -3,6 +3,7 @@ package errors import ( "encoding/hex" "fmt" + "strings" "github.com/ElrondNetwork/elrond-go/common" ) @@ -17,8 +18,11 @@ func IsGetNodeFromDBError(err error) bool { return false } - _, ok := err.(*GetNodeFromDBErr) - return ok + if strings.Contains(err.Error(), common.GetNodeFromDBErrorString) { + return true + } + + return false } // GetNodeFromDBErr defines a custom error for trie get node diff --git a/process/block/preprocess/rewardTxPreProcessor_test.go b/process/block/preprocess/rewardTxPreProcessor_test.go index 9629b956ac4..35c09c7347b 100644 --- a/process/block/preprocess/rewardTxPreProcessor_test.go +++ b/process/block/preprocess/rewardTxPreProcessor_test.go @@ -1,7 +1,7 @@ package preprocess import ( - "errors" + "fmt" "testing" "time" @@ -9,8 +9,8 @@ import ( "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/rewardTx" + "github.com/ElrondNetwork/elrond-go/common" "github.com/ElrondNetwork/elrond-go/dataRetriever" - commonErrors "github.com/ElrondNetwork/elrond-go/errors" "github.com/ElrondNetwork/elrond-go/process" "github.com/ElrondNetwork/elrond-go/process/mock" "github.com/ElrondNetwork/elrond-go/storage" @@ -684,7 +684,7 @@ func TestRewardTxPreprocessor_ProcessBlockTransactions(t *testing.T) { func TestRewardTxPreprocessor_ProcessBlockTransactionsMissingTrieNode(t *testing.T) { t.Parallel() - missingNodeErr := commonErrors.NewGetNodeFromDBErr([]byte("key"), errors.New("get error")) + missingNodeErr := fmt.Errorf(common.GetNodeFromDBErrorString) txHash := testTxHash tdp := initDataPool() rtp, _ := NewRewardTxPreprocessor( diff --git a/process/block/preprocess/smartContractResults_test.go b/process/block/preprocess/smartContractResults_test.go index 66287ae04fa..5fcad42a95f 100644 --- a/process/block/preprocess/smartContractResults_test.go +++ b/process/block/preprocess/smartContractResults_test.go @@ -2,7 +2,7 @@ package preprocess import ( "encoding/json" - "errors" + "fmt" "reflect" "testing" "time" @@ -12,8 +12,8 @@ import ( "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/smartContractResult" + "github.com/ElrondNetwork/elrond-go/common" "github.com/ElrondNetwork/elrond-go/dataRetriever" - commonErrors "github.com/ElrondNetwork/elrond-go/errors" "github.com/ElrondNetwork/elrond-go/process" "github.com/ElrondNetwork/elrond-go/process/mock" "github.com/ElrondNetwork/elrond-go/storage" @@ -1106,7 +1106,7 @@ func TestScrsPreprocessor_ProcessBlockTransactionsShouldWork(t *testing.T) { func TestScrsPreprocessor_ProcessBlockTransactionsMissingTrieNode(t *testing.T) { t.Parallel() - missingNodeErr := commonErrors.NewGetNodeFromDBErr([]byte("key"), errors.New("get error")) + missingNodeErr := fmt.Errorf(common.GetNodeFromDBErrorString) tdp := initDataPool() requestTransaction := func(shardID uint32, txHashes [][]byte) {} scrPreproc, _ := NewSmartContractResultPreprocessor( diff --git a/process/block/preprocess/transactionsV2_test.go b/process/block/preprocess/transactionsV2_test.go index 5823d5b33ce..1fa8a23d33e 100644 --- a/process/block/preprocess/transactionsV2_test.go +++ b/process/block/preprocess/transactionsV2_test.go @@ -2,7 +2,7 @@ package preprocess import ( "bytes" - "errors" + "fmt" "math/big" "testing" @@ -11,7 +11,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/smartContractResult" "github.com/ElrondNetwork/elrond-go-core/data/transaction" - commonErrors "github.com/ElrondNetwork/elrond-go/errors" + "github.com/ElrondNetwork/elrond-go/common" "github.com/ElrondNetwork/elrond-go/process" "github.com/ElrondNetwork/elrond-go/process/mock" "github.com/ElrondNetwork/elrond-go/storage/txcache" @@ -757,7 +757,7 @@ func TestTransactions_CreateAndProcessMiniBlocksFromMeV2ShouldWork(t *testing.T) func TestTransactions_CreateAndProcessMiniBlocksFromMeV2MissingTrieNode(t *testing.T) { t.Parallel() - missingNodeErr := commonErrors.NewGetNodeFromDBErr([]byte("key"), errors.New("get error")) + missingNodeErr := fmt.Errorf(common.GetNodeFromDBErrorString) preprocessor := createTransactionPreprocessor() preprocessor.txProcessor = &testscommon.TxProcessorMock{ ProcessTransactionCalled: func(transaction *transaction.Transaction) (vmcommon.ReturnCode, error) { diff --git a/process/block/preprocess/transactions_test.go b/process/block/preprocess/transactions_test.go index 2e20593c25b..d74bf26646d 100644 --- a/process/block/preprocess/transactions_test.go +++ b/process/block/preprocess/transactions_test.go @@ -24,7 +24,6 @@ import ( "github.com/ElrondNetwork/elrond-go-core/marshal" "github.com/ElrondNetwork/elrond-go/common" "github.com/ElrondNetwork/elrond-go/dataRetriever" - commonErrors "github.com/ElrondNetwork/elrond-go/errors" "github.com/ElrondNetwork/elrond-go/process" "github.com/ElrondNetwork/elrond-go/process/mock" "github.com/ElrondNetwork/elrond-go/sharding" @@ -1153,7 +1152,7 @@ func TestTransactionPreprocessor_ProcessTxsToMeShouldUseCorrectSenderAndReceiver func TestTransactionPreprocessor_ProcessTxsToMeMissingTrieNode(t *testing.T) { t.Parallel() - missingNodeErr := commonErrors.NewGetNodeFromDBErr([]byte("key"), errors.New("get error")) + missingNodeErr := fmt.Errorf(common.GetNodeFromDBErrorString) args := createDefaultTransactionsProcessorArgs() args.Accounts = &stateMock.AccountsStub{ diff --git a/process/rewardTransaction/process_test.go b/process/rewardTransaction/process_test.go index e560d5f6f47..427daf567b4 100644 --- a/process/rewardTransaction/process_test.go +++ b/process/rewardTransaction/process_test.go @@ -2,12 +2,13 @@ package rewardTransaction_test import ( "errors" + "fmt" "math/big" "testing" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/data/rewardTx" - commonErrors "github.com/ElrondNetwork/elrond-go/errors" + "github.com/ElrondNetwork/elrond-go/common" "github.com/ElrondNetwork/elrond-go/process" "github.com/ElrondNetwork/elrond-go/process/mock" "github.com/ElrondNetwork/elrond-go/process/rewardTransaction" @@ -217,7 +218,7 @@ func TestRewardTxProcessor_ProcessRewardTransactionShouldWork(t *testing.T) { func TestRewardTxProcessor_ProcessRewardTransactionMissingTrieNode(t *testing.T) { t.Parallel() - missingNodeErr := commonErrors.NewGetNodeFromDBErr([]byte("key"), errors.New("get error")) + missingNodeErr := fmt.Errorf(common.GetNodeFromDBErrorString) accountsDb := &stateMock.AccountsStub{ LoadAccountCalled: func(address []byte) (vmcommon.AccountHandler, error) { acc, _ := state.NewUserAccount(address) From 4462ceda6d573ac38858b9d3132b9c1e45f6fe40 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Wed, 23 Nov 2022 15:06:50 +0200 Subject: [PATCH 09/33] use identifier to determine storage type --- common/constants.go | 3 +++ errors/missingTrieNodeError.go | 18 ++++++++++++++---- process/sync/metablock.go | 34 +++++++++++++--------------------- trie/node.go | 10 ++++++++-- 4 files changed, 38 insertions(+), 27 deletions(-) diff --git a/common/constants.go b/common/constants.go index 4aa4033eaa0..bf2f53854c8 100644 --- a/common/constants.go +++ b/common/constants.go @@ -834,3 +834,6 @@ const MetricTrieSyncNumReceivedBytes = "erd_trie_sync_num_bytes_received" // MetricTrieSyncNumProcessedNodes is the metric that outputs the number of trie nodes processed for accounts during trie sync const MetricTrieSyncNumProcessedNodes = "erd_trie_sync_num_nodes_processed" + +const AccountsTrieIdentifier = "AccountsTrie" +const PeerAccountsTrieIdentifier = "PeerAccountsTrie" diff --git a/errors/missingTrieNodeError.go b/errors/missingTrieNodeError.go index 2de83156eac..cd9782df5a3 100644 --- a/errors/missingTrieNodeError.go +++ b/errors/missingTrieNodeError.go @@ -27,13 +27,18 @@ func IsGetNodeFromDBError(err error) bool { // GetNodeFromDBErr defines a custom error for trie get node type GetNodeFromDBErr struct { - getErr error - key []byte + getErr error + key []byte + identifier string } // NewGetNodeFromDBErr will create a new instance of GetNodeFromDBErr -func NewGetNodeFromDBErr(key []byte, err error) *GetNodeFromDBErr { - return &GetNodeFromDBErr{getErr: err, key: key} +func NewGetNodeFromDBErr(key []byte, err error, id string) *GetNodeFromDBErr { + return &GetNodeFromDBErr{ + getErr: err, + key: key, + identifier: id, + } } // Error returns the error as string @@ -50,3 +55,8 @@ func (e *GetNodeFromDBErr) Error() string { func (e *GetNodeFromDBErr) GetKey() []byte { return e.key } + +// GetKey will return the key that generated the error +func (e *GetNodeFromDBErr) GetIdentifier() string { + return e.identifier +} diff --git a/process/sync/metablock.go b/process/sync/metablock.go index 5021ba9b428..f673671ef8b 100644 --- a/process/sync/metablock.go +++ b/process/sync/metablock.go @@ -2,11 +2,13 @@ package sync import ( "context" + "fmt" "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/core/check" "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" + "github.com/ElrondNetwork/elrond-go/common" "github.com/ElrondNetwork/elrond-go/dataRetriever" "github.com/ElrondNetwork/elrond-go/errors" "github.com/ElrondNetwork/elrond-go/process" @@ -185,37 +187,27 @@ func (boot *MetaBootstrap) SyncBlock(ctx context.Context) error { return err } - errSync := boot.syncAccountsDBs(getNodeErr.GetKey()) + errSync := boot.syncAccountsDBs(getNodeErr.GetKey(), getNodeErr.GetIdentifier()) boot.handleTrieSyncError(errSync, ctx) } return err } -func (boot *MetaBootstrap) syncAccountsDBs(key []byte) error { - var err error - - err = boot.syncValidatorAccountsState() - if err != nil { - return err - } - - err = boot.syncUserAccountsState(key) - if err != nil { - return err +func (boot *MetaBootstrap) syncAccountsDBs(key []byte, id string) error { + switch id { + case common.AccountsTrieIdentifier: + return boot.syncUserAccountsState(key) + case common.PeerAccountsTrieIdentifier: + return boot.syncValidatorAccountsState(key) + default: + return fmt.Errorf("invalid trie identifier, id %s", id) } - - return nil } -func (boot *MetaBootstrap) syncValidatorAccountsState() error { - rootHash, err := boot.validatorAccountsDB.RootHash() - if err != nil { - return err - } - +func (boot *MetaBootstrap) syncValidatorAccountsState(key []byte) error { log.Warn("base sync: started syncValidatorAccountsState") - return boot.validatorStatisticsDBSyncer.SyncAccounts(rootHash) + return boot.validatorStatisticsDBSyncer.SyncAccounts(key) } // Close closes the synchronization loop diff --git a/trie/node.go b/trie/node.go index f6ee33bfe2e..53665f7c0c6 100644 --- a/trie/node.go +++ b/trie/node.go @@ -9,6 +9,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/marshal" "github.com/ElrondNetwork/elrond-go/common" "github.com/ElrondNetwork/elrond-go/errors" + "github.com/ElrondNetwork/elrond-go/storage/pruning" "github.com/ElrondNetwork/elrond-go/trie/keyBuilder" ) @@ -117,8 +118,13 @@ func computeAndSetNodeHash(n node) ([]byte, error) { func getNodeFromDBAndDecode(n []byte, db common.DBWriteCacher, marshalizer marshal.Marshalizer, hasher hashing.Hasher) (node, error) { encChild, err := db.Get(n) if err != nil { - log.Trace(common.GetNodeFromDBErrorString, "error", err, "key", n) - return nil, errors.NewGetNodeFromDBErr(n, err) + dbWithID, ok := db.(*pruning.TriePruningStorerWithID) + if !ok { + return nil, errors.NewGetNodeFromDBErr(n, err, "") + } + + log.Debug(common.GetNodeFromDBErrorString, "error", err, "key", n) + return nil, errors.NewGetNodeFromDBErr(n, err, dbWithID.GetIdentifier()) } return decodeNode(encChild, marshalizer, hasher) From 6c5d0730d75be58d69403e189206fc028203e537 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Wed, 23 Nov 2022 16:29:01 +0200 Subject: [PATCH 10/33] add custom interface for get identifier --- process/sync/export_test.go | 5 +++ process/sync/metablock_test.go | 54 +++++++++++++++++++++++----- process/sync/shardblock_test.go | 2 +- storage/pruning/triePruningStorer.go | 5 +++ trie/interface.go | 5 +++ trie/node.go | 3 +- 6 files changed, 62 insertions(+), 12 deletions(-) diff --git a/process/sync/export_test.go b/process/sync/export_test.go index 7f243a2fe54..53b7242e5f4 100644 --- a/process/sync/export_test.go +++ b/process/sync/export_test.go @@ -175,6 +175,11 @@ func (boot *MetaBootstrap) GetNotarizedInfo( } } +// SyncAccountsDBs - +func (boot *MetaBootstrap) SyncAccountsDBs(key []byte, id string) error { + return boot.syncAccountsDBs(key, id) +} + // ProcessReceivedHeader - func (boot *baseBootstrap) ProcessReceivedHeader(headerHandler data.HeaderHandler, headerHash []byte) { boot.processReceivedHeader(headerHandler, headerHash) diff --git a/process/sync/metablock_test.go b/process/sync/metablock_test.go index 07055519d90..3bb64c69a7a 100644 --- a/process/sync/metablock_test.go +++ b/process/sync/metablock_test.go @@ -1622,7 +1622,7 @@ func TestMetaBootstrap_SyncBlockErrGetNodeDBShouldSyncAccounts(t *testing.T) { } args.ChainHandler = blkc - errGetNodeFromDB := commonErrors.NewGetNodeFromDBErr([]byte("key"), errors.New("get error")) + errGetNodeFromDB := commonErrors.NewGetNodeFromDBErr([]byte("key"), errors.New("get error"), common.AccountsTrieIdentifier) blockProcessor := createMetaBlockProcessor(args.ChainHandler) blockProcessor.ProcessBlockCalled = func(header data.HeaderHandler, body data.BodyHandler, haveTime func() time.Duration) error { return errGetNodeFromDB @@ -1681,13 +1681,8 @@ func TestMetaBootstrap_SyncBlockErrGetNodeDBShouldSyncAccounts(t *testing.T) { SyncAccountsCalled: func(rootHash []byte) error { accountsSyncCalled = true return nil - }} - validatorSyncCalled := false - args.ValidatorStatisticsDBSyncer = &mock.AccountsDBSyncerStub{ - SyncAccountsCalled: func(rootHash []byte) error { - validatorSyncCalled = true - return nil - }} + }, + } args.Accounts = &stateMock.AccountsStub{RootHashCalled: func() ([]byte, error) { return []byte("roothash"), nil }} @@ -1700,5 +1695,46 @@ func TestMetaBootstrap_SyncBlockErrGetNodeDBShouldSyncAccounts(t *testing.T) { assert.Equal(t, errGetNodeFromDB, err) assert.True(t, accountsSyncCalled) - assert.True(t, validatorSyncCalled) +} + +func TestMetaBootstrap_SyncAccountsDBs(t *testing.T) { + t.Parallel() + + t.Run("sync user accounts state", func(t *testing.T) { + t.Parallel() + + args := CreateMetaBootstrapMockArguments() + accountsSyncCalled := false + args.AccountsDBSyncer = &mock.AccountsDBSyncerStub{ + SyncAccountsCalled: func(rootHash []byte) error { + accountsSyncCalled = true + return nil + }, + } + + bs, _ := sync.NewMetaBootstrap(args) + + err := bs.SyncAccountsDBs([]byte("key"), common.AccountsTrieIdentifier) + require.Nil(t, err) + require.True(t, accountsSyncCalled) + }) + + t.Run("sync validator accounts state", func(t *testing.T) { + t.Parallel() + + args := CreateMetaBootstrapMockArguments() + accountsSyncCalled := false + args.ValidatorStatisticsDBSyncer = &mock.AccountsDBSyncerStub{ + SyncAccountsCalled: func(rootHash []byte) error { + accountsSyncCalled = true + return nil + }, + } + + bs, _ := sync.NewMetaBootstrap(args) + + err := bs.SyncAccountsDBs([]byte("key"), common.PeerAccountsTrieIdentifier) + require.Nil(t, err) + require.True(t, accountsSyncCalled) + }) } diff --git a/process/sync/shardblock_test.go b/process/sync/shardblock_test.go index 396a7805ff2..000134eeb5b 100644 --- a/process/sync/shardblock_test.go +++ b/process/sync/shardblock_test.go @@ -2062,7 +2062,7 @@ func TestShardBootstrap_SyncBlockGetNodeDBErrorShouldSync(t *testing.T) { } args.ChainHandler = blkc - errGetNodeFromDB := commonErrors.NewGetNodeFromDBErr([]byte("key"), errors.New("get error")) + errGetNodeFromDB := commonErrors.NewGetNodeFromDBErr([]byte("key"), errors.New("get error"), "") blockProcessor := createBlockProcessor(args.ChainHandler) blockProcessor.ProcessBlockCalled = func(header data.HeaderHandler, body data.BodyHandler, haveTime func() time.Duration) error { return errGetNodeFromDB diff --git a/storage/pruning/triePruningStorer.go b/storage/pruning/triePruningStorer.go index bda72d5f23f..adfcbe3c195 100644 --- a/storage/pruning/triePruningStorer.go +++ b/storage/pruning/triePruningStorer.go @@ -164,6 +164,11 @@ func (ps *triePruningStorer) GetLatestStorageEpoch() (uint32, error) { return ps.activePersisters[currentEpochIndex].epoch, nil } +// GetIdentifier returns the identifier for storer +func (ps *triePruningStorer) GetIdentifier() string { + return ps.identifier +} + // IsInterfaceNil returns true if there is no value under the interface func (ps *triePruningStorer) IsInterfaceNil() bool { return ps == nil diff --git a/trie/interface.go b/trie/interface.go index a18d73947cb..0c52b345674 100644 --- a/trie/interface.go +++ b/trie/interface.go @@ -128,3 +128,8 @@ type storageManagerExtension interface { type StorageMarker interface { MarkStorerAsSyncedAndActive(storer common.StorageManager) } + +type dbWriteCacherWithIdentifier interface { + common.DBWriteCacher + GetIdentifier() string +} diff --git a/trie/node.go b/trie/node.go index 53665f7c0c6..c7b35b297d8 100644 --- a/trie/node.go +++ b/trie/node.go @@ -9,7 +9,6 @@ import ( "github.com/ElrondNetwork/elrond-go-core/marshal" "github.com/ElrondNetwork/elrond-go/common" "github.com/ElrondNetwork/elrond-go/errors" - "github.com/ElrondNetwork/elrond-go/storage/pruning" "github.com/ElrondNetwork/elrond-go/trie/keyBuilder" ) @@ -118,7 +117,7 @@ func computeAndSetNodeHash(n node) ([]byte, error) { func getNodeFromDBAndDecode(n []byte, db common.DBWriteCacher, marshalizer marshal.Marshalizer, hasher hashing.Hasher) (node, error) { encChild, err := db.Get(n) if err != nil { - dbWithID, ok := db.(*pruning.TriePruningStorerWithID) + dbWithID, ok := db.(dbWriteCacherWithIdentifier) if !ok { return nil, errors.NewGetNodeFromDBErr(n, err, "") } From d7e1b6bc381324c5b7ec2d23797844483a315818 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Wed, 23 Nov 2022 16:33:29 +0200 Subject: [PATCH 11/33] fix comments and log messages --- common/constants.go | 3 +++ process/sync/metablock.go | 2 +- trie/node.go | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/common/constants.go b/common/constants.go index bf2f53854c8..3736867b83f 100644 --- a/common/constants.go +++ b/common/constants.go @@ -835,5 +835,8 @@ const MetricTrieSyncNumReceivedBytes = "erd_trie_sync_num_bytes_received" // MetricTrieSyncNumProcessedNodes is the metric that outputs the number of trie nodes processed for accounts during trie sync const MetricTrieSyncNumProcessedNodes = "erd_trie_sync_num_nodes_processed" +// AccountsTrieIdentifier defines the identifier for accounts trie storer const AccountsTrieIdentifier = "AccountsTrie" + +// PeerAccountsTrieIdentifier defines the identifier for peer accounts storer const PeerAccountsTrieIdentifier = "PeerAccountsTrie" diff --git a/process/sync/metablock.go b/process/sync/metablock.go index f673671ef8b..6fb6f195e10 100644 --- a/process/sync/metablock.go +++ b/process/sync/metablock.go @@ -201,7 +201,7 @@ func (boot *MetaBootstrap) syncAccountsDBs(key []byte, id string) error { case common.PeerAccountsTrieIdentifier: return boot.syncValidatorAccountsState(key) default: - return fmt.Errorf("invalid trie identifier, id %s", id) + return fmt.Errorf("invalid trie identifier, id: %s", id) } } diff --git a/trie/node.go b/trie/node.go index c7b35b297d8..724ad650d8a 100644 --- a/trie/node.go +++ b/trie/node.go @@ -119,10 +119,11 @@ func getNodeFromDBAndDecode(n []byte, db common.DBWriteCacher, marshalizer marsh if err != nil { dbWithID, ok := db.(dbWriteCacherWithIdentifier) if !ok { + log.Trace(common.GetNodeFromDBErrorString, "error", err, "key", n) return nil, errors.NewGetNodeFromDBErr(n, err, "") } - log.Debug(common.GetNodeFromDBErrorString, "error", err, "key", n) + log.Trace(common.GetNodeFromDBErrorString, "error", err, "key", n) return nil, errors.NewGetNodeFromDBErr(n, err, dbWithID.GetIdentifier()) } From ac87c63fe245a3684bfbf5dfe72bdbd303016c2a Mon Sep 17 00:00:00 2001 From: ssd04 Date: Thu, 24 Nov 2022 09:52:17 +0200 Subject: [PATCH 12/33] fixes after review: comments fixes + renamings --- errors/missingTrieNodeError.go | 16 ++++++++-------- storage/pruning/pruningStorer.go | 5 +++++ storage/pruning/triePruningStorer.go | 5 ----- trie/interface.go | 1 - trie/node.go | 2 +- 5 files changed, 14 insertions(+), 15 deletions(-) diff --git a/errors/missingTrieNodeError.go b/errors/missingTrieNodeError.go index cd9782df5a3..a4a386f049e 100644 --- a/errors/missingTrieNodeError.go +++ b/errors/missingTrieNodeError.go @@ -27,17 +27,17 @@ func IsGetNodeFromDBError(err error) bool { // GetNodeFromDBErr defines a custom error for trie get node type GetNodeFromDBErr struct { - getErr error - key []byte - identifier string + getErr error + key []byte + dbIdentifier string } // NewGetNodeFromDBErr will create a new instance of GetNodeFromDBErr func NewGetNodeFromDBErr(key []byte, err error, id string) *GetNodeFromDBErr { return &GetNodeFromDBErr{ - getErr: err, - key: key, - identifier: id, + getErr: err, + key: key, + dbIdentifier: id, } } @@ -56,7 +56,7 @@ func (e *GetNodeFromDBErr) GetKey() []byte { return e.key } -// GetKey will return the key that generated the error +// GetIdentifier will return the db corresponding to the db func (e *GetNodeFromDBErr) GetIdentifier() string { - return e.identifier + return e.dbIdentifier } diff --git a/storage/pruning/pruningStorer.go b/storage/pruning/pruningStorer.go index 69c5946cb1b..dc3c5e7bf10 100644 --- a/storage/pruning/pruningStorer.go +++ b/storage/pruning/pruningStorer.go @@ -1033,6 +1033,11 @@ func (ps *PruningStorer) RangeKeys(_ func(key []byte, val []byte) bool) { debug.PrintStack() } +// GetIdentifier returns the identifier for storer +func (ps *PruningStorer) GetIdentifier() string { + return ps.identifier +} + // IsInterfaceNil returns true if there is no value under the interface func (ps *PruningStorer) IsInterfaceNil() bool { return ps == nil diff --git a/storage/pruning/triePruningStorer.go b/storage/pruning/triePruningStorer.go index adfcbe3c195..bda72d5f23f 100644 --- a/storage/pruning/triePruningStorer.go +++ b/storage/pruning/triePruningStorer.go @@ -164,11 +164,6 @@ func (ps *triePruningStorer) GetLatestStorageEpoch() (uint32, error) { return ps.activePersisters[currentEpochIndex].epoch, nil } -// GetIdentifier returns the identifier for storer -func (ps *triePruningStorer) GetIdentifier() string { - return ps.identifier -} - // IsInterfaceNil returns true if there is no value under the interface func (ps *triePruningStorer) IsInterfaceNil() bool { return ps == nil diff --git a/trie/interface.go b/trie/interface.go index 0c52b345674..99940af882e 100644 --- a/trie/interface.go +++ b/trie/interface.go @@ -130,6 +130,5 @@ type StorageMarker interface { } type dbWriteCacherWithIdentifier interface { - common.DBWriteCacher GetIdentifier() string } diff --git a/trie/node.go b/trie/node.go index 724ad650d8a..55735146775 100644 --- a/trie/node.go +++ b/trie/node.go @@ -119,7 +119,7 @@ func getNodeFromDBAndDecode(n []byte, db common.DBWriteCacher, marshalizer marsh if err != nil { dbWithID, ok := db.(dbWriteCacherWithIdentifier) if !ok { - log.Trace(common.GetNodeFromDBErrorString, "error", err, "key", n) + log.Warn("wrong type assertion on", common.GetNodeFromDBErrorString, "error", err, "key", n) return nil, errors.NewGetNodeFromDBErr(n, err, "") } From 113c8339fb2c9b4935f2b00b85f107058ce46cdf Mon Sep 17 00:00:00 2001 From: ssd04 Date: Wed, 14 Dec 2022 12:21:23 +0200 Subject: [PATCH 13/33] error from db renaming --- errors/missingTrieNodeError.go | 18 +++++++++--------- process/sync/metablock.go | 3 ++- process/sync/metablock_test.go | 2 +- process/sync/shardblock.go | 2 +- process/sync/shardblock_test.go | 2 +- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/errors/missingTrieNodeError.go b/errors/missingTrieNodeError.go index a4a386f049e..2e59df14156 100644 --- a/errors/missingTrieNodeError.go +++ b/errors/missingTrieNodeError.go @@ -25,16 +25,16 @@ func IsGetNodeFromDBError(err error) bool { return false } -// GetNodeFromDBErr defines a custom error for trie get node -type GetNodeFromDBErr struct { +// GetNodeFromDBErrWithKey defines a custom error for trie get node +type GetNodeFromDBErrWithKey struct { getErr error key []byte dbIdentifier string } -// NewGetNodeFromDBErr will create a new instance of GetNodeFromDBErr -func NewGetNodeFromDBErr(key []byte, err error, id string) *GetNodeFromDBErr { - return &GetNodeFromDBErr{ +// NewGetNodeFromDBErrWithKey will create a new instance of GetNodeFromDBErr +func NewGetNodeFromDBErrWithKey(key []byte, err error, id string) *GetNodeFromDBErrWithKey { + return &GetNodeFromDBErrWithKey{ getErr: err, key: key, dbIdentifier: id, @@ -42,7 +42,7 @@ func NewGetNodeFromDBErr(key []byte, err error, id string) *GetNodeFromDBErr { } // Error returns the error as string -func (e *GetNodeFromDBErr) Error() string { +func (e *GetNodeFromDBErrWithKey) Error() string { return fmt.Sprintf( "%s: %s for key %v", common.GetNodeFromDBErrorString, @@ -52,11 +52,11 @@ func (e *GetNodeFromDBErr) Error() string { } // GetKey will return the key that generated the error -func (e *GetNodeFromDBErr) GetKey() []byte { +func (e *GetNodeFromDBErrWithKey) GetKey() []byte { return e.key } -// GetIdentifier will return the db corresponding to the db -func (e *GetNodeFromDBErr) GetIdentifier() string { +// GetIdentifier will return the db identifier corresponding to the db +func (e *GetNodeFromDBErrWithKey) GetIdentifier() string { return e.dbIdentifier } diff --git a/process/sync/metablock.go b/process/sync/metablock.go index 6fb6f195e10..4d46c8fd1be 100644 --- a/process/sync/metablock.go +++ b/process/sync/metablock.go @@ -182,7 +182,7 @@ func (boot *MetaBootstrap) setLastEpochStartRound() { func (boot *MetaBootstrap) SyncBlock(ctx context.Context) error { err := boot.syncBlock() if errors.IsGetNodeFromDBError(err) { - getNodeErr, ok := err.(*errors.GetNodeFromDBErr) + getNodeErr, ok := err.(*errors.GetNodeFromDBErrWithKey) if !ok { return err } @@ -195,6 +195,7 @@ func (boot *MetaBootstrap) SyncBlock(ctx context.Context) error { } func (boot *MetaBootstrap) syncAccountsDBs(key []byte, id string) error { + // TODO: refactor this in order to avoid treatment based on identifier switch id { case common.AccountsTrieIdentifier: return boot.syncUserAccountsState(key) diff --git a/process/sync/metablock_test.go b/process/sync/metablock_test.go index 3bb64c69a7a..c997b55a602 100644 --- a/process/sync/metablock_test.go +++ b/process/sync/metablock_test.go @@ -1622,7 +1622,7 @@ func TestMetaBootstrap_SyncBlockErrGetNodeDBShouldSyncAccounts(t *testing.T) { } args.ChainHandler = blkc - errGetNodeFromDB := commonErrors.NewGetNodeFromDBErr([]byte("key"), errors.New("get error"), common.AccountsTrieIdentifier) + errGetNodeFromDB := commonErrors.NewGetNodeFromDBErrWithKey([]byte("key"), errors.New("get error"), common.AccountsTrieIdentifier) blockProcessor := createMetaBlockProcessor(args.ChainHandler) blockProcessor.ProcessBlockCalled = func(header data.HeaderHandler, body data.BodyHandler, haveTime func() time.Duration) error { return errGetNodeFromDB diff --git a/process/sync/shardblock.go b/process/sync/shardblock.go index 204718a6b9c..d39572b221a 100644 --- a/process/sync/shardblock.go +++ b/process/sync/shardblock.go @@ -144,7 +144,7 @@ func (boot *ShardBootstrap) StartSyncingBlocks() { func (boot *ShardBootstrap) SyncBlock(ctx context.Context) error { err := boot.syncBlock() if errors.IsGetNodeFromDBError(err) { - getNodeErr, ok := err.(*errors.GetNodeFromDBErr) + getNodeErr, ok := err.(*errors.GetNodeFromDBErrWithKey) if !ok { return err } diff --git a/process/sync/shardblock_test.go b/process/sync/shardblock_test.go index 000134eeb5b..8cb5f66d8ae 100644 --- a/process/sync/shardblock_test.go +++ b/process/sync/shardblock_test.go @@ -2062,7 +2062,7 @@ func TestShardBootstrap_SyncBlockGetNodeDBErrorShouldSync(t *testing.T) { } args.ChainHandler = blkc - errGetNodeFromDB := commonErrors.NewGetNodeFromDBErr([]byte("key"), errors.New("get error"), "") + errGetNodeFromDB := commonErrors.NewGetNodeFromDBErrWithKey([]byte("key"), errors.New("get error"), "") blockProcessor := createBlockProcessor(args.ChainHandler) blockProcessor.ProcessBlockCalled = func(header data.HeaderHandler, body data.BodyHandler, haveTime func() time.Duration) error { return errGetNodeFromDB From 482aecbc61d13e0bc30b1e05ff8406d4c635c972 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Thu, 15 Dec 2022 17:12:21 +0200 Subject: [PATCH 14/33] renamed error with key in trie node --- trie/node.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trie/node.go b/trie/node.go index 55735146775..f56a733e8ae 100644 --- a/trie/node.go +++ b/trie/node.go @@ -120,11 +120,11 @@ func getNodeFromDBAndDecode(n []byte, db common.DBWriteCacher, marshalizer marsh dbWithID, ok := db.(dbWriteCacherWithIdentifier) if !ok { log.Warn("wrong type assertion on", common.GetNodeFromDBErrorString, "error", err, "key", n) - return nil, errors.NewGetNodeFromDBErr(n, err, "") + return nil, errors.NewGetNodeFromDBErrWithKey(n, err, "") } log.Trace(common.GetNodeFromDBErrorString, "error", err, "key", n) - return nil, errors.NewGetNodeFromDBErr(n, err, dbWithID.GetIdentifier()) + return nil, errors.NewGetNodeFromDBErrWithKey(n, err, dbWithID.GetIdentifier()) } return decodeNode(encChild, marshalizer, hasher) From 737c93ed4ffd0eff8c0f596a5e54260a19b1fcf0 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Mon, 19 Dec 2022 16:18:22 +0200 Subject: [PATCH 15/33] add interface for db error with key --- process/sync/interface.go | 6 ++++++ process/sync/metablock.go | 2 +- process/sync/shardblock.go | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/process/sync/interface.go b/process/sync/interface.go index 7536f132e53..d9b2df014d0 100644 --- a/process/sync/interface.go +++ b/process/sync/interface.go @@ -29,3 +29,9 @@ type syncStarter interface { type forkDetector interface { computeFinalCheckpoint() } + +// getKeyHandler defines the behaviour of a component that can provide a trie node key and identifier +type getKeyHandler interface { + GetKey() []byte + GetIdentifier() string +} diff --git a/process/sync/metablock.go b/process/sync/metablock.go index 4d46c8fd1be..3bce4846d76 100644 --- a/process/sync/metablock.go +++ b/process/sync/metablock.go @@ -182,7 +182,7 @@ func (boot *MetaBootstrap) setLastEpochStartRound() { func (boot *MetaBootstrap) SyncBlock(ctx context.Context) error { err := boot.syncBlock() if errors.IsGetNodeFromDBError(err) { - getNodeErr, ok := err.(*errors.GetNodeFromDBErrWithKey) + getNodeErr, ok := err.(getKeyHandler) if !ok { return err } diff --git a/process/sync/shardblock.go b/process/sync/shardblock.go index d39572b221a..ddc9f524752 100644 --- a/process/sync/shardblock.go +++ b/process/sync/shardblock.go @@ -144,7 +144,7 @@ func (boot *ShardBootstrap) StartSyncingBlocks() { func (boot *ShardBootstrap) SyncBlock(ctx context.Context) error { err := boot.syncBlock() if errors.IsGetNodeFromDBError(err) { - getNodeErr, ok := err.(*errors.GetNodeFromDBErrWithKey) + getNodeErr, ok := err.(getKeyHandler) if !ok { return err } From 5f852a346dcec93c385aa2f6f10deee9464d62fb Mon Sep 17 00:00:00 2001 From: ssd04 Date: Mon, 19 Dec 2022 18:10:48 +0200 Subject: [PATCH 16/33] added todo for db identifier constants refactoring --- trie/factory/trieFactoryArgs.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/trie/factory/trieFactoryArgs.go b/trie/factory/trieFactoryArgs.go index 602feb8754b..06e6a011c67 100644 --- a/trie/factory/trieFactoryArgs.go +++ b/trie/factory/trieFactoryArgs.go @@ -7,6 +7,8 @@ import ( "github.com/ElrondNetwork/elrond-go/storage" ) +// TODO: refactor to align these constants with db filepath identifier + // UserAccountTrie represents the use account identifier const UserAccountTrie = "userAccount" From 349d22adad7ed5893eccb0658262a46529d0511f Mon Sep 17 00:00:00 2001 From: ssd04 Date: Thu, 22 Dec 2022 14:05:02 +0200 Subject: [PATCH 17/33] refactor to use storer ids for dataRetriever --- common/constants.go | 6 ---- .../metaResolversContainerFactory.go | 7 ++--- .../metaResolversContainerFactory_test.go | 5 ++-- .../shardResolversContainerFactory.go | 3 +- .../shardResolversContainerFactory_test.go | 5 ++-- epochStart/bootstrap/process.go | 4 +-- factory/consensus/consensusComponents.go | 6 ++-- factory/state/stateComponents.go | 4 +-- genesis/process/genesisBlockCreator.go | 3 +- genesis/process/genesisBlockCreator_test.go | 7 ++--- .../state/stateTrie/stateTrie_test.go | 8 ++--- .../state/stateTrieSync/stateTrieSync_test.go | 24 +++++++-------- integrationTests/testProcessorNode.go | 23 +++++++-------- node/nodeRunner.go | 5 ++-- process/sync/baseSync.go | 29 +++++++++++++++++++ process/sync/interface.go | 4 +++ process/sync/metablock.go | 11 +++++-- process/sync/shardblock.go | 5 ++++ testscommon/components/components.go | 10 +++---- testscommon/components/default.go | 8 ++--- trie/factory/trieCreator.go | 8 ++--- trie/factory/trieFactoryArgs.go | 8 ----- update/genesis/import.go | 8 ++--- update/genesis/import_test.go | 10 +++---- 24 files changed, 116 insertions(+), 95 deletions(-) diff --git a/common/constants.go b/common/constants.go index 3736867b83f..4aa4033eaa0 100644 --- a/common/constants.go +++ b/common/constants.go @@ -834,9 +834,3 @@ const MetricTrieSyncNumReceivedBytes = "erd_trie_sync_num_bytes_received" // MetricTrieSyncNumProcessedNodes is the metric that outputs the number of trie nodes processed for accounts during trie sync const MetricTrieSyncNumProcessedNodes = "erd_trie_sync_num_nodes_processed" - -// AccountsTrieIdentifier defines the identifier for accounts trie storer -const AccountsTrieIdentifier = "AccountsTrie" - -// PeerAccountsTrieIdentifier defines the identifier for peer accounts storer -const PeerAccountsTrieIdentifier = "PeerAccountsTrie" diff --git a/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory.go b/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory.go index e08ac70fb93..5928e7b6a8c 100644 --- a/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory.go +++ b/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory.go @@ -9,7 +9,6 @@ import ( "github.com/ElrondNetwork/elrond-go/dataRetriever" "github.com/ElrondNetwork/elrond-go/dataRetriever/factory/containers" "github.com/ElrondNetwork/elrond-go/dataRetriever/resolvers" - triesFactory "github.com/ElrondNetwork/elrond-go/trie/factory" "github.com/ElrondNetwork/elrond-go-core/marshal" "github.com/ElrondNetwork/elrond-go/process/factory" @@ -151,7 +150,7 @@ func (mrcf *metaResolversContainerFactory) AddShardTrieNodeResolvers(container d identifierTrieNodes := factory.AccountTrieNodesTopic + shardC.CommunicationIdentifier(idx) resolver, err := mrcf.createTrieNodesResolver( identifierTrieNodes, - triesFactory.UserAccountTrie, + dataRetriever.UserAccountsUnit.String(), mrcf.numCrossShardPeers, mrcf.numTotalPeers-mrcf.numCrossShardPeers, idx, @@ -310,7 +309,7 @@ func (mrcf *metaResolversContainerFactory) generateTrieNodesResolvers() error { identifierTrieNodes := factory.AccountTrieNodesTopic + core.CommunicationIdentifierBetweenShards(core.MetachainShardId, core.MetachainShardId) resolver, err := mrcf.createTrieNodesResolver( identifierTrieNodes, - triesFactory.UserAccountTrie, + dataRetriever.UserAccountsUnit.String(), 0, mrcf.numTotalPeers, core.MetachainShardId, @@ -325,7 +324,7 @@ func (mrcf *metaResolversContainerFactory) generateTrieNodesResolvers() error { identifierTrieNodes = factory.ValidatorTrieNodesTopic + core.CommunicationIdentifierBetweenShards(core.MetachainShardId, core.MetachainShardId) resolver, err = mrcf.createTrieNodesResolver( identifierTrieNodes, - triesFactory.PeerAccountTrie, + dataRetriever.PeerAccountsUnit.String(), 0, mrcf.numTotalPeers, core.MetachainShardId, diff --git a/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go b/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go index 017d107f291..13f862349bc 100644 --- a/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go +++ b/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go @@ -20,7 +20,6 @@ import ( "github.com/ElrondNetwork/elrond-go/testscommon/p2pmocks" storageStubs "github.com/ElrondNetwork/elrond-go/testscommon/storage" trieMock "github.com/ElrondNetwork/elrond-go/testscommon/trie" - triesFactory "github.com/ElrondNetwork/elrond-go/trie/factory" "github.com/stretchr/testify/assert" ) @@ -84,8 +83,8 @@ func createStoreForMeta() dataRetriever.StorageService { func createTriesHolderForMeta() common.TriesHolder { triesHolder := state.NewDataTriesHolder() - triesHolder.Put([]byte(triesFactory.UserAccountTrie), &trieMock.TrieStub{}) - triesHolder.Put([]byte(triesFactory.PeerAccountTrie), &trieMock.TrieStub{}) + triesHolder.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{}) + triesHolder.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{}) return triesHolder } diff --git a/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory.go b/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory.go index 01e3310a1f4..3136147797e 100644 --- a/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory.go +++ b/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory.go @@ -10,7 +10,6 @@ import ( "github.com/ElrondNetwork/elrond-go/dataRetriever/factory/containers" "github.com/ElrondNetwork/elrond-go/dataRetriever/resolvers" "github.com/ElrondNetwork/elrond-go/process/factory" - triesFactory "github.com/ElrondNetwork/elrond-go/trie/factory" ) var _ dataRetriever.ResolversContainerFactory = (*shardResolversContainerFactory)(nil) @@ -241,7 +240,7 @@ func (srcf *shardResolversContainerFactory) generateTrieNodesResolvers() error { identifierTrieNodes := factory.AccountTrieNodesTopic + shardC.CommunicationIdentifier(core.MetachainShardId) resolver, err := srcf.createTrieNodesResolver( identifierTrieNodes, - triesFactory.UserAccountTrie, + dataRetriever.UserAccountsUnit.String(), 0, srcf.numTotalPeers, core.MetachainShardId, diff --git a/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go b/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go index 6b731d36d0b..5a793430bd2 100644 --- a/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go +++ b/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go @@ -19,7 +19,6 @@ import ( "github.com/ElrondNetwork/elrond-go/testscommon/p2pmocks" storageStubs "github.com/ElrondNetwork/elrond-go/testscommon/storage" trieMock "github.com/ElrondNetwork/elrond-go/testscommon/trie" - triesFactory "github.com/ElrondNetwork/elrond-go/trie/factory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -90,8 +89,8 @@ func createStoreForShard() dataRetriever.StorageService { func createTriesHolderForShard() common.TriesHolder { triesHolder := state.NewDataTriesHolder() - triesHolder.Put([]byte(triesFactory.UserAccountTrie), &trieMock.TrieStub{}) - triesHolder.Put([]byte(triesFactory.PeerAccountTrie), &trieMock.TrieStub{}) + triesHolder.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{}) + triesHolder.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{}) return triesHolder } diff --git a/epochStart/bootstrap/process.go b/epochStart/bootstrap/process.go index bcdd9638ffb..0dcee5c4134 100644 --- a/epochStart/bootstrap/process.go +++ b/epochStart/bootstrap/process.go @@ -1046,7 +1046,7 @@ func (e *epochStartBootstrap) syncUserAccountsState(rootHash []byte) error { } e.mutTrieStorageManagers.RLock() - trieStorageManager := e.trieStorageManagers[factory.UserAccountTrie] + trieStorageManager := e.trieStorageManagers[dataRetriever.UserAccountsUnit.String()] e.mutTrieStorageManagers.RUnlock() argsUserAccountsSyncer := syncer.ArgsNewUserAccountsSyncer{ @@ -1115,7 +1115,7 @@ func (e *epochStartBootstrap) createStorageService( func (e *epochStartBootstrap) syncValidatorAccountsState(rootHash []byte) error { e.mutTrieStorageManagers.RLock() - peerTrieStorageManager := e.trieStorageManagers[factory.PeerAccountTrie] + peerTrieStorageManager := e.trieStorageManagers[dataRetriever.PeerAccountsUnit.String()] e.mutTrieStorageManagers.RUnlock() argsValidatorAccountsSyncer := syncer.ArgsNewValidatorAccountsSyncer{ diff --git a/factory/consensus/consensusComponents.go b/factory/consensus/consensusComponents.go index ef2c824d08f..ae69175bb29 100644 --- a/factory/consensus/consensusComponents.go +++ b/factory/consensus/consensusComponents.go @@ -17,6 +17,7 @@ import ( "github.com/ElrondNetwork/elrond-go/consensus/signing" "github.com/ElrondNetwork/elrond-go/consensus/spos" "github.com/ElrondNetwork/elrond-go/consensus/spos/sposFactory" + "github.com/ElrondNetwork/elrond-go/dataRetriever" "github.com/ElrondNetwork/elrond-go/errors" "github.com/ElrondNetwork/elrond-go/factory" "github.com/ElrondNetwork/elrond-go/process" @@ -24,7 +25,6 @@ import ( "github.com/ElrondNetwork/elrond-go/process/sync/storageBootstrap" "github.com/ElrondNetwork/elrond-go/sharding" "github.com/ElrondNetwork/elrond-go/state/syncer" - trieFactory "github.com/ElrondNetwork/elrond-go/trie/factory" "github.com/ElrondNetwork/elrond-go/trie/statistics" "github.com/ElrondNetwork/elrond-go/trie/storageMarker" "github.com/ElrondNetwork/elrond-go/update" @@ -516,7 +516,7 @@ func (ccf *consensusComponentsFactory) createArgsBaseAccountsSyncer(trieStorageM } func (ccf *consensusComponentsFactory) createValidatorAccountsSyncer() (process.AccountsDBSyncer, error) { - trieStorageManager, ok := ccf.stateComponents.TrieStorageManagers()[trieFactory.PeerAccountTrie] + trieStorageManager, ok := ccf.stateComponents.TrieStorageManagers()[dataRetriever.PeerAccountsUnit.String()] if !ok { return nil, errors.ErrNilTrieStorageManager } @@ -528,7 +528,7 @@ func (ccf *consensusComponentsFactory) createValidatorAccountsSyncer() (process. } func (ccf *consensusComponentsFactory) createUserAccountsSyncer() (process.AccountsDBSyncer, error) { - trieStorageManager, ok := ccf.stateComponents.TrieStorageManagers()[trieFactory.UserAccountTrie] + trieStorageManager, ok := ccf.stateComponents.TrieStorageManagers()[dataRetriever.UserAccountsUnit.String()] if !ok { return nil, errors.ErrNilTrieStorageManager } diff --git a/factory/state/stateComponents.go b/factory/state/stateComponents.go index 5f17b621ee4..169fccf8f89 100644 --- a/factory/state/stateComponents.go +++ b/factory/state/stateComponents.go @@ -128,7 +128,7 @@ func (scf *stateComponentsFactory) Create() (*stateComponents, error) { func (scf *stateComponentsFactory) createAccountsAdapters(triesContainer common.TriesHolder) (state.AccountsAdapter, state.AccountsAdapter, state.AccountsRepository, error) { accountFactory := factoryState.NewAccountCreator() - merkleTrie := triesContainer.Get([]byte(trieFactory.UserAccountTrie)) + merkleTrie := triesContainer.Get([]byte(dataRetriever.UserAccountsUnit.String())) storagePruning, err := scf.newStoragePruningManager() if err != nil { return nil, nil, nil, err @@ -192,7 +192,7 @@ func (scf *stateComponentsFactory) createAccountsAdapters(triesContainer common. func (scf *stateComponentsFactory) createPeerAdapter(triesContainer common.TriesHolder) (state.AccountsAdapter, error) { accountFactory := factoryState.NewPeerAccountCreator() - merkleTrie := triesContainer.Get([]byte(trieFactory.PeerAccountTrie)) + merkleTrie := triesContainer.Get([]byte(dataRetriever.PeerAccountsUnit.String())) storagePruning, err := scf.newStoragePruningManager() if err != nil { return nil, err diff --git a/genesis/process/genesisBlockCreator.go b/genesis/process/genesisBlockCreator.go index 629177b235b..1a8c945b1d4 100644 --- a/genesis/process/genesisBlockCreator.go +++ b/genesis/process/genesisBlockCreator.go @@ -27,7 +27,6 @@ import ( "github.com/ElrondNetwork/elrond-go/storage" "github.com/ElrondNetwork/elrond-go/storage/factory" "github.com/ElrondNetwork/elrond-go/storage/storageunit" - triesFactory "github.com/ElrondNetwork/elrond-go/trie/factory" "github.com/ElrondNetwork/elrond-go/update" hardfork "github.com/ElrondNetwork/elrond-go/update/genesis" hardForkProcess "github.com/ElrondNetwork/elrond-go/update/process" @@ -481,7 +480,7 @@ func (gbc *genesisBlockCreator) getNewArgForShard(shardID uint32) (ArgsGenesisBl newArgument.Core.InternalMarshalizer(), newArgument.Core.Hasher(), factoryState.NewAccountCreator(), - gbc.arg.TrieStorageManagers[triesFactory.UserAccountTrie], + gbc.arg.TrieStorageManagers[dataRetriever.UserAccountsUnit.String()], ) if err != nil { return ArgsGenesisBlockCreator{}, fmt.Errorf("'%w' while generating an in-memory accounts adapter for shard %d", diff --git a/genesis/process/genesisBlockCreator_test.go b/genesis/process/genesisBlockCreator_test.go index b5df301bfcb..bbdf1031bde 100644 --- a/genesis/process/genesisBlockCreator_test.go +++ b/genesis/process/genesisBlockCreator_test.go @@ -28,7 +28,6 @@ import ( stateMock "github.com/ElrondNetwork/elrond-go/testscommon/state" storageCommon "github.com/ElrondNetwork/elrond-go/testscommon/storage" "github.com/ElrondNetwork/elrond-go/trie" - "github.com/ElrondNetwork/elrond-go/trie/factory" "github.com/ElrondNetwork/elrond-go/update" updateMock "github.com/ElrondNetwork/elrond-go/update/mock" "github.com/ElrondNetwork/elrond-go/vm/systemSmartContracts/defaults" @@ -52,8 +51,8 @@ func createMockArgument( storageManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, options) trieStorageManagers := make(map[string]common.StorageManager) - trieStorageManagers[factory.UserAccountTrie] = storageManager - trieStorageManagers[factory.PeerAccountTrie] = storageManager + trieStorageManagers[dataRetriever.UserAccountsUnit.String()] = storageManager + trieStorageManagers[dataRetriever.PeerAccountsUnit.String()] = storageManager arg := ArgsGenesisBlockCreator{ GenesisTime: 0, @@ -146,7 +145,7 @@ func createMockArgument( &mock.MarshalizerMock{}, &hashingMocks.HasherMock{}, factoryState.NewAccountCreator(), - trieStorageManagers[factory.UserAccountTrie], + trieStorageManagers[dataRetriever.UserAccountsUnit.String()], ) require.Nil(t, err) diff --git a/integrationTests/state/stateTrie/stateTrie_test.go b/integrationTests/state/stateTrie/stateTrie_test.go index 11a0c8fa88a..3c7ec355a1f 100644 --- a/integrationTests/state/stateTrie/stateTrie_test.go +++ b/integrationTests/state/stateTrie/stateTrie_test.go @@ -25,6 +25,7 @@ import ( crypto "github.com/ElrondNetwork/elrond-go-crypto" "github.com/ElrondNetwork/elrond-go/common" "github.com/ElrondNetwork/elrond-go/config" + "github.com/ElrondNetwork/elrond-go/dataRetriever" "github.com/ElrondNetwork/elrond-go/integrationTests" "github.com/ElrondNetwork/elrond-go/integrationTests/mock" "github.com/ElrondNetwork/elrond-go/sharding" @@ -39,7 +40,6 @@ import ( "github.com/ElrondNetwork/elrond-go/testscommon/statusHandler" trieMock "github.com/ElrondNetwork/elrond-go/testscommon/trie" "github.com/ElrondNetwork/elrond-go/trie" - trieFactory "github.com/ElrondNetwork/elrond-go/trie/factory" vmcommon "github.com/ElrondNetwork/elrond-vm-common" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -1574,7 +1574,7 @@ func TestStatePruningIsBuffered(t *testing.T) { round, nonce = integrationTests.ProposeAndSyncOneBlock(t, nodes, idxProposers, round, nonce) rootHash := shardNode.BlockChain.GetCurrentBlockHeader().GetRootHash() - stateTrie := shardNode.TrieContainer.Get([]byte(trieFactory.UserAccountTrie)) + stateTrie := shardNode.TrieContainer.Get([]byte(dataRetriever.UserAccountsUnit.String())) delayRounds := 10 for i := 0; i < delayRounds; i++ { @@ -1764,7 +1764,7 @@ func testNodeStateCheckpointSnapshotAndPruning( prunedRootHashes [][]byte, ) { - stateTrie := node.TrieContainer.Get([]byte(trieFactory.UserAccountTrie)) + stateTrie := node.TrieContainer.Get([]byte(dataRetriever.UserAccountsUnit.String())) assert.Equal(t, 6, len(checkpointsRootHashes)) for i := range checkpointsRootHashes { tr, err := stateTrie.Recreate(checkpointsRootHashes[i]) @@ -1945,7 +1945,7 @@ func checkCodeConsistency( ) { for code := range codeMap { codeHash := integrationTests.TestHasher.Compute(code) - tr := shardNode.TrieContainer.Get([]byte(trieFactory.UserAccountTrie)) + tr := shardNode.TrieContainer.Get([]byte(dataRetriever.UserAccountsUnit.String())) if codeMap[code] != 0 { val, _, err := tr.Get(codeHash) diff --git a/integrationTests/state/stateTrieSync/stateTrieSync_test.go b/integrationTests/state/stateTrieSync/stateTrieSync_test.go index 06ecbd48a89..fbdf285e769 100644 --- a/integrationTests/state/stateTrieSync/stateTrieSync_test.go +++ b/integrationTests/state/stateTrieSync/stateTrieSync_test.go @@ -12,6 +12,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core/throttler" logger "github.com/ElrondNetwork/elrond-go-logger" "github.com/ElrondNetwork/elrond-go/common" + "github.com/ElrondNetwork/elrond-go/dataRetriever" "github.com/ElrondNetwork/elrond-go/epochStart/notifier" "github.com/ElrondNetwork/elrond-go/integrationTests" "github.com/ElrondNetwork/elrond-go/process/factory" @@ -21,7 +22,6 @@ import ( "github.com/ElrondNetwork/elrond-go/testscommon" testStorage "github.com/ElrondNetwork/elrond-go/testscommon/state" "github.com/ElrondNetwork/elrond-go/trie" - trieFactory "github.com/ElrondNetwork/elrond-go/trie/factory" "github.com/ElrondNetwork/elrond-go/trie/keyBuilder" "github.com/ElrondNetwork/elrond-go/trie/statistics" "github.com/ElrondNetwork/elrond-go/trie/storageMarker" @@ -94,7 +94,7 @@ func testNodeRequestInterceptTrieNodesWithMessenger(t *testing.T, version int) { time.Sleep(integrationTests.SyncDelay) - resolverTrie := nResolver.TrieContainer.Get([]byte(trieFactory.UserAccountTrie)) + resolverTrie := nResolver.TrieContainer.Get([]byte(dataRetriever.UserAccountsUnit.String())) // we have tested even with the 1000000 value and found out that it worked in a reasonable amount of time ~3.5 minutes numTrieLeaves := 10000 for i := 0; i < numTrieLeaves; i++ { @@ -116,7 +116,7 @@ func testNodeRequestInterceptTrieNodesWithMessenger(t *testing.T, version int) { numLeaves := getNumLeaves(t, resolverTrie, rootHash) assert.Equal(t, numTrieLeaves, numLeaves) - requesterTrie := nRequester.TrieContainer.Get([]byte(trieFactory.UserAccountTrie)) + requesterTrie := nRequester.TrieContainer.Get([]byte(dataRetriever.UserAccountsUnit.String())) nilRootHash, _ := requesterTrie.RootHash() tss := statistics.NewTrieSyncStatistics() @@ -224,7 +224,7 @@ func testNodeRequestInterceptTrieNodesWithMessengerNotSyncingShouldErr(t *testin time.Sleep(integrationTests.SyncDelay) - resolverTrie := nResolver.TrieContainer.Get([]byte(trieFactory.UserAccountTrie)) + resolverTrie := nResolver.TrieContainer.Get([]byte(dataRetriever.UserAccountsUnit.String())) // we have tested even with the 1000000 value and found out that it worked in a reasonable amount of time ~3.5 minutes numTrieLeaves := 10000 for i := 0; i < numTrieLeaves; i++ { @@ -246,7 +246,7 @@ func testNodeRequestInterceptTrieNodesWithMessengerNotSyncingShouldErr(t *testin numLeaves := getNumLeaves(t, resolverTrie, rootHash) assert.Equal(t, numTrieLeaves, numLeaves) - requesterTrie := nRequester.TrieContainer.Get([]byte(trieFactory.UserAccountTrie)) + requesterTrie := nRequester.TrieContainer.Get([]byte(dataRetriever.UserAccountsUnit.String())) tss := statistics.NewTrieSyncStatistics() arg := trie.ArgTrieSyncer{ @@ -356,7 +356,7 @@ func testMultipleDataTriesSync(t *testing.T, numAccounts int, numDataTrieLeaves err = common.GetErrorFromChanNonBlocking(leavesChannel.ErrChan) require.Nil(t, err) - requesterTrie := nRequester.TrieContainer.Get([]byte(trieFactory.UserAccountTrie)) + requesterTrie := nRequester.TrieContainer.Get([]byte(dataRetriever.UserAccountsUnit.String())) nilRootHash, _ := requesterTrie.RootHash() syncerArgs := getUserAccountSyncerArgs(nRequester, version) @@ -483,14 +483,14 @@ func testSyncMissingSnapshotNodes(t *testing.T, version int) { time.Sleep(integrationTests.StepDelay) } - resolverTrie := nResolver.TrieContainer.Get([]byte(trieFactory.UserAccountTrie)) + resolverTrie := nResolver.TrieContainer.Get([]byte(dataRetriever.UserAccountsUnit.String())) accState := nResolver.AccntState dataTrieRootHashes := addAccountsToState(t, numAccounts, numDataTrieLeaves, accState, valSize) rootHash, _ := accState.RootHash() numLeaves := getNumLeaves(t, resolverTrie, rootHash) require.Equal(t, numAccounts+numSystemAccounts, numLeaves) - requesterTrie := nRequester.TrieContainer.Get([]byte(trieFactory.UserAccountTrie)) + requesterTrie := nRequester.TrieContainer.Get([]byte(dataRetriever.UserAccountsUnit.String())) nilRootHash, _ := requesterTrie.RootHash() copyPartialState(t, nResolver, nRequester, dataTrieRootHashes) @@ -504,7 +504,7 @@ func testSyncMissingSnapshotNodes(t *testing.T, version int) { err = nRequester.AccntState.StartSnapshotIfNeeded() assert.Nil(t, err) - tsm := nRequester.TrieStorageManagers[trieFactory.UserAccountTrie] + tsm := nRequester.TrieStorageManagers[dataRetriever.UserAccountsUnit.String()] _ = tsm.PutInEpoch([]byte(common.ActiveDBKey), []byte(common.ActiveDBVal), 0) nRequester.AccntState.SnapshotState(rootHash) for tsm.IsPruningBlocked() { @@ -522,12 +522,12 @@ func testSyncMissingSnapshotNodes(t *testing.T, version int) { } func copyPartialState(t *testing.T, sourceNode, destinationNode *integrationTests.TestProcessorNode, dataTriesRootHashes [][]byte) { - resolverTrie := sourceNode.TrieContainer.Get([]byte(trieFactory.UserAccountTrie)) + resolverTrie := sourceNode.TrieContainer.Get([]byte(dataRetriever.UserAccountsUnit.String())) hashes, _ := resolverTrie.GetAllHashes() assert.NotEqual(t, 0, len(hashes)) hashes = append(hashes, getDataTriesHashes(t, resolverTrie, dataTriesRootHashes)...) - destStorage := destinationNode.TrieContainer.Get([]byte(trieFactory.UserAccountTrie)).GetStorageManager() + destStorage := destinationNode.TrieContainer.Get([]byte(dataRetriever.UserAccountsUnit.String())).GetStorageManager() for i, hash := range hashes { if i%1000 == 0 { @@ -599,7 +599,7 @@ func getUserAccountSyncerArgs(node *integrationTests.TestProcessorNode, version ArgsNewBaseAccountsSyncer: syncer.ArgsNewBaseAccountsSyncer{ Hasher: integrationTests.TestHasher, Marshalizer: integrationTests.TestMarshalizer, - TrieStorageManager: node.TrieStorageManagers[trieFactory.UserAccountTrie], + TrieStorageManager: node.TrieStorageManagers[dataRetriever.UserAccountsUnit.String()], RequestHandler: node.RequestHandler, Timeout: common.TimeoutGettingTrieNodes, Cacher: node.DataPool.TrieNodes(), diff --git a/integrationTests/testProcessorNode.go b/integrationTests/testProcessorNode.go index fc622efc15f..61fafaa609b 100644 --- a/integrationTests/testProcessorNode.go +++ b/integrationTests/testProcessorNode.go @@ -110,7 +110,6 @@ import ( statusHandlerMock "github.com/ElrondNetwork/elrond-go/testscommon/statusHandler" storageStubs "github.com/ElrondNetwork/elrond-go/testscommon/storage" trieMock "github.com/ElrondNetwork/elrond-go/testscommon/trie" - trieFactory "github.com/ElrondNetwork/elrond-go/trie/factory" "github.com/ElrondNetwork/elrond-go/trie/keyBuilder" "github.com/ElrondNetwork/elrond-go/update" "github.com/ElrondNetwork/elrond-go/update/trigger" @@ -525,15 +524,15 @@ func (tpn *TestProcessorNode) initAccountDBsWithPruningStorer() { tpn.TrieContainer = state.NewDataTriesHolder() var stateTrie common.Trie tpn.AccntState, stateTrie = CreateAccountsDB(UserAccount, trieStorageManager) - tpn.TrieContainer.Put([]byte(trieFactory.UserAccountTrie), stateTrie) + tpn.TrieContainer.Put([]byte(dataRetriever.UserAccountsUnit.String()), stateTrie) var peerTrie common.Trie tpn.PeerState, peerTrie = CreateAccountsDB(ValidatorAccount, trieStorageManager) - tpn.TrieContainer.Put([]byte(trieFactory.PeerAccountTrie), peerTrie) + tpn.TrieContainer.Put([]byte(dataRetriever.PeerAccountsUnit.String()), peerTrie) tpn.TrieStorageManagers = make(map[string]common.StorageManager) - tpn.TrieStorageManagers[trieFactory.UserAccountTrie] = trieStorageManager - tpn.TrieStorageManagers[trieFactory.PeerAccountTrie] = trieStorageManager + tpn.TrieStorageManagers[dataRetriever.UserAccountsUnit.String()] = trieStorageManager + tpn.TrieStorageManagers[dataRetriever.PeerAccountsUnit.String()] = trieStorageManager } func (tpn *TestProcessorNode) initAccountDBs(store storage.Storer) { @@ -541,15 +540,15 @@ func (tpn *TestProcessorNode) initAccountDBs(store storage.Storer) { tpn.TrieContainer = state.NewDataTriesHolder() var stateTrie common.Trie tpn.AccntState, stateTrie = CreateAccountsDB(UserAccount, trieStorageManager) - tpn.TrieContainer.Put([]byte(trieFactory.UserAccountTrie), stateTrie) + tpn.TrieContainer.Put([]byte(dataRetriever.UserAccountsUnit.String()), stateTrie) var peerTrie common.Trie tpn.PeerState, peerTrie = CreateAccountsDB(ValidatorAccount, trieStorageManager) - tpn.TrieContainer.Put([]byte(trieFactory.PeerAccountTrie), peerTrie) + tpn.TrieContainer.Put([]byte(dataRetriever.PeerAccountsUnit.String()), peerTrie) tpn.TrieStorageManagers = make(map[string]common.StorageManager) - tpn.TrieStorageManagers[trieFactory.UserAccountTrie] = trieStorageManager - tpn.TrieStorageManagers[trieFactory.PeerAccountTrie] = trieStorageManager + tpn.TrieStorageManagers[dataRetriever.UserAccountsUnit.String()] = trieStorageManager + tpn.TrieStorageManagers[dataRetriever.PeerAccountsUnit.String()] = trieStorageManager } func (tpn *TestProcessorNode) initValidatorStatistics() { @@ -3064,9 +3063,9 @@ func GetDefaultStateComponents() *testscommon.StateComponentsMock { AccountsRepo: &stateMock.AccountsRepositoryStub{}, Tries: &trieMock.TriesHolderStub{}, StorageManagers: map[string]common.StorageManager{ - "0": &testscommon.StorageManagerStub{}, - trieFactory.UserAccountTrie: &testscommon.StorageManagerStub{}, - trieFactory.PeerAccountTrie: &testscommon.StorageManagerStub{}, + "0": &testscommon.StorageManagerStub{}, + dataRetriever.UserAccountsUnit.String(): &testscommon.StorageManagerStub{}, + dataRetriever.PeerAccountsUnit.String(): &testscommon.StorageManagerStub{}, }, } } diff --git a/node/nodeRunner.go b/node/nodeRunner.go index a2729ddab5b..9f331f0ee68 100644 --- a/node/nodeRunner.go +++ b/node/nodeRunner.go @@ -59,7 +59,6 @@ import ( "github.com/ElrondNetwork/elrond-go/storage/cache" storageFactory "github.com/ElrondNetwork/elrond-go/storage/factory" "github.com/ElrondNetwork/elrond-go/storage/storageunit" - trieFactory "github.com/ElrondNetwork/elrond-go/trie/factory" trieStatistics "github.com/ElrondNetwork/elrond-go/trie/statistics" "github.com/ElrondNetwork/elrond-go/trie/storageMarker" "github.com/ElrondNetwork/elrond-go/update/trigger" @@ -600,7 +599,7 @@ func getUserAccountSyncer( processComponents mainFactory.ProcessComponentsHolder, ) (process.AccountsDBSyncer, error) { maxTrieLevelInMemory := config.StateTriesConfig.MaxStateTrieLevelInMemory - userTrie := stateComponents.TriesContainer().Get([]byte(trieFactory.UserAccountTrie)) + userTrie := stateComponents.TriesContainer().Get([]byte(dataRetriever.UserAccountsUnit.String())) storageManager := userTrie.GetStorageManager() thr, err := throttler.NewNumGoRoutinesThrottler(int32(config.TrieSync.NumConcurrentTrieSyncers)) @@ -633,7 +632,7 @@ func getValidatorAccountSyncer( processComponents mainFactory.ProcessComponentsHolder, ) (process.AccountsDBSyncer, error) { maxTrieLevelInMemory := config.StateTriesConfig.MaxPeerTrieLevelInMemory - peerTrie := stateComponents.TriesContainer().Get([]byte(trieFactory.PeerAccountTrie)) + peerTrie := stateComponents.TriesContainer().Get([]byte(dataRetriever.PeerAccountsUnit.String())) storageManager := peerTrie.GetStorageManager() args := syncer.ArgsNewValidatorAccountsSyncer{ diff --git a/process/sync/baseSync.go b/process/sync/baseSync.go index 1b07c400124..fd56a4d141e 100644 --- a/process/sync/baseSync.go +++ b/process/sync/baseSync.go @@ -21,6 +21,7 @@ import ( "github.com/ElrondNetwork/elrond-go/consensus" "github.com/ElrondNetwork/elrond-go/dataRetriever" "github.com/ElrondNetwork/elrond-go/dblookupext" + "github.com/ElrondNetwork/elrond-go/errors" "github.com/ElrondNetwork/elrond-go/outport" "github.com/ElrondNetwork/elrond-go/process" "github.com/ElrondNetwork/elrond-go/process/sync/storageBootstrap/metricsLoader" @@ -123,6 +124,9 @@ type baseBootstrap struct { isInImportMode bool scheduledTxsExecutionHandler process.ScheduledTxsExecutionHandler processWaitTime time.Duration + + userAccountsStorerIdentifier string + peerAccountsStorerIdentifier string } // setRequestedHeaderNonce method sets the header nonce requested by the sync mechanism @@ -1123,6 +1127,31 @@ func (boot *baseBootstrap) waitForMiniBlocks() error { } } +func (boot *baseBootstrap) setAccountsStorerIdentifiers() error { + userStorer, err := boot.store.GetStorer(dataRetriever.UserAccountsUnit) + if err != nil { + return err + } + dbWithID, ok := userStorer.(dbStorerWithIdentifier) + if !ok { + return errors.ErrWrongTypeAssertion + } + boot.userAccountsStorerIdentifier = dbWithID.GetIdentifier() + + peerStorer, err := boot.store.GetStorer(dataRetriever.PeerAccountsUnit) + if err != nil { + return err + } + dbPeerWithID, ok := peerStorer.(dbStorerWithIdentifier) + if !ok { + return errors.ErrWrongTypeAssertion + } + + boot.peerAccountsStorerIdentifier = dbPeerWithID.GetIdentifier() + + return nil +} + func (boot *baseBootstrap) init() { boot.forkInfo = process.NewForkInfo() diff --git a/process/sync/interface.go b/process/sync/interface.go index d9b2df014d0..f2f717a56f9 100644 --- a/process/sync/interface.go +++ b/process/sync/interface.go @@ -35,3 +35,7 @@ type getKeyHandler interface { GetKey() []byte GetIdentifier() string } + +type dbStorerWithIdentifier interface { + GetIdentifier() string +} diff --git a/process/sync/metablock.go b/process/sync/metablock.go index 3bce4846d76..915ef619445 100644 --- a/process/sync/metablock.go +++ b/process/sync/metablock.go @@ -8,7 +8,6 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core/check" "github.com/ElrondNetwork/elrond-go-core/data" "github.com/ElrondNetwork/elrond-go-core/data/block" - "github.com/ElrondNetwork/elrond-go/common" "github.com/ElrondNetwork/elrond-go/dataRetriever" "github.com/ElrondNetwork/elrond-go/errors" "github.com/ElrondNetwork/elrond-go/process" @@ -108,6 +107,11 @@ func NewMetaBootstrap(arguments ArgMetaBootstrapper) (*MetaBootstrap, error) { return nil, err } + err = base.setAccountsStorerIdentifiers() + if err != nil { + return nil, err + } + base.init() return &boot, nil @@ -195,11 +199,12 @@ func (boot *MetaBootstrap) SyncBlock(ctx context.Context) error { } func (boot *MetaBootstrap) syncAccountsDBs(key []byte, id string) error { + // TODO: refactor this in order to avoid treatment based on identifier switch id { - case common.AccountsTrieIdentifier: + case boot.userAccountsStorerIdentifier: return boot.syncUserAccountsState(key) - case common.PeerAccountsTrieIdentifier: + case boot.peerAccountsStorerIdentifier: return boot.syncValidatorAccountsState(key) default: return fmt.Errorf("invalid trie identifier, id: %s", id) diff --git a/process/sync/shardblock.go b/process/sync/shardblock.go index ddc9f524752..60fabc3e556 100644 --- a/process/sync/shardblock.go +++ b/process/sync/shardblock.go @@ -92,6 +92,11 @@ func NewShardBootstrap(arguments ArgShardBootstrapper) (*ShardBootstrap, error) return nil, err } + err = base.setAccountsStorerIdentifiers() + if err != nil { + return nil, err + } + base.init() return &boot, nil diff --git a/testscommon/components/components.go b/testscommon/components/components.go index e2333ebd84d..541cdd40981 100644 --- a/testscommon/components/components.go +++ b/testscommon/components/components.go @@ -13,6 +13,7 @@ import ( commonFactory "github.com/ElrondNetwork/elrond-go/common/factory" "github.com/ElrondNetwork/elrond-go/config" "github.com/ElrondNetwork/elrond-go/consensus/spos" + "github.com/ElrondNetwork/elrond-go/dataRetriever" "github.com/ElrondNetwork/elrond-go/epochStart/bootstrap/disabled" "github.com/ElrondNetwork/elrond-go/factory" bootstrapComp "github.com/ElrondNetwork/elrond-go/factory/bootstrap" @@ -40,7 +41,6 @@ import ( "github.com/ElrondNetwork/elrond-go/testscommon/shardingMocks" statusHandlerMock "github.com/ElrondNetwork/elrond-go/testscommon/statusHandler" "github.com/ElrondNetwork/elrond-go/trie" - trieFactory "github.com/ElrondNetwork/elrond-go/trie/factory" "github.com/ElrondNetwork/elrond-go/trie/hashesHolder" arwenConfig "github.com/ElrondNetwork/wasm-vm-v1_4/config" "github.com/stretchr/testify/require" @@ -334,14 +334,14 @@ func GetStateFactoryArgs(coreComponents factory.CoreComponentsHolder, shardCoord storageManagerPeer, _ := trie.NewTrieStorageManagerWithoutPruning(tsm) trieStorageManagers := make(map[string]common.StorageManager) - trieStorageManagers[trieFactory.UserAccountTrie] = storageManagerUser - trieStorageManagers[trieFactory.PeerAccountTrie] = storageManagerPeer + trieStorageManagers[dataRetriever.UserAccountsUnit.String()] = storageManagerUser + trieStorageManagers[dataRetriever.PeerAccountsUnit.String()] = storageManagerPeer triesHolder := state.NewDataTriesHolder() trieUsers, _ := trie.NewTrie(storageManagerUser, coreComponents.InternalMarshalizer(), coreComponents.Hasher(), 5) triePeers, _ := trie.NewTrie(storageManagerPeer, coreComponents.InternalMarshalizer(), coreComponents.Hasher(), 5) - triesHolder.Put([]byte(trieFactory.UserAccountTrie), trieUsers) - triesHolder.Put([]byte(trieFactory.PeerAccountTrie), triePeers) + triesHolder.Put([]byte(dataRetriever.UserAccountsUnit.String()), trieUsers) + triesHolder.Put([]byte(dataRetriever.PeerAccountsUnit.String()), triePeers) stateComponentsFactoryArgs := stateComp.StateComponentsFactoryArgs{ Config: GetGeneralConfig(), diff --git a/testscommon/components/default.go b/testscommon/components/default.go index 18ffb4e509b..3cf22223c77 100644 --- a/testscommon/components/default.go +++ b/testscommon/components/default.go @@ -5,6 +5,7 @@ import ( crypto "github.com/ElrondNetwork/elrond-go-crypto" "github.com/ElrondNetwork/elrond-go/common" + "github.com/ElrondNetwork/elrond-go/dataRetriever" "github.com/ElrondNetwork/elrond-go/factory/mock" "github.com/ElrondNetwork/elrond-go/sharding" "github.com/ElrondNetwork/elrond-go/testscommon" @@ -17,7 +18,6 @@ import ( stateMock "github.com/ElrondNetwork/elrond-go/testscommon/state" "github.com/ElrondNetwork/elrond-go/testscommon/storage" trieMock "github.com/ElrondNetwork/elrond-go/testscommon/trie" - trieFactory "github.com/ElrondNetwork/elrond-go/trie/factory" ) // GetDefaultCoreComponents - @@ -85,9 +85,9 @@ func GetDefaultStateComponents() *testscommon.StateComponentsMock { Accounts: &stateMock.AccountsStub{}, Tries: &trieMock.TriesHolderStub{}, StorageManagers: map[string]common.StorageManager{ - "0": &testscommon.StorageManagerStub{}, - trieFactory.UserAccountTrie: &testscommon.StorageManagerStub{}, - trieFactory.PeerAccountTrie: &testscommon.StorageManagerStub{}, + "0": &testscommon.StorageManagerStub{}, + dataRetriever.UserAccountsUnit.String(): &testscommon.StorageManagerStub{}, + dataRetriever.PeerAccountsUnit.String(): &testscommon.StorageManagerStub{}, }, } } diff --git a/trie/factory/trieCreator.go b/trie/factory/trieCreator.go index c80ff2d09ee..bdd8f9c637e 100644 --- a/trie/factory/trieCreator.go +++ b/trie/factory/trieCreator.go @@ -148,8 +148,8 @@ func CreateTriesComponentsForShardId( trieContainer := state.NewDataTriesHolder() trieStorageManagers := make(map[string]common.StorageManager) - trieContainer.Put([]byte(UserAccountTrie), userAccountTrie) - trieStorageManagers[UserAccountTrie] = userStorageManager + trieContainer.Put([]byte(dataRetriever.UserAccountsUnit.String()), userAccountTrie) + trieStorageManagers[dataRetriever.UserAccountsUnit.String()] = userStorageManager mainStorer, err = storageService.GetStorer(dataRetriever.PeerAccountsUnit) if err != nil { @@ -175,8 +175,8 @@ func CreateTriesComponentsForShardId( return nil, nil, err } - trieContainer.Put([]byte(PeerAccountTrie), peerAccountsTrie) - trieStorageManagers[PeerAccountTrie] = peerStorageManager + trieContainer.Put([]byte(dataRetriever.PeerAccountsUnit.String()), peerAccountsTrie) + trieStorageManagers[dataRetriever.PeerAccountsUnit.String()] = peerStorageManager return trieContainer, trieStorageManagers, nil } diff --git a/trie/factory/trieFactoryArgs.go b/trie/factory/trieFactoryArgs.go index 06e6a011c67..71639870cfd 100644 --- a/trie/factory/trieFactoryArgs.go +++ b/trie/factory/trieFactoryArgs.go @@ -7,14 +7,6 @@ import ( "github.com/ElrondNetwork/elrond-go/storage" ) -// TODO: refactor to align these constants with db filepath identifier - -// UserAccountTrie represents the use account identifier -const UserAccountTrie = "userAccount" - -// PeerAccountTrie represents the peer account identifier -const PeerAccountTrie = "peerAccount" - // TrieFactoryArgs holds the arguments for creating a trie factory type TrieFactoryArgs struct { Marshalizer marshal.Marshalizer diff --git a/update/genesis/import.go b/update/genesis/import.go index ab74f41070d..2c15f298a67 100644 --- a/update/genesis/import.go +++ b/update/genesis/import.go @@ -16,11 +16,11 @@ import ( "github.com/ElrondNetwork/elrond-go/common" commonDisabled "github.com/ElrondNetwork/elrond-go/common/disabled" "github.com/ElrondNetwork/elrond-go/config" + "github.com/ElrondNetwork/elrond-go/dataRetriever" "github.com/ElrondNetwork/elrond-go/state" "github.com/ElrondNetwork/elrond-go/state/factory" "github.com/ElrondNetwork/elrond-go/state/storagePruningManager/disabled" "github.com/ElrondNetwork/elrond-go/trie" - triesFactory "github.com/ElrondNetwork/elrond-go/trie/factory" "github.com/ElrondNetwork/elrond-go/update" ) @@ -286,9 +286,9 @@ func (si *stateImport) getTrie(shardID uint32, accType Type) (common.Trie, error return trieForShard, nil } - trieStorageManager := si.trieStorageManagers[triesFactory.UserAccountTrie] + trieStorageManager := si.trieStorageManagers[dataRetriever.PeerAccountsUnit.String()] if accType == ValidatorAccount { - trieStorageManager = si.trieStorageManagers[triesFactory.PeerAccountTrie] + trieStorageManager = si.trieStorageManagers[dataRetriever.PeerAccountsUnit.String()] } trieForShard, err := trie.NewTrie(trieStorageManager, si.marshalizer, si.hasher, maxTrieLevelInMemory) @@ -323,7 +323,7 @@ func (si *stateImport) importDataTrie(identifier string, shID uint32, keys [][]b return fmt.Errorf("%w wanted a roothash", update.ErrWrongTypeAssertion) } - dataTrie, err := trie.NewTrie(si.trieStorageManagers[triesFactory.UserAccountTrie], si.marshalizer, si.hasher, maxTrieLevelInMemory) + dataTrie, err := trie.NewTrie(si.trieStorageManagers[dataRetriever.UserAccountsUnit.String()], si.marshalizer, si.hasher, maxTrieLevelInMemory) if err != nil { return err } diff --git a/update/genesis/import_test.go b/update/genesis/import_test.go index 7273b2f058d..ae7ca7d0468 100644 --- a/update/genesis/import_test.go +++ b/update/genesis/import_test.go @@ -10,9 +10,9 @@ import ( "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go/common" "github.com/ElrondNetwork/elrond-go/config" + "github.com/ElrondNetwork/elrond-go/dataRetriever" "github.com/ElrondNetwork/elrond-go/testscommon" "github.com/ElrondNetwork/elrond-go/testscommon/hashingMocks" - "github.com/ElrondNetwork/elrond-go/trie/factory" "github.com/ElrondNetwork/elrond-go/update" "github.com/ElrondNetwork/elrond-go/update/mock" "github.com/stretchr/testify/assert" @@ -23,7 +23,7 @@ import ( func TestNewStateImport(t *testing.T) { trieStorageManagers := make(map[string]common.StorageManager) - trieStorageManagers[factory.UserAccountTrie] = &testscommon.StorageManagerStub{} + trieStorageManagers[dataRetriever.UserAccountsUnit.String()] = &testscommon.StorageManagerStub{} tests := []struct { name string args ArgsNewStateImport @@ -82,8 +82,8 @@ func TestImportAll(t *testing.T) { t.Parallel() trieStorageManagers := make(map[string]common.StorageManager) - trieStorageManagers[factory.UserAccountTrie] = &testscommon.StorageManagerStub{} - trieStorageManagers[factory.PeerAccountTrie] = &testscommon.StorageManagerStub{} + trieStorageManagers[dataRetriever.UserAccountsUnit.String()] = &testscommon.StorageManagerStub{} + trieStorageManagers[dataRetriever.PeerAccountsUnit.String()] = &testscommon.StorageManagerStub{} args := ArgsNewStateImport{ HardforkStorer: &mock.HardforkStorerStub{}, @@ -105,7 +105,7 @@ func TestStateImport_ImportUnFinishedMetaBlocksShouldWork(t *testing.T) { t.Parallel() trieStorageManagers := make(map[string]common.StorageManager) - trieStorageManagers[factory.UserAccountTrie] = &testscommon.StorageManagerStub{} + trieStorageManagers[dataRetriever.UserAccountsUnit.String()] = &testscommon.StorageManagerStub{} hasher := &hashingMocks.HasherMock{} marshahlizer := &mock.MarshalizerMock{} From 3548ad0ffc3c26c4e8e13c89b5c8b609302fa6ad Mon Sep 17 00:00:00 2001 From: ssd04 Date: Fri, 23 Dec 2022 01:15:03 +0200 Subject: [PATCH 18/33] fix and update unit tests for db identifier set --- process/sync/metablock.go | 9 ++-- process/sync/metablock_test.go | 82 +++++++++++++++++++++++++++++-- process/sync/shardblock.go | 5 -- process/sync/shardblock_test.go | 5 +- testscommon/storage/storerStub.go | 9 ++++ 5 files changed, 94 insertions(+), 16 deletions(-) diff --git a/process/sync/metablock.go b/process/sync/metablock.go index 915ef619445..e2e4906b89a 100644 --- a/process/sync/metablock.go +++ b/process/sync/metablock.go @@ -107,11 +107,6 @@ func NewMetaBootstrap(arguments ArgMetaBootstrapper) (*MetaBootstrap, error) { return nil, err } - err = base.setAccountsStorerIdentifiers() - if err != nil { - return nil, err - } - base.init() return &boot, nil @@ -199,6 +194,10 @@ func (boot *MetaBootstrap) SyncBlock(ctx context.Context) error { } func (boot *MetaBootstrap) syncAccountsDBs(key []byte, id string) error { + err := boot.setAccountsStorerIdentifiers() + if err != nil { + return err + } // TODO: refactor this in order to avoid treatment based on identifier switch id { diff --git a/process/sync/metablock_test.go b/process/sync/metablock_test.go index c997b55a602..8b6922c0fba 100644 --- a/process/sync/metablock_test.go +++ b/process/sync/metablock_test.go @@ -59,6 +59,8 @@ func createMetaStore() dataRetriever.StorageService { store.AddStorer(dataRetriever.MetaBlockUnit, generateTestUnit()) store.AddStorer(dataRetriever.ShardHdrNonceHashDataUnit, generateTestUnit()) store.AddStorer(dataRetriever.MetaHdrNonceHashDataUnit, generateTestUnit()) + store.AddStorer(dataRetriever.UserAccountsUnit, generateTestUnit()) + store.AddStorer(dataRetriever.PeerAccountsUnit, generateTestUnit()) return store } @@ -1127,7 +1129,8 @@ func TestMetaBootstrap_ReceivedHeadersFoundInPoolShouldAddToForkDetector(t *test args.ShardCoordinator = shardCoordinator args.RoundHandler = initRoundHandler() - bs, _ := sync.NewMetaBootstrap(args) + bs, err := sync.NewMetaBootstrap(args) + require.Nil(t, err) bs.ReceivedHeaders(addedHdr, addedHash) time.Sleep(500 * time.Millisecond) @@ -1178,7 +1181,8 @@ func TestMetaBootstrap_ReceivedHeadersNotFoundInPoolShouldNotAddToForkDetector(t args.ChainHandler, _ = blockchain.NewBlockChain(&statusHandlerMock.AppStatusHandlerStub{}) args.RoundHandler = initRoundHandler() - bs, _ := sync.NewMetaBootstrap(args) + bs, err := sync.NewMetaBootstrap(args) + require.Nil(t, err) bs.ReceivedHeaders(addedHdr, addedHash) time.Sleep(500 * time.Millisecond) @@ -1622,7 +1626,7 @@ func TestMetaBootstrap_SyncBlockErrGetNodeDBShouldSyncAccounts(t *testing.T) { } args.ChainHandler = blkc - errGetNodeFromDB := commonErrors.NewGetNodeFromDBErrWithKey([]byte("key"), errors.New("get error"), common.AccountsTrieIdentifier) + errGetNodeFromDB := commonErrors.NewGetNodeFromDBErrWithKey([]byte("key"), errors.New("get error"), "userAccountsUnit") blockProcessor := createMetaBlockProcessor(args.ChainHandler) blockProcessor.ProcessBlockCalled = func(header data.HeaderHandler, body data.BodyHandler, haveTime func() time.Duration) error { return errGetNodeFromDB @@ -1690,6 +1694,32 @@ func TestMetaBootstrap_SyncBlockErrGetNodeDBShouldSyncAccounts(t *testing.T) { return []byte("roothash"), nil }} + args.Store = &storageStubs.ChainStorerStub{ + GetStorerCalled: func(unitType dataRetriever.UnitType) (storage.Storer, error) { + var dbIdentifier string + switch unitType { + case dataRetriever.UserAccountsUnit: + dbIdentifier = "userAccountsUnit" + case dataRetriever.PeerAccountsUnit: + dbIdentifier = "peerAccountsUnit" + default: + dbIdentifier = "" + } + + return &storageStubs.StorerStub{ + GetCalled: func(key []byte) ([]byte, error) { + return nil, process.ErrMissingHeader + }, + RemoveCalled: func(key []byte) error { + return nil + }, + GetIdentifierCalled: func() string { + return dbIdentifier + }, + }, nil + }, + } + bs, _ := sync.NewMetaBootstrap(args) err := bs.SyncBlock(context.Background()) @@ -1712,9 +1742,30 @@ func TestMetaBootstrap_SyncAccountsDBs(t *testing.T) { }, } + dbIdentifier := "userAccountsTrie" + args.Store = &storageStubs.ChainStorerStub{ + GetStorerCalled: func(unitType dataRetriever.UnitType) (storage.Storer, error) { + if unitType != dataRetriever.UserAccountsUnit { + return &storageStubs.StorerStub{}, nil + } + + return &storageStubs.StorerStub{ + GetCalled: func(key []byte) ([]byte, error) { + return nil, process.ErrMissingHeader + }, + RemoveCalled: func(key []byte) error { + return nil + }, + GetIdentifierCalled: func() string { + return dbIdentifier + }, + }, nil + }, + } + bs, _ := sync.NewMetaBootstrap(args) - err := bs.SyncAccountsDBs([]byte("key"), common.AccountsTrieIdentifier) + err := bs.SyncAccountsDBs([]byte("key"), dbIdentifier) require.Nil(t, err) require.True(t, accountsSyncCalled) }) @@ -1731,9 +1782,30 @@ func TestMetaBootstrap_SyncAccountsDBs(t *testing.T) { }, } + dbIdentifier := "peerAccountsTrie" + args.Store = &storageStubs.ChainStorerStub{ + GetStorerCalled: func(unitType dataRetriever.UnitType) (storage.Storer, error) { + if unitType != dataRetriever.PeerAccountsUnit { + return &storageStubs.StorerStub{}, nil + } + + return &storageStubs.StorerStub{ + GetCalled: func(key []byte) ([]byte, error) { + return nil, process.ErrMissingHeader + }, + RemoveCalled: func(key []byte) error { + return nil + }, + GetIdentifierCalled: func() string { + return dbIdentifier + }, + }, nil + }, + } + bs, _ := sync.NewMetaBootstrap(args) - err := bs.SyncAccountsDBs([]byte("key"), common.PeerAccountsTrieIdentifier) + err := bs.SyncAccountsDBs([]byte("key"), dbIdentifier) require.Nil(t, err) require.True(t, accountsSyncCalled) }) diff --git a/process/sync/shardblock.go b/process/sync/shardblock.go index 60fabc3e556..ddc9f524752 100644 --- a/process/sync/shardblock.go +++ b/process/sync/shardblock.go @@ -92,11 +92,6 @@ func NewShardBootstrap(arguments ArgShardBootstrapper) (*ShardBootstrap, error) return nil, err } - err = base.setAccountsStorerIdentifiers() - if err != nil { - return nil, err - } - base.init() return &boot, nil diff --git a/process/sync/shardblock_test.go b/process/sync/shardblock_test.go index 8cb5f66d8ae..c4e9c23d0bf 100644 --- a/process/sync/shardblock_test.go +++ b/process/sync/shardblock_test.go @@ -107,6 +107,8 @@ func createFullStore() dataRetriever.StorageService { store.AddStorer(dataRetriever.ShardHdrNonceHashDataUnit, generateTestUnit()) store.AddStorer(dataRetriever.ReceiptsUnit, generateTestUnit()) store.AddStorer(dataRetriever.ScheduledSCRsUnit, generateTestUnit()) + store.AddStorer(dataRetriever.UserAccountsUnit, generateTestUnit()) + store.AddStorer(dataRetriever.PeerAccountsUnit, generateTestUnit()) return store } @@ -1739,7 +1741,8 @@ func TestBootstrap_GetTxBodyHavingHashNotFoundInCacherOrStorageShouldRetEmptySli args.Store = createFullStore() args.Store.AddStorer(dataRetriever.TransactionUnit, txBlockUnit) - bs, _ := sync.NewShardBootstrap(args) + bs, err := sync.NewShardBootstrap(args) + require.Nil(t, err) gotMbsAndHashes, _ := bs.GetMiniBlocks(requestedHash) assert.Equal(t, 0, len(gotMbsAndHashes)) diff --git a/testscommon/storage/storerStub.go b/testscommon/storage/storerStub.go index a9fe5647880..a4e08a90095 100644 --- a/testscommon/storage/storerStub.go +++ b/testscommon/storage/storerStub.go @@ -19,6 +19,7 @@ type StorerStub struct { GetBulkFromEpochCalled func(keys [][]byte, epoch uint32) ([]storage.KeyValuePair, error) GetOldestEpochCalled func() (uint32, error) RangeKeysCalled func(handler func(key []byte, val []byte) bool) + GetIdentifierCalled func() string CloseCalled func() error } @@ -124,6 +125,14 @@ func (ss *StorerStub) RangeKeys(handler func(key []byte, val []byte) bool) { } } +// GetIdentifier - +func (ss *StorerStub) GetIdentifier() string { + if ss.GetIdentifierCalled != nil { + return ss.GetIdentifierCalled() + } + return "" +} + // Close - func (ss *StorerStub) Close() error { if ss.CloseCalled != nil { From bec60f1ba0fa86fcfa55862081558b8e4d38fbce Mon Sep 17 00:00:00 2001 From: ssd04 Date: Fri, 23 Dec 2022 11:03:02 +0200 Subject: [PATCH 19/33] fix factory processing unit tests --- factory/processing/blockProcessorCreator_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/factory/processing/blockProcessorCreator_test.go b/factory/processing/blockProcessorCreator_test.go index 165078792d6..8aaa99f4efe 100644 --- a/factory/processing/blockProcessorCreator_test.go +++ b/factory/processing/blockProcessorCreator_test.go @@ -8,6 +8,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/hashing" "github.com/ElrondNetwork/elrond-go-core/marshal" "github.com/ElrondNetwork/elrond-go/common" + "github.com/ElrondNetwork/elrond-go/dataRetriever" dataComp "github.com/ElrondNetwork/elrond-go/factory/data" "github.com/ElrondNetwork/elrond-go/factory/mock" processComp "github.com/ElrondNetwork/elrond-go/factory/processing" @@ -24,7 +25,6 @@ import ( storageManager "github.com/ElrondNetwork/elrond-go/testscommon/storage" trieMock "github.com/ElrondNetwork/elrond-go/testscommon/trie" "github.com/ElrondNetwork/elrond-go/trie" - trieFactory "github.com/ElrondNetwork/elrond-go/trie/factory" vmcommon "github.com/ElrondNetwork/elrond-vm-common" "github.com/stretchr/testify/require" ) @@ -104,14 +104,14 @@ func Test_newBlockProcessorCreatorForMeta(t *testing.T) { storageManagerPeer, _ := trie.CreateTrieStorageManager(storageManagerArgs, options) trieStorageManagers := make(map[string]common.StorageManager) - trieStorageManagers[trieFactory.UserAccountTrie] = storageManagerUser - trieStorageManagers[trieFactory.PeerAccountTrie] = storageManagerPeer + trieStorageManagers[dataRetriever.UserAccountsUnit.String()] = storageManagerUser + trieStorageManagers[dataRetriever.PeerAccountsUnit.String()] = storageManagerPeer accounts, err := createAccountAdapter( &mock.MarshalizerMock{}, &hashingMocks.HasherMock{}, factoryState.NewAccountCreator(), - trieStorageManagers[trieFactory.UserAccountTrie], + trieStorageManagers[dataRetriever.UserAccountsUnit.String()], ) require.Nil(t, err) From ff5674dbd37f67b7902692520ba29392d5cb8e48 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Fri, 23 Dec 2022 15:10:12 +0200 Subject: [PATCH 20/33] refactor the set of accounts db identifier --- process/sync/baseSync.go | 27 ++++++--------------------- process/sync/metablock.go | 11 ++++++++--- 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/process/sync/baseSync.go b/process/sync/baseSync.go index fd56a4d141e..7c204ad0891 100644 --- a/process/sync/baseSync.go +++ b/process/sync/baseSync.go @@ -124,9 +124,6 @@ type baseBootstrap struct { isInImportMode bool scheduledTxsExecutionHandler process.ScheduledTxsExecutionHandler processWaitTime time.Duration - - userAccountsStorerIdentifier string - peerAccountsStorerIdentifier string } // setRequestedHeaderNonce method sets the header nonce requested by the sync mechanism @@ -1127,29 +1124,17 @@ func (boot *baseBootstrap) waitForMiniBlocks() error { } } -func (boot *baseBootstrap) setAccountsStorerIdentifiers() error { - userStorer, err := boot.store.GetStorer(dataRetriever.UserAccountsUnit) - if err != nil { - return err - } - dbWithID, ok := userStorer.(dbStorerWithIdentifier) - if !ok { - return errors.ErrWrongTypeAssertion - } - boot.userAccountsStorerIdentifier = dbWithID.GetIdentifier() - - peerStorer, err := boot.store.GetStorer(dataRetriever.PeerAccountsUnit) +func (boot *baseBootstrap) getStorerIdentifier(unitType dataRetriever.UnitType) (string, error) { + storer, err := boot.store.GetStorer(unitType) if err != nil { - return err + return "", err } - dbPeerWithID, ok := peerStorer.(dbStorerWithIdentifier) + dbWithID, ok := storer.(dbStorerWithIdentifier) if !ok { - return errors.ErrWrongTypeAssertion + return "", errors.ErrWrongTypeAssertion } - boot.peerAccountsStorerIdentifier = dbPeerWithID.GetIdentifier() - - return nil + return dbWithID.GetIdentifier(), nil } func (boot *baseBootstrap) init() { diff --git a/process/sync/metablock.go b/process/sync/metablock.go index e2e4906b89a..a610b59f0e2 100644 --- a/process/sync/metablock.go +++ b/process/sync/metablock.go @@ -194,16 +194,21 @@ func (boot *MetaBootstrap) SyncBlock(ctx context.Context) error { } func (boot *MetaBootstrap) syncAccountsDBs(key []byte, id string) error { - err := boot.setAccountsStorerIdentifiers() + userAccountsStorerIdentifier, err := boot.getStorerIdentifier(dataRetriever.UserAccountsUnit) + if err != nil { + return err + } + + peerAccountsStorerIdentifier, err := boot.getStorerIdentifier(dataRetriever.PeerAccountsUnit) if err != nil { return err } // TODO: refactor this in order to avoid treatment based on identifier switch id { - case boot.userAccountsStorerIdentifier: + case userAccountsStorerIdentifier: return boot.syncUserAccountsState(key) - case boot.peerAccountsStorerIdentifier: + case peerAccountsStorerIdentifier: return boot.syncValidatorAccountsState(key) default: return fmt.Errorf("invalid trie identifier, id: %s", id) From 0a17029aafdfc9bfdf971a94cc641e3a44cb2419 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Mon, 9 Jan 2023 11:00:37 +0200 Subject: [PATCH 21/33] changed log warn to trace in get node from db --- trie/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trie/node.go b/trie/node.go index f56a733e8ae..b399fab5ca0 100644 --- a/trie/node.go +++ b/trie/node.go @@ -119,7 +119,7 @@ func getNodeFromDBAndDecode(n []byte, db common.DBWriteCacher, marshalizer marsh if err != nil { dbWithID, ok := db.(dbWriteCacherWithIdentifier) if !ok { - log.Warn("wrong type assertion on", common.GetNodeFromDBErrorString, "error", err, "key", n) + log.Trace("wrong type assertion on", common.GetNodeFromDBErrorString, "error", err, "key", n) return nil, errors.NewGetNodeFromDBErrWithKey(n, err, "") } From 73275b12cd93e8e0ca7845fc249228e039016ebb Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Thu, 12 Jan 2023 10:27:45 +0200 Subject: [PATCH 22/33] return err on CreateAndProcessMiniBlocks if getNodeFromDB err --- process/block/preprocess/transactions.go | 4 ++++ trie/node.go | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/process/block/preprocess/transactions.go b/process/block/preprocess/transactions.go index 35dd0900a43..1093033bc01 100644 --- a/process/block/preprocess/transactions.go +++ b/process/block/preprocess/transactions.go @@ -1083,6 +1083,10 @@ func (txs *transactions) CreateAndProcessMiniBlocks(haveTime func() bool, random ) if err != nil { + if elrondErr.IsGetNodeFromDBError(err) { + return nil, err + } + log.Debug("createAndProcessMiniBlocksFromMe", "error", err.Error()) return make(block.MiniBlockSlice, 0), nil } diff --git a/trie/node.go b/trie/node.go index b399fab5ca0..76afbee0223 100644 --- a/trie/node.go +++ b/trie/node.go @@ -3,6 +3,7 @@ package trie import ( "context" + "fmt" "time" "github.com/ElrondNetwork/elrond-go-core/hashing" @@ -119,7 +120,7 @@ func getNodeFromDBAndDecode(n []byte, db common.DBWriteCacher, marshalizer marsh if err != nil { dbWithID, ok := db.(dbWriteCacherWithIdentifier) if !ok { - log.Trace("wrong type assertion on", common.GetNodeFromDBErrorString, "error", err, "key", n) + log.Trace(common.GetNodeFromDBErrorString, "error", err, "key", n, "db type", fmt.Sprintf("%T", db)) return nil, errors.NewGetNodeFromDBErrWithKey(n, err, "") } From a2deeffc2903991993e20652cdb9dccf0c66bd6c Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Thu, 12 Jan 2023 10:33:18 +0200 Subject: [PATCH 23/33] small refactor --- process/block/preprocess/transactions.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/process/block/preprocess/transactions.go b/process/block/preprocess/transactions.go index 1093033bc01..e9b72498f67 100644 --- a/process/block/preprocess/transactions.go +++ b/process/block/preprocess/transactions.go @@ -1083,11 +1083,12 @@ func (txs *transactions) CreateAndProcessMiniBlocks(haveTime func() bool, random ) if err != nil { + log.Debug("createAndProcessMiniBlocksFromMe", "error", err.Error()) + if elrondErr.IsGetNodeFromDBError(err) { return nil, err } - log.Debug("createAndProcessMiniBlocksFromMe", "error", err.Error()) return make(block.MiniBlockSlice, 0), nil } From 65e517f6ae5a992d25791c92afa10cb31d119140 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Fri, 3 Feb 2023 12:13:48 +0200 Subject: [PATCH 24/33] return error if key not found --- process/block/shardblock.go | 5 +++++ process/coordinator/process.go | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/process/block/shardblock.go b/process/block/shardblock.go index 01cb1fb8ae3..fa1e61e9932 100644 --- a/process/block/shardblock.go +++ b/process/block/shardblock.go @@ -15,6 +15,7 @@ import ( logger "github.com/ElrondNetwork/elrond-go-logger" "github.com/ElrondNetwork/elrond-go/common" "github.com/ElrondNetwork/elrond-go/dataRetriever" + "github.com/ElrondNetwork/elrond-go/errors" processOutport "github.com/ElrondNetwork/elrond-go/outport/process" "github.com/ElrondNetwork/elrond-go/process" "github.com/ElrondNetwork/elrond-go/process/block/bootstrapStorage" @@ -2021,6 +2022,10 @@ func (sp *shardProcessor) createMiniBlocks(haveTime func() bool, randomness []by log.Debug("elapsed time to create mbs to me", "time", elapsedTime) if err != nil { log.Debug("createAndProcessCrossMiniBlocksDstMe", "error", err.Error()) + + if errors.IsGetNodeFromDBError(err) { + return nil, nil, err + } } if createAndProcessMBsDestMeInfo != nil { processedMiniBlocksDestMeInfo = createAndProcessMBsDestMeInfo.allProcessedMiniBlocksInfo diff --git a/process/coordinator/process.go b/process/coordinator/process.go index ff532881e3b..f3b7643fbc9 100644 --- a/process/coordinator/process.go +++ b/process/coordinator/process.go @@ -17,6 +17,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/marshal" logger "github.com/ElrondNetwork/elrond-go-logger" "github.com/ElrondNetwork/elrond-go/common" + "github.com/ElrondNetwork/elrond-go/errors" "github.com/ElrondNetwork/elrond-go/process" "github.com/ElrondNetwork/elrond-go/process/block/preprocess" "github.com/ElrondNetwork/elrond-go/process/block/processedMb" @@ -719,6 +720,11 @@ func (tc *transactionCoordinator) CreateMbsAndProcessCrossShardTransactionsDstMe "total gas penalized", tc.gasHandler.TotalGasPenalized(), "error", errProc, ) + + if errors.IsGetNodeFromDBError(errProc) { + return nil, 0, false, err + } + continue } From a683564b237deafbb1b46d0951a2aee8636a2cd2 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Fri, 3 Feb 2023 14:26:12 +0200 Subject: [PATCH 25/33] fix after merge --- .../resolverscontainer/metaResolversContainerFactory.go | 1 - .../metaResolversContainerFactory_test.go | 1 - .../resolverscontainer/shardResolversContainerFactory.go | 1 - .../shardResolversContainerFactory_test.go | 1 - genesis/process/genesisBlockCreator.go | 1 - genesis/process/genesisBlockCreator_test.go | 1 - integrationTests/state/stateTrie/stateTrie_test.go | 9 ++++----- .../state/stateTrieSync/stateTrieSync_test.go | 2 +- integrationTests/testProcessorNode.go | 1 - node/nodeRunner.go | 1 - process/block/preprocess/transactions.go | 2 +- process/block/shardblock.go | 1 + process/coordinator/process.go | 1 + process/sync/metablock_test.go | 1 + process/sync/shardblock_test.go | 1 + testscommon/components/components.go | 1 - testscommon/components/default.go | 1 + update/genesis/import.go | 2 +- update/genesis/import_test.go | 2 +- 19 files changed, 13 insertions(+), 18 deletions(-) diff --git a/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory.go b/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory.go index 788a71e0cdd..889481e9fde 100644 --- a/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory.go +++ b/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory.go @@ -8,7 +8,6 @@ import ( "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/dataRetriever/factory/containers" "github.com/multiversx/mx-chain-go/dataRetriever/resolvers" - triesFactory "github.com/multiversx/mx-chain-go/trie/factory" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/process/factory" diff --git a/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go b/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go index 15d8154110d..7d95585277c 100644 --- a/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go +++ b/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go @@ -18,7 +18,6 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/p2pmocks" storageStubs "github.com/multiversx/mx-chain-go/testscommon/storage" trieMock "github.com/multiversx/mx-chain-go/testscommon/trie" - triesFactory "github.com/multiversx/mx-chain-go/trie/factory" "github.com/stretchr/testify/assert" ) diff --git a/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory.go b/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory.go index 9bb58357816..7a4fb1a282a 100644 --- a/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory.go +++ b/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory.go @@ -9,7 +9,6 @@ import ( "github.com/multiversx/mx-chain-go/dataRetriever/factory/containers" "github.com/multiversx/mx-chain-go/dataRetriever/resolvers" "github.com/multiversx/mx-chain-go/process/factory" - triesFactory "github.com/multiversx/mx-chain-go/trie/factory" ) var _ dataRetriever.ResolversContainerFactory = (*shardResolversContainerFactory)(nil) diff --git a/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go b/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go index c62d1e04eab..b33cdbfa64a 100644 --- a/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go +++ b/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go @@ -18,7 +18,6 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/p2pmocks" storageStubs "github.com/multiversx/mx-chain-go/testscommon/storage" trieMock "github.com/multiversx/mx-chain-go/testscommon/trie" - triesFactory "github.com/multiversx/mx-chain-go/trie/factory" "github.com/stretchr/testify/assert" ) diff --git a/genesis/process/genesisBlockCreator.go b/genesis/process/genesisBlockCreator.go index fd6a5b273e7..247cb6ae49a 100644 --- a/genesis/process/genesisBlockCreator.go +++ b/genesis/process/genesisBlockCreator.go @@ -28,7 +28,6 @@ import ( "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/factory" "github.com/multiversx/mx-chain-go/storage/storageunit" - triesFactory "github.com/multiversx/mx-chain-go/trie/factory" "github.com/multiversx/mx-chain-go/update" hardfork "github.com/multiversx/mx-chain-go/update/genesis" hardForkProcess "github.com/multiversx/mx-chain-go/update/process" diff --git a/genesis/process/genesisBlockCreator_test.go b/genesis/process/genesisBlockCreator_test.go index ea1a50b0218..30bfe94d609 100644 --- a/genesis/process/genesisBlockCreator_test.go +++ b/genesis/process/genesisBlockCreator_test.go @@ -28,7 +28,6 @@ import ( stateMock "github.com/multiversx/mx-chain-go/testscommon/state" storageCommon "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/trie" - "github.com/multiversx/mx-chain-go/trie/factory" "github.com/multiversx/mx-chain-go/update" updateMock "github.com/multiversx/mx-chain-go/update/mock" "github.com/multiversx/mx-chain-go/vm/systemSmartContracts/defaults" diff --git a/integrationTests/state/stateTrie/stateTrie_test.go b/integrationTests/state/stateTrie/stateTrie_test.go index 97d7939252d..f71c42d8a85 100644 --- a/integrationTests/state/stateTrie/stateTrie_test.go +++ b/integrationTests/state/stateTrie/stateTrie_test.go @@ -5,13 +5,13 @@ import ( "encoding/base64" "encoding/binary" "encoding/hex" - "errors" "fmt" "math" "math/big" "math/rand" "runtime" "strconv" + "strings" "sync" "sync/atomic" "testing" @@ -39,7 +39,6 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/statusHandler" trieMock "github.com/multiversx/mx-chain-go/testscommon/trie" "github.com/multiversx/mx-chain-go/trie" - trieFactory "github.com/multiversx/mx-chain-go/trie/factory" vmcommon "github.com/multiversx/mx-chain-vm-common-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -1368,7 +1367,7 @@ func TestRollbackBlockAndCheckThatPruningIsCancelledOnAccountsTrie(t *testing.T) if !bytes.Equal(rootHash, rootHashOfRollbackedBlock) { time.Sleep(time.Second * 3) err = shardNode.AccntState.RecreateTrie(rootHashOfRollbackedBlock) - require.True(t, errors.Is(err, trie.ErrKeyNotFound)) + require.True(t, strings.Contains(err.Error(), trie.ErrKeyNotFound.Error())) } nonces := []*uint64{new(uint64), new(uint64)} @@ -1529,7 +1528,7 @@ func TestTriePruningWhenBlockIsFinal(t *testing.T) { require.Equal(t, uint64(17), nodes[1].BlockChain.GetCurrentBlockHeader().GetNonce()) err := shardNode.AccntState.RecreateTrie(rootHashOfFirstBlock) - require.True(t, errors.Is(err, trie.ErrKeyNotFound)) + require.True(t, strings.Contains(err.Error(), trie.ErrKeyNotFound.Error())) } func TestStatePruningIsNotBuffered(t *testing.T) { @@ -1674,7 +1673,7 @@ func checkTrieCanBeRecreated(tb testing.TB, node *integrationTests.TestProcessor return } - stateTrie := node.TrieContainer.Get([]byte(trieFactory.UserAccountTrie)) + stateTrie := node.TrieContainer.Get([]byte(dataRetriever.UserAccountsUnit.String())) roothash := node.BlockChain.GetCurrentBlockRootHash() tr, err := stateTrie.Recreate(roothash) require.Nil(tb, err) diff --git a/integrationTests/state/stateTrieSync/stateTrieSync_test.go b/integrationTests/state/stateTrieSync/stateTrieSync_test.go index 656cc1b8d14..5a46fc9d9a3 100644 --- a/integrationTests/state/stateTrieSync/stateTrieSync_test.go +++ b/integrationTests/state/stateTrieSync/stateTrieSync_test.go @@ -11,6 +11,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-go/common" + "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/epochStart/notifier" "github.com/multiversx/mx-chain-go/integrationTests" "github.com/multiversx/mx-chain-go/process/factory" @@ -20,7 +21,6 @@ import ( "github.com/multiversx/mx-chain-go/testscommon" testStorage "github.com/multiversx/mx-chain-go/testscommon/state" "github.com/multiversx/mx-chain-go/trie" - trieFactory "github.com/multiversx/mx-chain-go/trie/factory" "github.com/multiversx/mx-chain-go/trie/keyBuilder" "github.com/multiversx/mx-chain-go/trie/statistics" "github.com/multiversx/mx-chain-go/trie/storageMarker" diff --git a/integrationTests/testProcessorNode.go b/integrationTests/testProcessorNode.go index 54dd3eb2146..e154cbfb7a0 100644 --- a/integrationTests/testProcessorNode.go +++ b/integrationTests/testProcessorNode.go @@ -112,7 +112,6 @@ import ( statusHandlerMock "github.com/multiversx/mx-chain-go/testscommon/statusHandler" storageStubs "github.com/multiversx/mx-chain-go/testscommon/storage" trieMock "github.com/multiversx/mx-chain-go/testscommon/trie" - trieFactory "github.com/multiversx/mx-chain-go/trie/factory" "github.com/multiversx/mx-chain-go/trie/keyBuilder" "github.com/multiversx/mx-chain-go/update" "github.com/multiversx/mx-chain-go/update/trigger" diff --git a/node/nodeRunner.go b/node/nodeRunner.go index e6ed4036870..1ea5b5cbd5e 100644 --- a/node/nodeRunner.go +++ b/node/nodeRunner.go @@ -59,7 +59,6 @@ import ( "github.com/multiversx/mx-chain-go/storage/cache" storageFactory "github.com/multiversx/mx-chain-go/storage/factory" "github.com/multiversx/mx-chain-go/storage/storageunit" - trieFactory "github.com/multiversx/mx-chain-go/trie/factory" trieStatistics "github.com/multiversx/mx-chain-go/trie/statistics" "github.com/multiversx/mx-chain-go/trie/storageMarker" "github.com/multiversx/mx-chain-go/update/trigger" diff --git a/process/block/preprocess/transactions.go b/process/block/preprocess/transactions.go index 6610af0d68d..3d1b86024d6 100644 --- a/process/block/preprocess/transactions.go +++ b/process/block/preprocess/transactions.go @@ -1091,7 +1091,7 @@ func (txs *transactions) CreateAndProcessMiniBlocks(haveTime func() bool, random if err != nil { log.Debug("createAndProcessMiniBlocksFromMe", "error", err.Error()) - if elrondErr.IsGetNodeFromDBError(err) { + if chainErr.IsGetNodeFromDBError(err) { return nil, err } diff --git a/process/block/shardblock.go b/process/block/shardblock.go index 2b94a99091a..f007d6d746e 100644 --- a/process/block/shardblock.go +++ b/process/block/shardblock.go @@ -13,6 +13,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data/headerVersionData" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" + "github.com/multiversx/mx-chain-go/errors" processOutport "github.com/multiversx/mx-chain-go/outport/process" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/block/bootstrapStorage" diff --git a/process/coordinator/process.go b/process/coordinator/process.go index 0ebfeb2790e..5a1a685a478 100644 --- a/process/coordinator/process.go +++ b/process/coordinator/process.go @@ -16,6 +16,7 @@ import ( "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" + "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/block/preprocess" "github.com/multiversx/mx-chain-go/process/block/processedMb" diff --git a/process/sync/metablock_test.go b/process/sync/metablock_test.go index eba20a78c78..019076d66b9 100644 --- a/process/sync/metablock_test.go +++ b/process/sync/metablock_test.go @@ -19,6 +19,7 @@ import ( "github.com/multiversx/mx-chain-go/consensus/round" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/dataRetriever/blockchain" + commonErrors "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/process/sync" diff --git a/process/sync/shardblock_test.go b/process/sync/shardblock_test.go index 53e20ed75db..7c3f2ab1d9c 100644 --- a/process/sync/shardblock_test.go +++ b/process/sync/shardblock_test.go @@ -20,6 +20,7 @@ import ( "github.com/multiversx/mx-chain-go/consensus/round" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/dataRetriever/blockchain" + commonErrors "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/process/sync" diff --git a/testscommon/components/components.go b/testscommon/components/components.go index cc20dbc6b74..e5ab892f0b9 100644 --- a/testscommon/components/components.go +++ b/testscommon/components/components.go @@ -40,7 +40,6 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/shardingMocks" statusHandlerMock "github.com/multiversx/mx-chain-go/testscommon/statusHandler" "github.com/multiversx/mx-chain-go/trie" - trieFactory "github.com/multiversx/mx-chain-go/trie/factory" "github.com/multiversx/mx-chain-go/trie/hashesHolder" logger "github.com/multiversx/mx-chain-logger-go" wasmConfig "github.com/multiversx/mx-chain-vm-v1_4-go/config" diff --git a/testscommon/components/default.go b/testscommon/components/default.go index 7212144c234..7ebcd67d507 100644 --- a/testscommon/components/default.go +++ b/testscommon/components/default.go @@ -6,6 +6,7 @@ import ( crypto "github.com/multiversx/mx-chain-crypto-go" "github.com/multiversx/mx-chain-go/common" consensusMocks "github.com/multiversx/mx-chain-go/consensus/mock" + "github.com/multiversx/mx-chain-go/dataRetriever" dataRetrieverMock "github.com/multiversx/mx-chain-go/dataRetriever/mock" "github.com/multiversx/mx-chain-go/factory/mock" "github.com/multiversx/mx-chain-go/sharding" diff --git a/update/genesis/import.go b/update/genesis/import.go index f482dac0674..ce9f3c33f53 100644 --- a/update/genesis/import.go +++ b/update/genesis/import.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "github.com/multiversx/mx-chain-go/dataRetriever" "strings" "github.com/multiversx/mx-chain-core-go/core" @@ -20,7 +21,6 @@ import ( "github.com/multiversx/mx-chain-go/state/factory" "github.com/multiversx/mx-chain-go/state/storagePruningManager/disabled" "github.com/multiversx/mx-chain-go/trie" - triesFactory "github.com/multiversx/mx-chain-go/trie/factory" "github.com/multiversx/mx-chain-go/update" ) diff --git a/update/genesis/import_test.go b/update/genesis/import_test.go index 9cc938129a5..01a8b50e82e 100644 --- a/update/genesis/import_test.go +++ b/update/genesis/import_test.go @@ -10,9 +10,9 @@ import ( "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/config" + "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" - "github.com/multiversx/mx-chain-go/trie/factory" "github.com/multiversx/mx-chain-go/update" "github.com/multiversx/mx-chain-go/update/mock" "github.com/stretchr/testify/assert" From 36ddeb41cbecb03b18a5f9c0c20de625428da294 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Mon, 13 Feb 2023 11:45:50 +0200 Subject: [PATCH 26/33] remove error wrapping from trie Get() --- trie/patriciaMerkleTrie.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trie/patriciaMerkleTrie.go b/trie/patriciaMerkleTrie.go index 1fb8c3abce7..75bbe70f9bc 100644 --- a/trie/patriciaMerkleTrie.go +++ b/trie/patriciaMerkleTrie.go @@ -88,7 +88,7 @@ func (tr *patriciaMerkleTrie) Get(key []byte) ([]byte, uint32, error) { val, depth, err := tr.root.tryGet(hexKey, rootDepthLevel, tr.trieStorage) if err != nil { - err = fmt.Errorf("trie get error: %w, for key %v", err, hex.EncodeToString(key)) + log.Error("trie get error", "error", err.Error(), "key", hex.EncodeToString(key)) return nil, depth, err } From 7123970e7e72aac8596249accfb4e09d06ffbbaa Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Tue, 14 Feb 2023 17:04:41 +0200 Subject: [PATCH 27/33] small logs refactor --- trie/node.go | 2 +- trie/patriciaMerkleTrie.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/trie/node.go b/trie/node.go index e7678e0ab25..004bea97c26 100644 --- a/trie/node.go +++ b/trie/node.go @@ -120,7 +120,7 @@ func getNodeFromDBAndDecode(n []byte, db common.DBWriteCacher, marshalizer marsh if err != nil { dbWithID, ok := db.(dbWriteCacherWithIdentifier) if !ok { - log.Trace(common.GetNodeFromDBErrorString, "error", err, "key", n, "db type", fmt.Sprintf("%T", db)) + log.Warn(common.GetNodeFromDBErrorString, "error", err, "key", n, "db type", fmt.Sprintf("%T", db)) return nil, errors.NewGetNodeFromDBErrWithKey(n, err, "") } diff --git a/trie/patriciaMerkleTrie.go b/trie/patriciaMerkleTrie.go index 75bbe70f9bc..8d08857b2aa 100644 --- a/trie/patriciaMerkleTrie.go +++ b/trie/patriciaMerkleTrie.go @@ -88,7 +88,7 @@ func (tr *patriciaMerkleTrie) Get(key []byte) ([]byte, uint32, error) { val, depth, err := tr.root.tryGet(hexKey, rootDepthLevel, tr.trieStorage) if err != nil { - log.Error("trie get error", "error", err.Error(), "key", hex.EncodeToString(key)) + log.Error("trie get error", "error", err.Error(), "key", key) return nil, depth, err } From a99d8d7ba51066aa3c5dc7e6febe003377f51d02 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Mon, 6 Mar 2023 16:03:45 +0200 Subject: [PATCH 28/33] remove error wrapping when loading data trie --- state/accountsDB.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/state/accountsDB.go b/state/accountsDB.go index 7ff1617e04b..f1514d58b05 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -524,7 +524,8 @@ func (adb *AccountsDB) loadDataTrie(accountHandler baseAccountHandler, mainTrie dataTrie, err := mainTrie.Recreate(accountHandler.GetRootHash()) if err != nil { - return fmt.Errorf("trie was not found for hash, rootHash = %s, err = %w", hex.EncodeToString(accountHandler.GetRootHash()), err) + log.Error("trie was not found for hash", "rootHash", accountHandler.GetRootHash(), "err", err) + return err } accountHandler.SetDataTrie(dataTrie) From aea10143dc8f4caed759ebaec72b3c77fe1d6825 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Thu, 16 Mar 2023 11:46:08 +0200 Subject: [PATCH 29/33] add GetIdentifier to StorageManager interface --- common/interface.go | 1 + testscommon/storageManagerStub.go | 10 ++++++++++ trie/trieStorageManager.go | 11 +++++++++++ 3 files changed, 22 insertions(+) diff --git a/common/interface.go b/common/interface.go index a58b6aa94db..1836d4b51e6 100644 --- a/common/interface.go +++ b/common/interface.go @@ -78,6 +78,7 @@ type StorageManager interface { SetEpochForPutOperation(uint32) ShouldTakeSnapshot() bool GetBaseTrieStorageManager() StorageManager + GetIdentifier() string IsClosed() bool Close() error IsInterfaceNil() bool diff --git a/testscommon/storageManagerStub.go b/testscommon/storageManagerStub.go index b7673a4b4cd..924340d4df7 100644 --- a/testscommon/storageManagerStub.go +++ b/testscommon/storageManagerStub.go @@ -28,6 +28,7 @@ type StorageManagerStub struct { IsClosedCalled func() bool RemoveFromCheckpointHashesHolderCalled func([]byte) GetBaseTrieStorageManagerCalled func() common.StorageManager + GetIdentifierCalled func() string } // Put - @@ -214,6 +215,15 @@ func (sms *StorageManagerStub) GetBaseTrieStorageManager() common.StorageManager return nil } +// GetIdentifier - +func (sms *StorageManagerStub) GetIdentifier() string { + if sms.GetIdentifierCalled != nil { + return sms.GetIdentifierCalled() + } + + return "" +} + // IsInterfaceNil - func (sms *StorageManagerStub) IsInterfaceNil() bool { return sms == nil diff --git a/trie/trieStorageManager.go b/trie/trieStorageManager.go index dc50faff711..5ac0979f943 100644 --- a/trie/trieStorageManager.go +++ b/trie/trieStorageManager.go @@ -687,6 +687,17 @@ func (tsm *trieStorageManager) GetBaseTrieStorageManager() common.StorageManager return tsm } +// GetIdentifier returns the identifier of the main storer +func (tsm *trieStorageManager) GetIdentifier() string { + dbWithIdentifier, ok := tsm.mainStorer.(dbWriteCacherWithIdentifier) + if !ok { + log.Warn("trieStorageManager.GetIdentifier mainStorer is not of type dbWriteCacherWithIdentifier", "type", fmt.Sprintf("%T", tsm.mainStorer)) + return "" + } + + return dbWithIdentifier.GetIdentifier() +} + // IsInterfaceNil returns true if there is no value under the interface func (tsm *trieStorageManager) IsInterfaceNil() bool { return tsm == nil From 6d65587633cdb92602c3cd25e5225c90ae3aed0a Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Thu, 16 Mar 2023 12:00:56 +0200 Subject: [PATCH 30/33] add stack trace print if getNodeFromDb error --- trie/node.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/trie/node.go b/trie/node.go index 3dbc6f62c8d..29a829b057a 100644 --- a/trie/node.go +++ b/trie/node.go @@ -4,6 +4,7 @@ package trie import ( "context" "fmt" + "runtime/debug" "time" "github.com/multiversx/mx-chain-core-go/hashing" @@ -124,7 +125,7 @@ func getNodeFromDBAndDecode(n []byte, db common.DBWriteCacher, marshalizer marsh return nil, errors.NewGetNodeFromDBErrWithKey(n, err, "") } - log.Trace(common.GetNodeFromDBErrorString, "error", err, "key", n) + log.Trace(common.GetNodeFromDBErrorString, "error", err, "key", n, "stack trace", string(debug.Stack())) return nil, errors.NewGetNodeFromDBErrWithKey(n, err, dbWithID.GetIdentifier()) } From 25dc1d432a5e4375b9751f7c21ebe13e7765eaf3 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Thu, 16 Mar 2023 12:47:46 +0200 Subject: [PATCH 31/33] refactor log prints --- trie/node.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/trie/node.go b/trie/node.go index 29a829b057a..190bb41dd9b 100644 --- a/trie/node.go +++ b/trie/node.go @@ -119,13 +119,14 @@ func computeAndSetNodeHash(n node) ([]byte, error) { func getNodeFromDBAndDecode(n []byte, db common.DBWriteCacher, marshalizer marshal.Marshalizer, hasher hashing.Hasher) (node, error) { encChild, err := db.Get(n) if err != nil { + log.Trace(common.GetNodeFromDBErrorString, "error", err, "key", n, "stack trace", string(debug.Stack())) + dbWithID, ok := db.(dbWriteCacherWithIdentifier) if !ok { - log.Warn(common.GetNodeFromDBErrorString, "error", err, "key", n, "db type", fmt.Sprintf("%T", db)) + log.Warn("db does not have an identifier", "db type", fmt.Sprintf("%T", db)) return nil, errors.NewGetNodeFromDBErrWithKey(n, err, "") } - log.Trace(common.GetNodeFromDBErrorString, "error", err, "key", n, "stack trace", string(debug.Stack())) return nil, errors.NewGetNodeFromDBErrWithKey(n, err, dbWithID.GetIdentifier()) } From 90c6e2130b0a9b8103ee890f2d5c3adbf7e1d55c Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Mon, 3 Apr 2023 16:54:39 +0300 Subject: [PATCH 32/33] move IsGetNodeFromDbErr() to core --- common/logging/errorLogging.go | 4 +- epochStart/shardchain/trigger.go | 3 +- errors/closingError_test.go | 45 ------------------- errors/missingTrieNodeError.go | 19 -------- go.mod | 6 +-- go.sum | 12 ++--- process/block/baseProcess.go | 3 +- process/block/preprocess/basePreProcess.go | 3 +- process/block/preprocess/transactions.go | 7 ++- process/block/preprocess/transactionsV2.go | 7 ++- process/block/shardblock.go | 3 +- process/coordinator/process.go | 3 +- process/rewardTransaction/process.go | 3 +- process/smartContract/process.go | 15 +++---- process/sync/metablock.go | 3 +- process/sync/shardblock.go | 3 +- process/transaction/shardProcess.go | 3 +- .../indexHashedNodesCoordinator.go | 3 +- state/accountsDB.go | 3 +- .../storagePruningManager.go | 3 +- trie/node.go | 3 +- trie/patriciaMerkleTrie.go | 4 +- trie/snapshotTrieStorageManager.go | 2 +- trie/trieStorageManager.go | 6 +-- trie/trieStorageManagerInEpoch.go | 3 +- 25 files changed, 47 insertions(+), 122 deletions(-) delete mode 100644 errors/closingError_test.go diff --git a/common/logging/errorLogging.go b/common/logging/errorLogging.go index 17693e3b4f3..94bc88ae74d 100644 --- a/common/logging/errorLogging.go +++ b/common/logging/errorLogging.go @@ -1,8 +1,8 @@ package logging import ( + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" - chainErrors "github.com/multiversx/mx-chain-go/errors" logger "github.com/multiversx/mx-chain-logger-go" ) @@ -24,7 +24,7 @@ func logErrAsLevelExceptAsDebugIfClosingError(logInstance logger.Logger, logLeve return } - if chainErrors.IsClosingError(err) { + if core.IsClosingError(err) { logLevel = logger.LogDebug } diff --git a/epochStart/shardchain/trigger.go b/epochStart/shardchain/trigger.go index cd0637d724e..76a949b6961 100644 --- a/epochStart/shardchain/trigger.go +++ b/epochStart/shardchain/trigger.go @@ -22,7 +22,6 @@ import ( "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/epochStart" - "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-logger-go" @@ -997,7 +996,7 @@ func (t *trigger) SetProcessed(header data.HeaderHandler, _ data.BodyHandler) { errNotCritical = t.shardHdrStorage.Put([]byte(epochStartIdentifier), shardHdrBuff) if errNotCritical != nil { logLevel := logger.LogWarning - if errors.IsClosingError(errNotCritical) { + if core.IsClosingError(errNotCritical) { logLevel = logger.LogDebug } log.Log(logLevel, "SetProcessed put to shard header storage error", "error", errNotCritical) diff --git a/errors/closingError_test.go b/errors/closingError_test.go deleted file mode 100644 index 27316a4dc42..00000000000 --- a/errors/closingError_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package errors_test - -import ( - "fmt" - "testing" - - "github.com/multiversx/mx-chain-go/errors" - "github.com/multiversx/mx-chain-go/storage" - "github.com/stretchr/testify/assert" -) - -func TestIsClosingError(t *testing.T) { - t.Parallel() - - t.Run("nil error should return false", func(t *testing.T) { - t.Parallel() - - assert.False(t, errors.IsClosingError(nil)) - }) - t.Run("context closing error should return true", func(t *testing.T) { - t.Parallel() - - assert.True(t, errors.IsClosingError(fmt.Errorf("%w random string", errors.ErrContextClosing))) - }) - t.Run("DB closed error should return true", func(t *testing.T) { - t.Parallel() - - assert.True(t, errors.IsClosingError(fmt.Errorf("%w random string", storage.ErrDBIsClosed))) - }) - t.Run("contains 'DB is closed' should return true", func(t *testing.T) { - t.Parallel() - - assert.True(t, errors.IsClosingError(fmt.Errorf("random string DB is closed random string"))) - }) - t.Run("contains 'DB is closed' should return true", func(t *testing.T) { - t.Parallel() - - assert.True(t, errors.IsClosingError(fmt.Errorf("random string context closing random string"))) - }) - t.Run("random error should return false", func(t *testing.T) { - t.Parallel() - - assert.False(t, errors.IsClosingError(fmt.Errorf("random error"))) - }) -} diff --git a/errors/missingTrieNodeError.go b/errors/missingTrieNodeError.go index c78ef8bd794..ebb92003085 100644 --- a/errors/missingTrieNodeError.go +++ b/errors/missingTrieNodeError.go @@ -3,28 +3,9 @@ package errors import ( "encoding/hex" "fmt" - "strings" - "github.com/multiversx/mx-chain-go/common" ) -// IsGetNodeFromDBError returns true if the provided error is of type getNodeFromDB -func IsGetNodeFromDBError(err error) bool { - if err == nil { - return false - } - - if IsClosingError(err) { - return false - } - - if strings.Contains(err.Error(), common.GetNodeFromDBErrorString) { - return true - } - - return false -} - // GetNodeFromDBErrWithKey defines a custom error for trie get node type GetNodeFromDBErrWithKey struct { getErr error diff --git a/go.mod b/go.mod index 09573845409..9fc383bb950 100644 --- a/go.mod +++ b/go.mod @@ -13,13 +13,13 @@ require ( github.com/google/gops v0.3.18 github.com/gorilla/websocket v1.5.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/multiversx/mx-chain-core-go v1.2.0 + github.com/multiversx/mx-chain-core-go v1.2.1-0.20230403113932-916b16d18978 github.com/multiversx/mx-chain-crypto-go v1.2.5 github.com/multiversx/mx-chain-es-indexer-go v1.4.0 github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-p2p-go v1.0.15 - github.com/multiversx/mx-chain-storage-go v1.0.7 - github.com/multiversx/mx-chain-vm-common-go v1.4.0 + github.com/multiversx/mx-chain-storage-go v1.0.8-0.20230403115027-9139fce478e0 + github.com/multiversx/mx-chain-vm-common-go v1.4.1-0.20230403123953-7fc57accc0c6 github.com/multiversx/mx-chain-vm-v1_2-go v1.2.50 github.com/multiversx/mx-chain-vm-v1_3-go v1.3.51 github.com/multiversx/mx-chain-vm-v1_4-go v1.4.77 diff --git a/go.sum b/go.sum index 1cc5571c731..90649986acf 100644 --- a/go.sum +++ b/go.sum @@ -611,9 +611,8 @@ github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUY github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.1.31/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.1.34/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.2.0 h1:K539hKZKcHjBiFQpowFbA3BUd95Fe5+FLC+rKBOFZF0= -github.com/multiversx/mx-chain-core-go v1.2.0/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230403113932-916b16d18978 h1:caHg1OhZmaA8oX3TbimkBaty+eHvhpNSO8rQOicrS7o= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230403113932-916b16d18978/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-crypto-go v1.2.5 h1:tuq3BUNMhKud5DQbZi9DiVAAHUXypizy8zPH0NpTGZk= github.com/multiversx/mx-chain-crypto-go v1.2.5/go.mod h1:teqhNyWEqfMPgNn8sgWXlgtJ1a36jGCnhs/tRpXW6r4= github.com/multiversx/mx-chain-es-indexer-go v1.4.0 h1:t2UCfbLRbFPBWK1IC1/qOVg+2D6y189xZZ1BoV83gq8= @@ -624,10 +623,13 @@ github.com/multiversx/mx-chain-p2p-go v1.0.15 h1:H7273huZG/zAR6MPvWuXwBEVBsJWH1M github.com/multiversx/mx-chain-p2p-go v1.0.15/go.mod h1:hUE4H8kGJk3u9gTqeetF3uhjJpnfdV/hALKsJ6bMI+8= github.com/multiversx/mx-chain-storage-go v1.0.7 h1:UqLo/OLTD3IHiE/TB/SEdNRV1GG2f1R6vIP5ehHwCNw= github.com/multiversx/mx-chain-storage-go v1.0.7/go.mod h1:gtKoV32Cg2Uy8deHzF8Ud0qAl0zv92FvWgPSYIP0Zmg= +github.com/multiversx/mx-chain-storage-go v1.0.8-0.20230403115027-9139fce478e0 h1:jTGuq0IAQdghGLoNx2BgkxWvkcZV9ZmJ0qB8/oU4MNQ= +github.com/multiversx/mx-chain-storage-go v1.0.8-0.20230403115027-9139fce478e0/go.mod h1:FGhaeTNIcLZOPqsJZQ1TdcMaPVLhj642OzRNmt6+RQs= +github.com/multiversx/mx-chain-vm-common-go v1.3.34/go.mod h1:sZ2COLCxvf2GxAAJHGmGqWybObLtFuk2tZUyGqnMXE8= github.com/multiversx/mx-chain-vm-common-go v1.3.36/go.mod h1:sZ2COLCxvf2GxAAJHGmGqWybObLtFuk2tZUyGqnMXE8= github.com/multiversx/mx-chain-vm-common-go v1.3.37/go.mod h1:sZ2COLCxvf2GxAAJHGmGqWybObLtFuk2tZUyGqnMXE8= -github.com/multiversx/mx-chain-vm-common-go v1.4.0 h1:0i0cJZJOXGzqYzwtKFHSr2yGmnFAdizOuISK8HgsnYo= -github.com/multiversx/mx-chain-vm-common-go v1.4.0/go.mod h1:odBJC92ANA8zLtPh/wwajUUGJOaS88F5QYGf0t8Wgzw= +github.com/multiversx/mx-chain-vm-common-go v1.4.1-0.20230403123953-7fc57accc0c6 h1:3G8BHyVfz1DkeZcds4iME5vDHzg8Yg2++wet0DDYZ3c= +github.com/multiversx/mx-chain-vm-common-go v1.4.1-0.20230403123953-7fc57accc0c6/go.mod h1:rxb8laeh06wayB/dZPpN5LT3qcwv4SgpNHiSvPsNjuw= github.com/multiversx/mx-chain-vm-v1_2-go v1.2.50 h1:ScUq7/wq78vthMTQ6v5Ux1DvSMQMHxQ2Sl7aPP26q1w= github.com/multiversx/mx-chain-vm-v1_2-go v1.2.50/go.mod h1:e3uYdgoKzs3puaznbmSjDcRisJc5Do4tpg7VqyYwoek= github.com/multiversx/mx-chain-vm-v1_3-go v1.3.51 h1:axtp5/mpA+xYJ1cu4KtAGETV4t6v6/tNfQh0HCclBYY= diff --git a/process/block/baseProcess.go b/process/block/baseProcess.go index 160f4cccfa7..191e91972de 100644 --- a/process/block/baseProcess.go +++ b/process/block/baseProcess.go @@ -28,7 +28,6 @@ import ( "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/dblookupext" debugFactory "github.com/multiversx/mx-chain-go/debug/factory" - "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/outport" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/block/bootstrapStorage" @@ -1685,7 +1684,7 @@ func (bp *baseProcessor) recordBlockInHistory(blockHeaderHash []byte, blockHeade err := bp.historyRepo.RecordBlock(blockHeaderHash, blockHeader, blockBody, scrResultsFromPool, receiptsFromPool, intraMiniBlocks, logs) if err != nil { logLevel := logger.LogError - if errors.IsClosingError(err) { + if core.IsClosingError(err) { logLevel = logger.LogDebug } log.Log(logLevel, "historyRepo.RecordBlock()", "blockHeaderHash", blockHeaderHash, "error", err.Error()) diff --git a/process/block/preprocess/basePreProcess.go b/process/block/preprocess/basePreProcess.go index 826d1b6bd35..08f3e4cfa37 100644 --- a/process/block/preprocess/basePreProcess.go +++ b/process/block/preprocess/basePreProcess.go @@ -14,7 +14,6 @@ import ( "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" - "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/storage" @@ -414,7 +413,7 @@ func (bpp *basePreProcess) saveAccountBalanceForAddress(address []byte) error { balance, err := bpp.getBalanceForAddress(address) if err != nil { - if errors.IsGetNodeFromDBError(err) { + if core.IsGetNodeFromDBError(err) { return err } balance = big.NewInt(0) diff --git a/process/block/preprocess/transactions.go b/process/block/preprocess/transactions.go index 3d1b86024d6..deb7dacb733 100644 --- a/process/block/preprocess/transactions.go +++ b/process/block/preprocess/transactions.go @@ -17,7 +17,6 @@ import ( "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" - chainErr "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/state" @@ -1091,7 +1090,7 @@ func (txs *transactions) CreateAndProcessMiniBlocks(haveTime func() bool, random if err != nil { log.Debug("createAndProcessMiniBlocksFromMe", "error", err.Error()) - if chainErr.IsGetNodeFromDBError(err) { + if core.IsGetNodeFromDBError(err) { return nil, err } @@ -1206,7 +1205,7 @@ func (txs *transactions) createAndProcessMiniBlocksFromMeV1( err = txs.processMiniBlockBuilderTx(mbBuilder, wtx, tx) if err != nil { - if chainErr.IsGetNodeFromDBError(err) { + if core.IsGetNodeFromDBError(err) { return nil, nil, err } continue @@ -1298,7 +1297,7 @@ func (txs *transactions) handleBadTransaction( ) { log.Trace("bad tx", "error", err.Error(), "hash", wtx.TxHash) errRevert := txs.accounts.RevertToSnapshot(snapshot) - if errRevert != nil && !chainErr.IsClosingError(errRevert) { + if errRevert != nil && !core.IsClosingError(errRevert) { log.Warn("revert to snapshot", "error", err.Error()) } diff --git a/process/block/preprocess/transactionsV2.go b/process/block/preprocess/transactionsV2.go index 430aa373028..d94434965d7 100644 --- a/process/block/preprocess/transactionsV2.go +++ b/process/block/preprocess/transactionsV2.go @@ -9,7 +9,6 @@ import ( "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-go/common" - chainErr "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/storage/txcache" ) @@ -71,7 +70,7 @@ func (txs *transactions) createAndProcessMiniBlocksFromMeV2( receiverShardID, mbInfo) if err != nil { - if chainErr.IsGetNodeFromDBError(err) { + if core.IsGetNodeFromDBError(err) { return nil, nil, nil, err } if shouldAddToRemaining { @@ -189,7 +188,7 @@ func (txs *transactions) processTransaction( log.Trace("bad tx", "error", err.Error(), "hash", txHash) errRevert := txs.accounts.RevertToSnapshot(snapshot) - if errRevert != nil && !chainErr.IsClosingError(errRevert) { + if errRevert != nil && !core.IsClosingError(errRevert) { log.Warn("revert to snapshot", "error", errRevert.Error()) } @@ -316,7 +315,7 @@ func (txs *transactions) createScheduledMiniBlocks( receiverShardID, mbInfo) if err != nil { - if chainErr.IsGetNodeFromDBError(err) { + if core.IsGetNodeFromDBError(err) { return nil, err } continue diff --git a/process/block/shardblock.go b/process/block/shardblock.go index dd1d36fcd0f..9ab6c655780 100644 --- a/process/block/shardblock.go +++ b/process/block/shardblock.go @@ -13,7 +13,6 @@ import ( "github.com/multiversx/mx-chain-core-go/data/headerVersionData" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" - "github.com/multiversx/mx-chain-go/errors" processOutport "github.com/multiversx/mx-chain-go/outport/process" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/block/bootstrapStorage" @@ -2028,7 +2027,7 @@ func (sp *shardProcessor) createMiniBlocks(haveTime func() bool, randomness []by if err != nil { log.Debug("createAndProcessCrossMiniBlocksDstMe", "error", err.Error()) - if errors.IsGetNodeFromDBError(err) { + if core.IsGetNodeFromDBError(err) { return nil, nil, err } } diff --git a/process/coordinator/process.go b/process/coordinator/process.go index 5a1a685a478..29ca913d13f 100644 --- a/process/coordinator/process.go +++ b/process/coordinator/process.go @@ -16,7 +16,6 @@ import ( "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" - "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/block/preprocess" "github.com/multiversx/mx-chain-go/process/block/processedMb" @@ -721,7 +720,7 @@ func (tc *transactionCoordinator) CreateMbsAndProcessCrossShardTransactionsDstMe "error", errProc, ) - if errors.IsGetNodeFromDBError(errProc) { + if core.IsGetNodeFromDBError(errProc) { return nil, 0, false, err } diff --git a/process/rewardTransaction/process.go b/process/rewardTransaction/process.go index fed488bb606..e641ef5d0cd 100644 --- a/process/rewardTransaction/process.go +++ b/process/rewardTransaction/process.go @@ -6,7 +6,6 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/rewardTx" - "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/state" @@ -120,7 +119,7 @@ func (rtp *rewardTxProcessor) saveAccumulatedRewards( existingReward.SetBytes(val) } - if errors.IsGetNodeFromDBError(err) { + if core.IsGetNodeFromDBError(err) { return err } diff --git a/process/smartContract/process.go b/process/smartContract/process.go index 361a5ebd4b3..620dafae8b2 100644 --- a/process/smartContract/process.go +++ b/process/smartContract/process.go @@ -19,7 +19,6 @@ import ( "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" - "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/state" @@ -339,7 +338,7 @@ func (sc *scProcessor) doExecuteSmartContractTransaction( var results []data.TransactionHandler results, err = sc.processVMOutput(vmOutput, txHash, tx, vmInput.CallType, vmInput.GasProvided) if err != nil { - if errors.IsGetNodeFromDBError(err) { + if core.IsGetNodeFromDBError(err) { return vmcommon.ExecutionFailed, err } log.Trace("process vm output returned with problem ", "err", err.Error()) @@ -381,7 +380,7 @@ func (sc *scProcessor) executeSmartContractCall( vmOutput, err = vmExec.RunSmartContractCall(vmInput) sc.wasmVMChangeLocker.RUnlock() if err != nil { - if errors.IsGetNodeFromDBError(err) { + if core.IsGetNodeFromDBError(err) { return nil, err } log.Debug("run smart contract call error", "error", err.Error()) @@ -984,7 +983,7 @@ func (sc *scProcessor) doExecuteBuiltInFunction( tmpCreatedAsyncCallback := false tmpCreatedAsyncCallback, newSCRTxs, err = sc.processSCOutputAccounts(newVMOutput, vmInput.CallType, outPutAccounts, tx, txHash) if err != nil { - if errors.IsGetNodeFromDBError(err) { + if core.IsGetNodeFromDBError(err) { return vmcommon.ExecutionFailed, err } return vmcommon.ExecutionFailed, sc.ProcessIfError(acntSnd, txHash, tx, err.Error(), []byte(err.Error()), snapshot, vmInput.GasLocked) @@ -1086,7 +1085,7 @@ func (sc *scProcessor) resolveBuiltInFunctions( GasRemaining: 0, } - if errors.IsGetNodeFromDBError(err) { + if core.IsGetNodeFromDBError(err) { return nil, err } @@ -1376,7 +1375,7 @@ func (sc *scProcessor) processIfErrorWithAddedLogs( err := sc.accounts.RevertToSnapshot(snapshot) if err != nil { - if !errors.IsClosingError(err) { + if !core.IsClosingError(err) { log.Warn("revert to snapshot", "error", err.Error()) } @@ -1730,7 +1729,7 @@ func (sc *scProcessor) doDeploySmartContract( sc.wasmVMChangeLocker.RUnlock() if err != nil { log.Debug("VM error", "error", err.Error()) - if errors.IsGetNodeFromDBError(err) { + if core.IsGetNodeFromDBError(err) { return vmcommon.ExecutionFailed, err } return vmcommon.UserError, sc.ProcessIfError(acntSnd, txHash, tx, err.Error(), []byte(""), snapshot, vmInput.GasLocked) @@ -1755,7 +1754,7 @@ func (sc *scProcessor) doDeploySmartContract( results, err := sc.processVMOutput(vmOutput, txHash, tx, vmInput.CallType, vmInput.GasProvided) if err != nil { log.Trace("Processing error", "error", err.Error()) - if errors.IsGetNodeFromDBError(err) { + if core.IsGetNodeFromDBError(err) { return vmcommon.ExecutionFailed, err } return vmcommon.ExecutionFailed, sc.ProcessIfError(acntSnd, txHash, tx, err.Error(), []byte(vmOutput.ReturnMessage), snapshot, vmInput.GasLocked) diff --git a/process/sync/metablock.go b/process/sync/metablock.go index c71d95fe321..d8ca3cf4954 100644 --- a/process/sync/metablock.go +++ b/process/sync/metablock.go @@ -9,7 +9,6 @@ import ( "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-go/dataRetriever" - "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/storage" @@ -180,7 +179,7 @@ func (boot *MetaBootstrap) setLastEpochStartRound() { // in the blockchain, and all this mechanism will be reiterated for the next block. func (boot *MetaBootstrap) SyncBlock(ctx context.Context) error { err := boot.syncBlock() - if errors.IsGetNodeFromDBError(err) { + if core.IsGetNodeFromDBError(err) { getNodeErr, ok := err.(getKeyHandler) if !ok { return err diff --git a/process/sync/shardblock.go b/process/sync/shardblock.go index be48c42de00..68a3c70f52d 100644 --- a/process/sync/shardblock.go +++ b/process/sync/shardblock.go @@ -9,7 +9,6 @@ import ( "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-go/dataRetriever" - "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/storage" ) @@ -143,7 +142,7 @@ func (boot *ShardBootstrap) StartSyncingBlocks() { // in the blockchain, and all this mechanism will be reiterated for the next block. func (boot *ShardBootstrap) SyncBlock(ctx context.Context) error { err := boot.syncBlock() - if errors.IsGetNodeFromDBError(err) { + if core.IsGetNodeFromDBError(err) { getNodeErr, ok := err.(getKeyHandler) if !ok { return err diff --git a/process/transaction/shardProcess.go b/process/transaction/shardProcess.go index aeeec86b2e8..7ebb6faa014 100644 --- a/process/transaction/shardProcess.go +++ b/process/transaction/shardProcess.go @@ -16,7 +16,6 @@ import ( "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" - errorsCommon "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/state" @@ -218,7 +217,7 @@ func (txProc *txProcessor) executeAfterFailedMoveBalanceTransaction( tx *transaction.Transaction, txError error, ) error { - if errorsCommon.IsGetNodeFromDBError(txError) { + if core.IsGetNodeFromDBError(txError) { return txError } diff --git a/sharding/nodesCoordinator/indexHashedNodesCoordinator.go b/sharding/nodesCoordinator/indexHashedNodesCoordinator.go index fbdfe04914b..6840eec6cd1 100644 --- a/sharding/nodesCoordinator/indexHashedNodesCoordinator.go +++ b/sharding/nodesCoordinator/indexHashedNodesCoordinator.go @@ -17,7 +17,6 @@ import ( "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/epochStart" - "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/storage" logger "github.com/multiversx/mx-chain-logger-go" @@ -810,7 +809,7 @@ func (ihnc *indexHashedNodesCoordinator) handleErrorLog(err error, message strin } logLevel := logger.LogError - if errors.IsClosingError(err) { + if core.IsClosingError(err) { logLevel = logger.LogDebug } diff --git a/state/accountsDB.go b/state/accountsDB.go index 2c6b57c2559..289f940cfd7 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -17,7 +17,6 @@ import ( "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/holders" - "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/trie/keyBuilder" "github.com/multiversx/mx-chain-go/trie/statistics" logger "github.com/multiversx/mx-chain-logger-go" @@ -219,7 +218,7 @@ func handleLoggingWhenError(message string, err error, extraArguments ...interfa if err == nil { return } - if errors.IsClosingError(err) { + if core.IsClosingError(err) { args := []interface{}{"reason", err} log.Debug(message, append(args, extraArguments...)...) return diff --git a/state/storagePruningManager/storagePruningManager.go b/state/storagePruningManager/storagePruningManager.go index 73d9af30847..c985a5378ab 100644 --- a/state/storagePruningManager/storagePruningManager.go +++ b/state/storagePruningManager/storagePruningManager.go @@ -7,7 +7,6 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-go/common" - "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/storagePruningManager/pruningBuffer" logger "github.com/multiversx/mx-chain-logger-go" @@ -175,7 +174,7 @@ func (spm *storagePruningManager) prune(rootHash []byte, tsm common.StorageManag err := spm.removeFromDb(rootHash, tsm, handler) if err != nil { - if errors.IsClosingError(err) { + if core.IsClosingError(err) { log.Debug("did not remove hash", "rootHash", rootHash, "error", err) return } diff --git a/trie/node.go b/trie/node.go index 190bb41dd9b..b464d7ff510 100644 --- a/trie/node.go +++ b/trie/node.go @@ -7,6 +7,7 @@ import ( "runtime/debug" "time" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" @@ -269,7 +270,7 @@ func shouldStopIfContextDoneBlockingIfBusy(ctx context.Context, idleProvider Idl } func treatCommitSnapshotError(err error, hash []byte, missingNodesChan chan []byte) { - if errors.IsClosingError(err) { + if core.IsClosingError(err) { log.Debug("context closing", "hash", hash) return } diff --git a/trie/patriciaMerkleTrie.go b/trie/patriciaMerkleTrie.go index dbe5e9fa7a0..b3fa3019dd3 100644 --- a/trie/patriciaMerkleTrie.go +++ b/trie/patriciaMerkleTrie.go @@ -7,12 +7,12 @@ import ( "fmt" "sync" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" - "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/trie/statistics" logger "github.com/multiversx/mx-chain-logger-go" ) @@ -276,7 +276,7 @@ func (tr *patriciaMerkleTrie) recreate(root []byte, tsm common.StorageManager) ( newTr, _, err := tr.recreateFromDb(root, tsm) if err != nil { - if errors.IsClosingError(err) { + if core.IsClosingError(err) { log.Debug("could not recreate", "rootHash", root, "error", err) return nil, err } diff --git a/trie/snapshotTrieStorageManager.go b/trie/snapshotTrieStorageManager.go index 5fe208be6df..784533fec3a 100644 --- a/trie/snapshotTrieStorageManager.go +++ b/trie/snapshotTrieStorageManager.go @@ -41,7 +41,7 @@ func (stsm *snapshotTrieStorageManager) Get(key []byte) ([]byte, error) { // test point get during snapshot val, epoch, err := stsm.mainSnapshotStorer.GetFromOldEpochsWithoutAddingToCache(key) - if errors.IsClosingError(err) { + if core.IsClosingError(err) { return nil, err } if len(val) != 0 { diff --git a/trie/trieStorageManager.go b/trie/trieStorageManager.go index 5ac0979f943..b4c5e6e857c 100644 --- a/trie/trieStorageManager.go +++ b/trie/trieStorageManager.go @@ -180,7 +180,7 @@ func (tsm *trieStorageManager) Get(key []byte) ([]byte, error) { } val, err := tsm.mainStorer.Get(key) - if errors.IsClosingError(err) { + if core.IsClosingError(err) { return nil, err } if len(val) != 0 { @@ -214,7 +214,7 @@ func (tsm *trieStorageManager) GetFromCurrentEpoch(key []byte) ([]byte, error) { func (tsm *trieStorageManager) getFromOtherStorers(key []byte) ([]byte, error) { val, err := tsm.checkpointsStorer.Get(key) - if errors.IsClosingError(err) { + if core.IsClosingError(err) { return nil, err } if len(val) != 0 { @@ -516,7 +516,7 @@ func (tsm *trieStorageManager) takeCheckpoint(checkpointEntry *snapshotsQueueEnt } func treatSnapshotError(err error, message string, rootHash []byte, mainTrieRootHash []byte) { - if errors.IsClosingError(err) { + if core.IsClosingError(err) { log.Debug("context closing", "message", message, "rootHash", rootHash, "mainTrieRootHash", mainTrieRootHash) return } diff --git a/trie/trieStorageManagerInEpoch.go b/trie/trieStorageManagerInEpoch.go index fee9a9dad31..c4cc7c9d195 100644 --- a/trie/trieStorageManagerInEpoch.go +++ b/trie/trieStorageManagerInEpoch.go @@ -3,6 +3,7 @@ package trie import ( "fmt" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/errors" @@ -73,7 +74,7 @@ func treatGetFromEpochError(err error, epoch uint32) { return } - if errors.IsClosingError(err) { + if core.IsClosingError(err) { log.Debug("trieStorageManagerInEpoch closing err", "error", err.Error(), "epoch", epoch) return } From 2a46bcca585028d651e4fcd81dd4b272f2565532 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Wed, 19 Apr 2023 11:39:44 +0300 Subject: [PATCH 33/33] fix after review --- common/constants.go | 3 -- errors/closingError.go | 17 ---------- errors/errors.go | 7 ++-- errors/missingTrieNodeError.go | 7 ++-- .../preprocess/rewardTxPreProcessor_test.go | 3 +- .../preprocess/smartContractResults_test.go | 3 +- process/block/preprocess/transactions.go | 5 --- .../block/preprocess/transactionsV2_test.go | 3 +- process/block/preprocess/transactions_test.go | 2 +- process/block/shardblock.go | 4 --- process/coordinator/process.go | 4 --- process/rewardTransaction/process_test.go | 3 +- process/sync/export_test.go | 5 +++ process/sync/interface.go | 1 + process/sync/metablock.go | 4 +-- process/sync/shardblock.go | 20 +++++++++-- process/sync/shardblock_test.go | 20 +++++++++++ state/accountsDB.go | 3 +- trie/branchNode.go | 7 ++-- trie/branchNode_test.go | 8 ++--- trie/depthFirstSync.go | 3 +- trie/depthFirstSync_test.go | 3 +- trie/doubleListSync.go | 3 +- trie/doubleListSync_test.go | 3 +- trie/extensionNode.go | 7 ++-- trie/extensionNode_test.go | 8 ++--- trie/leafNode.go | 5 ++- trie/leafNode_test.go | 5 ++- trie/node.go | 6 ++-- trie/patriciaMerkleTrie.go | 2 +- trie/snapshotTrieStorageManager.go | 7 ++-- trie/sync.go | 3 +- trie/trieStorageManager.go | 13 ++++--- trie/trieStorageManagerInEpoch.go | 3 +- trie/trieStorageManager_test.go | 34 +++++++++++++++++-- update/genesis/import.go | 4 +-- 36 files changed, 129 insertions(+), 109 deletions(-) delete mode 100644 errors/closingError.go diff --git a/common/constants.go b/common/constants.go index dcf620095ac..71306020854 100644 --- a/common/constants.go +++ b/common/constants.go @@ -768,9 +768,6 @@ const HardforkResolversIdentifier = "hardfork resolver" // EpochStartInterceptorsIdentifier represents the identifier that is used in the start-in-epoch process const EpochStartInterceptorsIdentifier = "epoch start interceptor" -// GetNodeFromDBErrorString represents the string which is returned when a getting node from DB returns an error -const GetNodeFromDBErrorString = "getNodeFromDB error" - // TimeoutGettingTrieNodes defines the timeout in trie sync operation if no node is received const TimeoutGettingTrieNodes = 2 * time.Minute // to consider syncing a very large trie node of 64MB at ~1MB/s diff --git a/errors/closingError.go b/errors/closingError.go deleted file mode 100644 index 81d051990b6..00000000000 --- a/errors/closingError.go +++ /dev/null @@ -1,17 +0,0 @@ -package errors - -import ( - "strings" - - "github.com/multiversx/mx-chain-go/storage" -) - -// IsClosingError returns true if the provided error is used whenever the node is in the closing process -func IsClosingError(err error) bool { - if err == nil { - return false - } - - return strings.Contains(err.Error(), storage.ErrDBIsClosed.Error()) || - strings.Contains(err.Error(), ErrContextClosing.Error()) -} diff --git a/errors/errors.go b/errors/errors.go index 255ac1362c2..b897f2f1a6f 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -1,6 +1,8 @@ package errors -import "errors" +import ( + "errors" +) // ErrAccountsAdapterCreation signals that the accounts adapter cannot be created based on provided data var ErrAccountsAdapterCreation = errors.New("error creating accounts adapter") @@ -470,9 +472,6 @@ var ErrNilScheduledTxsExecutionHandler = errors.New("nil scheduled transactions // ErrNilScheduledProcessor signals that a nil scheduled processor was provided var ErrNilScheduledProcessor = errors.New("nil scheduled processor") -// ErrContextClosing signals that the parent context requested the closing of its children -var ErrContextClosing = errors.New("context closing") - // ErrNilTxsSender signals that a nil transactions sender has been provided var ErrNilTxsSender = errors.New("nil transactions sender has been provided") diff --git a/errors/missingTrieNodeError.go b/errors/missingTrieNodeError.go index ebb92003085..7ddfefbfcbc 100644 --- a/errors/missingTrieNodeError.go +++ b/errors/missingTrieNodeError.go @@ -3,7 +3,8 @@ package errors import ( "encoding/hex" "fmt" - "github.com/multiversx/mx-chain-go/common" + + "github.com/multiversx/mx-chain-core-go/core" ) // GetNodeFromDBErrWithKey defines a custom error for trie get node @@ -13,7 +14,7 @@ type GetNodeFromDBErrWithKey struct { dbIdentifier string } -// NewGetNodeFromDBErrWithKey will create a new instance of GetNodeFromDBErr +// NewGetNodeFromDBErrWithKey will create a new instance of GetNodeFromDBErrWithKey func NewGetNodeFromDBErrWithKey(key []byte, err error, id string) *GetNodeFromDBErrWithKey { return &GetNodeFromDBErrWithKey{ getErr: err, @@ -26,7 +27,7 @@ func NewGetNodeFromDBErrWithKey(key []byte, err error, id string) *GetNodeFromDB func (e *GetNodeFromDBErrWithKey) Error() string { return fmt.Sprintf( "%s: %s for key %v", - common.GetNodeFromDBErrorString, + core.GetNodeFromDBErrorString, e.getErr.Error(), hex.EncodeToString(e.key), ) diff --git a/process/block/preprocess/rewardTxPreProcessor_test.go b/process/block/preprocess/rewardTxPreProcessor_test.go index 9871ba22081..80b29223e34 100644 --- a/process/block/preprocess/rewardTxPreProcessor_test.go +++ b/process/block/preprocess/rewardTxPreProcessor_test.go @@ -9,7 +9,6 @@ import ( "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/rewardTx" - "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/mock" @@ -684,7 +683,7 @@ func TestRewardTxPreprocessor_ProcessBlockTransactions(t *testing.T) { func TestRewardTxPreprocessor_ProcessBlockTransactionsMissingTrieNode(t *testing.T) { t.Parallel() - missingNodeErr := fmt.Errorf(common.GetNodeFromDBErrorString) + missingNodeErr := fmt.Errorf(core.GetNodeFromDBErrorString) txHash := testTxHash tdp := initDataPool() rtp, _ := NewRewardTxPreprocessor( diff --git a/process/block/preprocess/smartContractResults_test.go b/process/block/preprocess/smartContractResults_test.go index 1834156392e..af8b9ffaa14 100644 --- a/process/block/preprocess/smartContractResults_test.go +++ b/process/block/preprocess/smartContractResults_test.go @@ -12,7 +12,6 @@ import ( "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/smartContractResult" - "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/mock" @@ -1200,7 +1199,7 @@ func TestScrsPreprocessor_ProcessBlockTransactionsShouldWork(t *testing.T) { func TestScrsPreprocessor_ProcessBlockTransactionsMissingTrieNode(t *testing.T) { t.Parallel() - missingNodeErr := fmt.Errorf(common.GetNodeFromDBErrorString) + missingNodeErr := fmt.Errorf(core.GetNodeFromDBErrorString) tdp := initDataPool() requestTransaction := func(shardID uint32, txHashes [][]byte) {} scrPreproc, _ := NewSmartContractResultPreprocessor( diff --git a/process/block/preprocess/transactions.go b/process/block/preprocess/transactions.go index deb7dacb733..645ac0d8cf0 100644 --- a/process/block/preprocess/transactions.go +++ b/process/block/preprocess/transactions.go @@ -1089,11 +1089,6 @@ func (txs *transactions) CreateAndProcessMiniBlocks(haveTime func() bool, random if err != nil { log.Debug("createAndProcessMiniBlocksFromMe", "error", err.Error()) - - if core.IsGetNodeFromDBError(err) { - return nil, err - } - return make(block.MiniBlockSlice, 0), nil } diff --git a/process/block/preprocess/transactionsV2_test.go b/process/block/preprocess/transactionsV2_test.go index 28aeff4c8a5..72624deafb5 100644 --- a/process/block/preprocess/transactionsV2_test.go +++ b/process/block/preprocess/transactionsV2_test.go @@ -11,7 +11,6 @@ import ( "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/smartContractResult" "github.com/multiversx/mx-chain-core-go/data/transaction" - "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/storage/txcache" @@ -757,7 +756,7 @@ func TestTransactions_CreateAndProcessMiniBlocksFromMeV2ShouldWork(t *testing.T) func TestTransactions_CreateAndProcessMiniBlocksFromMeV2MissingTrieNode(t *testing.T) { t.Parallel() - missingNodeErr := fmt.Errorf(common.GetNodeFromDBErrorString) + missingNodeErr := fmt.Errorf(core.GetNodeFromDBErrorString) preprocessor := createTransactionPreprocessor() preprocessor.txProcessor = &testscommon.TxProcessorMock{ ProcessTransactionCalled: func(transaction *transaction.Transaction) (vmcommon.ReturnCode, error) { diff --git a/process/block/preprocess/transactions_test.go b/process/block/preprocess/transactions_test.go index d51760840a7..69fd9d71c39 100644 --- a/process/block/preprocess/transactions_test.go +++ b/process/block/preprocess/transactions_test.go @@ -1156,7 +1156,7 @@ func TestTransactionPreprocessor_ProcessTxsToMeShouldUseCorrectSenderAndReceiver func TestTransactionPreprocessor_ProcessTxsToMeMissingTrieNode(t *testing.T) { t.Parallel() - missingNodeErr := fmt.Errorf(common.GetNodeFromDBErrorString) + missingNodeErr := fmt.Errorf(core.GetNodeFromDBErrorString) args := createDefaultTransactionsProcessorArgs() args.Accounts = &stateMock.AccountsStub{ diff --git a/process/block/shardblock.go b/process/block/shardblock.go index 9ab6c655780..14faf2a8507 100644 --- a/process/block/shardblock.go +++ b/process/block/shardblock.go @@ -2026,10 +2026,6 @@ func (sp *shardProcessor) createMiniBlocks(haveTime func() bool, randomness []by log.Debug("elapsed time to create mbs to me", "time", elapsedTime) if err != nil { log.Debug("createAndProcessCrossMiniBlocksDstMe", "error", err.Error()) - - if core.IsGetNodeFromDBError(err) { - return nil, nil, err - } } if createAndProcessMBsDestMeInfo != nil { processedMiniBlocksDestMeInfo = createAndProcessMBsDestMeInfo.allProcessedMiniBlocksInfo diff --git a/process/coordinator/process.go b/process/coordinator/process.go index 29ca913d13f..32e75824e5c 100644 --- a/process/coordinator/process.go +++ b/process/coordinator/process.go @@ -720,10 +720,6 @@ func (tc *transactionCoordinator) CreateMbsAndProcessCrossShardTransactionsDstMe "error", errProc, ) - if core.IsGetNodeFromDBError(errProc) { - return nil, 0, false, err - } - continue } diff --git a/process/rewardTransaction/process_test.go b/process/rewardTransaction/process_test.go index c54848addaf..97112e792b3 100644 --- a/process/rewardTransaction/process_test.go +++ b/process/rewardTransaction/process_test.go @@ -8,7 +8,6 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/rewardTx" - "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/process/rewardTransaction" @@ -218,7 +217,7 @@ func TestRewardTxProcessor_ProcessRewardTransactionShouldWork(t *testing.T) { func TestRewardTxProcessor_ProcessRewardTransactionMissingTrieNode(t *testing.T) { t.Parallel() - missingNodeErr := fmt.Errorf(common.GetNodeFromDBErrorString) + missingNodeErr := fmt.Errorf(core.GetNodeFromDBErrorString) accountsDb := &stateMock.AccountsStub{ LoadAccountCalled: func(address []byte) (vmcommon.AccountHandler, error) { acc, _ := state.NewUserAccount(address) diff --git a/process/sync/export_test.go b/process/sync/export_test.go index 719e7599f9f..dae5be09c68 100644 --- a/process/sync/export_test.go +++ b/process/sync/export_test.go @@ -288,3 +288,8 @@ func (boot *baseBootstrap) IsInImportMode() bool { func (boot *baseBootstrap) ProcessWaitTime() time.Duration { return boot.processWaitTime } + +// UnwrapGetNodeFromDBErr - +func UnwrapGetNodeFromDBErr(wrappedErr error) getKeyHandler { + return unwrapGetNodeFromDBErr(wrappedErr) +} diff --git a/process/sync/interface.go b/process/sync/interface.go index e87d9c537d6..fb7e11e3b5f 100644 --- a/process/sync/interface.go +++ b/process/sync/interface.go @@ -32,6 +32,7 @@ type forkDetector interface { // getKeyHandler defines the behaviour of a component that can provide a trie node key and identifier type getKeyHandler interface { + Error() string GetKey() []byte GetIdentifier() string } diff --git a/process/sync/metablock.go b/process/sync/metablock.go index d8ca3cf4954..6b820235074 100644 --- a/process/sync/metablock.go +++ b/process/sync/metablock.go @@ -180,8 +180,8 @@ func (boot *MetaBootstrap) setLastEpochStartRound() { func (boot *MetaBootstrap) SyncBlock(ctx context.Context) error { err := boot.syncBlock() if core.IsGetNodeFromDBError(err) { - getNodeErr, ok := err.(getKeyHandler) - if !ok { + getNodeErr := unwrapGetNodeFromDBErr(err) + if getNodeErr == nil { return err } diff --git a/process/sync/shardblock.go b/process/sync/shardblock.go index 68a3c70f52d..55f6c7f6d84 100644 --- a/process/sync/shardblock.go +++ b/process/sync/shardblock.go @@ -2,6 +2,7 @@ package sync import ( "context" + "errors" "math" "github.com/multiversx/mx-chain-core-go/core" @@ -143,8 +144,8 @@ func (boot *ShardBootstrap) StartSyncingBlocks() { func (boot *ShardBootstrap) SyncBlock(ctx context.Context) error { err := boot.syncBlock() if core.IsGetNodeFromDBError(err) { - getNodeErr, ok := err.(getKeyHandler) - if !ok { + getNodeErr := unwrapGetNodeFromDBErr(err) + if getNodeErr == nil { return err } @@ -155,6 +156,21 @@ func (boot *ShardBootstrap) SyncBlock(ctx context.Context) error { return err } +func unwrapGetNodeFromDBErr(wrappedErr error) getKeyHandler { + errWithKeyHandler, ok := wrappedErr.(getKeyHandler) + for !ok { + if wrappedErr == nil { + return nil + } + + err := errors.Unwrap(wrappedErr) + errWithKeyHandler, ok = err.(getKeyHandler) + wrappedErr = err + } + + return errWithKeyHandler +} + // Close closes the synchronization loop func (boot *ShardBootstrap) Close() error { if check.IfNil(boot.baseBootstrap) { diff --git a/process/sync/shardblock_test.go b/process/sync/shardblock_test.go index 7c3f2ab1d9c..b0175890e43 100644 --- a/process/sync/shardblock_test.go +++ b/process/sync/shardblock_test.go @@ -2156,3 +2156,23 @@ func TestShardBootstrap_NilInnerBootstrapperClose(t *testing.T) { bootstrapper := &sync.ShardBootstrap{} assert.Nil(t, bootstrapper.Close()) } + +func TestUnwrapGetNodeFromDBErr(t *testing.T) { + t.Parallel() + + key := []byte("key") + identifier := "identifier" + err := fmt.Errorf("key not found") + + getNodeFromDbErr := commonErrors.NewGetNodeFromDBErrWithKey(key, err, identifier) + wrappedErr1 := fmt.Errorf("wrapped error 1: %w", getNodeFromDbErr) + wrappedErr2 := fmt.Errorf("wrapped error 2: %w", wrappedErr1) + wrappedErr3 := fmt.Errorf("wrapped error 3: %w", wrappedErr2) + + assert.Nil(t, sync.UnwrapGetNodeFromDBErr(nil)) + assert.Nil(t, sync.UnwrapGetNodeFromDBErr(err)) + assert.Equal(t, getNodeFromDbErr, sync.UnwrapGetNodeFromDBErr(getNodeFromDbErr)) + assert.Equal(t, getNodeFromDbErr, sync.UnwrapGetNodeFromDBErr(wrappedErr1)) + assert.Equal(t, getNodeFromDbErr, sync.UnwrapGetNodeFromDBErr(wrappedErr2)) + assert.Equal(t, getNodeFromDbErr, sync.UnwrapGetNodeFromDBErr(wrappedErr3)) +} diff --git a/state/accountsDB.go b/state/accountsDB.go index 289f940cfd7..ef200093467 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -523,8 +523,7 @@ func (adb *AccountsDB) loadDataTrie(accountHandler baseAccountHandler, mainTrie dataTrie, err := mainTrie.Recreate(accountHandler.GetRootHash()) if err != nil { - log.Error("trie was not found for hash", "rootHash", accountHandler.GetRootHash(), "err", err) - return err + return fmt.Errorf("trie was not found for hash, rootHash = %s, err = %w", hex.EncodeToString(accountHandler.GetRootHash()), err) } accountHandler.SetDataTrie(dataTrie) diff --git a/trie/branchNode.go b/trie/branchNode.go index 3e6f26768b5..4b2591d0982 100644 --- a/trie/branchNode.go +++ b/trie/branchNode.go @@ -13,7 +13,6 @@ import ( "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" - "github.com/multiversx/mx-chain-go/errors" ) var _ = node(&branchNode{}) @@ -299,7 +298,7 @@ func (bn *branchNode) commitCheckpoint( depthLevel int, ) error { if shouldStopIfContextDoneBlockingIfBusy(ctx, idleProvider) { - return errors.ErrContextClosing + return core.ErrContextClosing } err := bn.isEmptyOrNil() @@ -347,7 +346,7 @@ func (bn *branchNode) commitSnapshot( depthLevel int, ) error { if shouldStopIfContextDoneBlockingIfBusy(ctx, idleProvider) { - return errors.ErrContextClosing + return core.ErrContextClosing } err := bn.isEmptyOrNil() @@ -358,7 +357,7 @@ func (bn *branchNode) commitSnapshot( for i := range bn.children { err = resolveIfCollapsed(bn, byte(i), db) if err != nil { - if strings.Contains(err.Error(), common.GetNodeFromDBErrorString) { + if strings.Contains(err.Error(), core.GetNodeFromDBErrorString) { treatCommitSnapshotError(err, bn.EncodedChildren[i], missingNodesChan) continue } diff --git a/trie/branchNode_test.go b/trie/branchNode_test.go index a121e8b21aa..70665720bb4 100644 --- a/trie/branchNode_test.go +++ b/trie/branchNode_test.go @@ -7,12 +7,12 @@ import ( "fmt" "testing" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/mock" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/config" - chainErrors "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" @@ -1352,10 +1352,10 @@ func TestBranchNode_commitContextDone(t *testing.T) { cancel() err := bn.commitCheckpoint(db, db, nil, nil, ctx, statistics.NewTrieStatistics(), &testscommon.ProcessStatusHandlerStub{}, 0) - assert.Equal(t, chainErrors.ErrContextClosing, err) + assert.Equal(t, core.ErrContextClosing, err) err = bn.commitSnapshot(db, nil, nil, ctx, statistics.NewTrieStatistics(), &testscommon.ProcessStatusHandlerStub{}, 0) - assert.Equal(t, chainErrors.ErrContextClosing, err) + assert.Equal(t, core.ErrContextClosing, err) } func TestBranchNode_commitSnapshotDbIsClosing(t *testing.T) { @@ -1363,7 +1363,7 @@ func TestBranchNode_commitSnapshotDbIsClosing(t *testing.T) { db := &mock.StorerStub{ GetCalled: func(key []byte) ([]byte, error) { - return nil, chainErrors.ErrContextClosing + return nil, core.ErrContextClosing }, } _, collapsedBn := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) diff --git a/trie/depthFirstSync.go b/trie/depthFirstSync.go index 5f2d088fc7d..a26e0bdfe2a 100644 --- a/trie/depthFirstSync.go +++ b/trie/depthFirstSync.go @@ -9,7 +9,6 @@ import ( "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" - "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/storage" ) @@ -104,7 +103,7 @@ func (d *depthFirstTrieSyncer) StartSyncing(rootHash []byte, ctx context.Context case <-time.After(d.waitTimeBetweenChecks): continue case <-ctx.Done(): - return errors.ErrContextClosing + return core.ErrContextClosing } } } diff --git a/trie/depthFirstSync_test.go b/trie/depthFirstSync_test.go index 6ace7fbdb3f..95ea4db62aa 100644 --- a/trie/depthFirstSync_test.go +++ b/trie/depthFirstSync_test.go @@ -10,7 +10,6 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-go/common" - "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/storage" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -80,7 +79,7 @@ func TestDepthFirstTrieSyncer_StartSyncingCanTimeout(t *testing.T) { defer cancelFunc() err := d.StartSyncing(roothash, ctx) - require.Equal(t, errors.ErrContextClosing, err) + require.Equal(t, core.ErrContextClosing, err) } func TestDepthFirstTrieSyncer_StartSyncingTimeoutNoNodesReceived(t *testing.T) { diff --git a/trie/doubleListSync.go b/trie/doubleListSync.go index 6477023c7d2..b855e942d28 100644 --- a/trie/doubleListSync.go +++ b/trie/doubleListSync.go @@ -9,7 +9,6 @@ import ( "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" - "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/storage" ) @@ -121,7 +120,7 @@ func (d *doubleListTrieSyncer) StartSyncing(rootHash []byte, ctx context.Context case <-time.After(d.waitTimeBetweenChecks): continue case <-ctx.Done(): - return errors.ErrContextClosing + return core.ErrContextClosing } } } diff --git a/trie/doubleListSync_test.go b/trie/doubleListSync_test.go index 719d578e5c6..a1d67e102fa 100644 --- a/trie/doubleListSync_test.go +++ b/trie/doubleListSync_test.go @@ -11,7 +11,6 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/config" - "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/database" "github.com/multiversx/mx-chain-go/storage/storageunit" @@ -185,7 +184,7 @@ func TestDoubleListTrieSyncer_StartSyncingCanTimeout(t *testing.T) { defer cancelFunc() err := d.StartSyncing(roothash, ctx) - require.Equal(t, errors.ErrContextClosing, err) + require.Equal(t, core.ErrContextClosing, err) } func TestDoubleListTrieSyncer_StartSyncingTimeoutNoNodesReceived(t *testing.T) { diff --git a/trie/extensionNode.go b/trie/extensionNode.go index 8130a761233..79d8e16cdad 100644 --- a/trie/extensionNode.go +++ b/trie/extensionNode.go @@ -14,7 +14,6 @@ import ( "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" - "github.com/multiversx/mx-chain-go/errors" ) var _ = node(&extensionNode{}) @@ -211,7 +210,7 @@ func (en *extensionNode) commitCheckpoint( depthLevel int, ) error { if shouldStopIfContextDoneBlockingIfBusy(ctx, idleProvider) { - return errors.ErrContextClosing + return core.ErrContextClosing } err := en.isEmptyOrNil() @@ -253,7 +252,7 @@ func (en *extensionNode) commitSnapshot( depthLevel int, ) error { if shouldStopIfContextDoneBlockingIfBusy(ctx, idleProvider) { - return errors.ErrContextClosing + return core.ErrContextClosing } err := en.isEmptyOrNil() @@ -264,7 +263,7 @@ func (en *extensionNode) commitSnapshot( err = resolveIfCollapsed(en, 0, db) isMissingNodeErr := false if err != nil { - isMissingNodeErr = strings.Contains(err.Error(), common.GetNodeFromDBErrorString) + isMissingNodeErr = strings.Contains(err.Error(), core.GetNodeFromDBErrorString) if !isMissingNodeErr { return err } diff --git a/trie/extensionNode_test.go b/trie/extensionNode_test.go index cc8dd806d2c..34b87199e99 100644 --- a/trie/extensionNode_test.go +++ b/trie/extensionNode_test.go @@ -6,9 +6,9 @@ import ( "errors" "testing" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/mock" "github.com/multiversx/mx-chain-go/common" - chainErrors "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" @@ -1021,10 +1021,10 @@ func TestExtensionNode_commitContextDone(t *testing.T) { cancel() err := en.commitCheckpoint(db, db, nil, nil, ctx, statistics.NewTrieStatistics(), &testscommon.ProcessStatusHandlerStub{}, 0) - assert.Equal(t, chainErrors.ErrContextClosing, err) + assert.Equal(t, core.ErrContextClosing, err) err = en.commitSnapshot(db, nil, nil, ctx, statistics.NewTrieStatistics(), &testscommon.ProcessStatusHandlerStub{}, 0) - assert.Equal(t, chainErrors.ErrContextClosing, err) + assert.Equal(t, core.ErrContextClosing, err) } func TestExtensionNode_getValueReturnsEmptyByteSlice(t *testing.T) { @@ -1039,7 +1039,7 @@ func TestExtensionNode_commitSnapshotDbIsClosing(t *testing.T) { db := &mock.StorerStub{ GetCalled: func(key []byte) ([]byte, error) { - return nil, chainErrors.ErrContextClosing + return nil, core.ErrContextClosing }, } _, collapsedEn := getEnAndCollapsedEn() diff --git a/trie/leafNode.go b/trie/leafNode.go index cb4c4bfdc76..ed037c7f0e0 100644 --- a/trie/leafNode.go +++ b/trie/leafNode.go @@ -14,7 +14,6 @@ import ( "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" - "github.com/multiversx/mx-chain-go/errors" ) var _ = node(&leafNode{}) @@ -139,7 +138,7 @@ func (ln *leafNode) commitCheckpoint( depthLevel int, ) error { if shouldStopIfContextDoneBlockingIfBusy(ctx, idleProvider) { - return errors.ErrContextClosing + return core.ErrContextClosing } err := ln.isEmptyOrNil() @@ -184,7 +183,7 @@ func (ln *leafNode) commitSnapshot( depthLevel int, ) error { if shouldStopIfContextDoneBlockingIfBusy(ctx, idleProvider) { - return errors.ErrContextClosing + return core.ErrContextClosing } err := ln.isEmptyOrNil() diff --git a/trie/leafNode_test.go b/trie/leafNode_test.go index b534e700d44..bf9cab8209b 100644 --- a/trie/leafNode_test.go +++ b/trie/leafNode_test.go @@ -8,7 +8,6 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" - chainErrors "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" @@ -733,10 +732,10 @@ func TestLeafNode_commitContextDone(t *testing.T) { cancel() err := ln.commitCheckpoint(db, db, nil, nil, ctx, statistics.NewTrieStatistics(), &testscommon.ProcessStatusHandlerStub{}, 0) - assert.Equal(t, chainErrors.ErrContextClosing, err) + assert.Equal(t, core.ErrContextClosing, err) err = ln.commitSnapshot(db, nil, nil, ctx, statistics.NewTrieStatistics(), &testscommon.ProcessStatusHandlerStub{}, 0) - assert.Equal(t, chainErrors.ErrContextClosing, err) + assert.Equal(t, core.ErrContextClosing, err) } func TestLeafNode_getValue(t *testing.T) { diff --git a/trie/node.go b/trie/node.go index b464d7ff510..9a127a7ecf7 100644 --- a/trie/node.go +++ b/trie/node.go @@ -120,12 +120,12 @@ func computeAndSetNodeHash(n node) ([]byte, error) { func getNodeFromDBAndDecode(n []byte, db common.DBWriteCacher, marshalizer marshal.Marshalizer, hasher hashing.Hasher) (node, error) { encChild, err := db.Get(n) if err != nil { - log.Trace(common.GetNodeFromDBErrorString, "error", err, "key", n, "stack trace", string(debug.Stack())) + log.Trace(core.GetNodeFromDBErrorString, "error", err, "key", n, "stack trace", string(debug.Stack())) dbWithID, ok := db.(dbWriteCacherWithIdentifier) if !ok { - log.Warn("db does not have an identifier", "db type", fmt.Sprintf("%T", db)) - return nil, errors.NewGetNodeFromDBErrWithKey(n, err, "") + getNodeFromDbErr := errors.NewGetNodeFromDBErrWithKey(n, err, "") + return nil, fmt.Errorf("db does not have an identifier, db type: %T, error: %w", db, getNodeFromDbErr) } return nil, errors.NewGetNodeFromDBErrWithKey(n, err, dbWithID.GetIdentifier()) diff --git a/trie/patriciaMerkleTrie.go b/trie/patriciaMerkleTrie.go index b3fa3019dd3..75b035966af 100644 --- a/trie/patriciaMerkleTrie.go +++ b/trie/patriciaMerkleTrie.go @@ -88,7 +88,7 @@ func (tr *patriciaMerkleTrie) Get(key []byte) ([]byte, uint32, error) { val, depth, err := tr.root.tryGet(hexKey, rootDepthLevel, tr.trieStorage) if err != nil { - log.Error("trie get error", "error", err.Error(), "key", key) + err = fmt.Errorf("trie get error: %w, for key %v", err, hex.EncodeToString(key)) return nil, depth, err } diff --git a/trie/snapshotTrieStorageManager.go b/trie/snapshotTrieStorageManager.go index 784533fec3a..133cb9080e4 100644 --- a/trie/snapshotTrieStorageManager.go +++ b/trie/snapshotTrieStorageManager.go @@ -6,7 +6,6 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-go/common" - "github.com/multiversx/mx-chain-go/errors" ) type snapshotTrieStorageManager struct { @@ -35,7 +34,7 @@ func (stsm *snapshotTrieStorageManager) Get(key []byte) ([]byte, error) { if stsm.closed { log.Debug("snapshotTrieStorageManager get context closing", "key", key) - return nil, errors.ErrContextClosing + return nil, core.ErrContextClosing } // test point get during snapshot @@ -86,7 +85,7 @@ func (stsm *snapshotTrieStorageManager) Put(key, data []byte) error { if stsm.closed { log.Debug("snapshotTrieStorageManager put context closing", "key", key, "data", data) - return errors.ErrContextClosing + return core.ErrContextClosing } log.Trace("put hash in snapshot storer", "hash", key, "epoch", stsm.epoch) @@ -100,7 +99,7 @@ func (stsm *snapshotTrieStorageManager) GetFromLastEpoch(key []byte) ([]byte, er if stsm.closed { log.Debug("snapshotTrieStorageManager getFromLastEpoch context closing", "key", key) - return nil, errors.ErrContextClosing + return nil, core.ErrContextClosing } return stsm.mainSnapshotStorer.GetFromLastEpoch(key) diff --git a/trie/sync.go b/trie/sync.go index 465ebf71a99..2ef4bb807ca 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -13,7 +13,6 @@ import ( "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" - "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/storage" ) @@ -160,7 +159,7 @@ func (ts *trieSyncer) StartSyncing(rootHash []byte, ctx context.Context) error { case <-time.After(ts.waitTimeBetweenRequests): continue case <-ctx.Done(): - return errors.ErrContextClosing + return core.ErrContextClosing } } } diff --git a/trie/trieStorageManager.go b/trie/trieStorageManager.go index b4c5e6e857c..45c251d4983 100644 --- a/trie/trieStorageManager.go +++ b/trie/trieStorageManager.go @@ -16,7 +16,6 @@ import ( "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/config" - "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/trie/statistics" ) @@ -176,7 +175,7 @@ func (tsm *trieStorageManager) Get(key []byte) ([]byte, error) { if tsm.closed { log.Trace("trieStorageManager get context closing", "key", key) - return nil, errors.ErrContextClosing + return nil, core.ErrContextClosing } val, err := tsm.mainStorer.Get(key) @@ -197,7 +196,7 @@ func (tsm *trieStorageManager) GetFromCurrentEpoch(key []byte) ([]byte, error) { if tsm.closed { log.Trace("trieStorageManager get context closing", "key", key) tsm.storageOperationMutex.Unlock() - return nil, errors.ErrContextClosing + return nil, core.ErrContextClosing } storer, ok := tsm.mainStorer.(snapshotPruningStorer) @@ -232,7 +231,7 @@ func (tsm *trieStorageManager) Put(key []byte, val []byte) error { if tsm.closed { log.Trace("trieStorageManager put context closing", "key", key, "value", val) - return errors.ErrContextClosing + return core.ErrContextClosing } return tsm.mainStorer.Put(key, val) @@ -246,7 +245,7 @@ func (tsm *trieStorageManager) PutInEpoch(key []byte, val []byte, epoch uint32) if tsm.closed { log.Trace("trieStorageManager putInEpoch context closing", "key", key, "value", val, "epoch", epoch) - return errors.ErrContextClosing + return core.ErrContextClosing } storer, ok := tsm.mainStorer.(snapshotPruningStorer) @@ -265,7 +264,7 @@ func (tsm *trieStorageManager) PutInEpochWithoutCache(key []byte, val []byte, ep if tsm.closed { log.Trace("trieStorageManager putInEpochWithoutCache context closing", "key", key, "value", val, "epoch", epoch) - return errors.ErrContextClosing + return core.ErrContextClosing } storer, ok := tsm.mainStorer.(snapshotPruningStorer) @@ -533,7 +532,7 @@ func newSnapshotNode( ) (snapshotNode, error) { newRoot, err := getNodeFromDBAndDecode(rootHash, db, msh, hsh) if err != nil { - if strings.Contains(err.Error(), common.GetNodeFromDBErrorString) { + if strings.Contains(err.Error(), core.GetNodeFromDBErrorString) { treatCommitSnapshotError(err, rootHash, missingNodesCh) } return nil, err diff --git a/trie/trieStorageManagerInEpoch.go b/trie/trieStorageManagerInEpoch.go index c4cc7c9d195..5726290b3fd 100644 --- a/trie/trieStorageManagerInEpoch.go +++ b/trie/trieStorageManagerInEpoch.go @@ -6,7 +6,6 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-go/common" - "github.com/multiversx/mx-chain-go/errors" ) // numEpochsToVerify needs to be at least 2 due to a snapshotting edge-case. @@ -50,7 +49,7 @@ func (tsmie *trieStorageManagerInEpoch) Get(key []byte) ([]byte, error) { if tsmie.closed { log.Debug("trieStorageManagerInEpoch get context closing", "key", key) - return nil, errors.ErrContextClosing + return nil, core.ErrContextClosing } for i := uint32(0); i < numEpochsToVerify; i++ { diff --git a/trie/trieStorageManager_test.go b/trie/trieStorageManager_test.go index f634024514d..dc742c03afc 100644 --- a/trie/trieStorageManager_test.go +++ b/trie/trieStorageManager_test.go @@ -10,9 +10,9 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/config" - "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" + "github.com/multiversx/mx-chain-go/testscommon/storage" trieMock "github.com/multiversx/mx-chain-go/testscommon/trie" "github.com/multiversx/mx-chain-go/trie" "github.com/multiversx/mx-chain-go/trie/hashesHolder" @@ -248,7 +248,7 @@ func TestTrieStorageManager_PutInEpochClosedDb(t *testing.T) { key := []byte("key") value := []byte("value") err := ts.PutInEpoch(key, value, 0) - assert.Equal(t, errors.ErrContextClosing, err) + assert.Equal(t, core.ErrContextClosing, err) } func TestTrieStorageManager_PutInEpochInvalidStorer(t *testing.T) { @@ -390,7 +390,7 @@ func TestTrieStorageManager_TakeSnapshotWithGetNodeFromDBError(t *testing.T) { require.Equal(t, 1, len(iteratorChannels.ErrChan)) errRecovered := <-iteratorChannels.ErrChan - assert.True(t, strings.Contains(errRecovered.Error(), common.GetNodeFromDBErrorString)) + assert.True(t, strings.Contains(errRecovered.Error(), core.GetNodeFromDBErrorString)) } func TestTrieStorageManager_ShouldTakeSnapshotInvalidStorer(t *testing.T) { @@ -518,3 +518,31 @@ func TestWriteInChanNonBlocking(t *testing.T) { assert.Equal(t, err2, recovered) }) } + +func TestTrieStorageManager_GetIdentifier(t *testing.T) { + t.Parallel() + + t.Run("db without identifier", func(t *testing.T) { + t.Parallel() + + ts, _ := trie.NewTrieStorageManager(getNewTrieStorageManagerArgs()) + id := ts.GetIdentifier() + assert.Equal(t, "", id) + }) + + t.Run("db with identifier", func(t *testing.T) { + t.Parallel() + + expectedIdentifier := "identifier" + args := getNewTrieStorageManagerArgs() + args.MainStorer = &storage.StorerStub{ + GetIdentifierCalled: func() string { + return expectedIdentifier + }, + } + ts, _ := trie.NewTrieStorageManager(args) + + id := ts.GetIdentifier() + assert.Equal(t, expectedIdentifier, id) + }) +} diff --git a/update/genesis/import.go b/update/genesis/import.go index ce9f3c33f53..e740564c424 100644 --- a/update/genesis/import.go +++ b/update/genesis/import.go @@ -5,7 +5,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "github.com/multiversx/mx-chain-go/dataRetriever" "strings" "github.com/multiversx/mx-chain-core-go/core" @@ -17,6 +16,7 @@ import ( "github.com/multiversx/mx-chain-go/common" commonDisabled "github.com/multiversx/mx-chain-go/common/disabled" "github.com/multiversx/mx-chain-go/config" + "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/factory" "github.com/multiversx/mx-chain-go/state/storagePruningManager/disabled" @@ -292,7 +292,7 @@ func (si *stateImport) getTrie(shardID uint32, accType Type) (common.Trie, error return trieForShard, nil } - trieStorageManager := si.trieStorageManagers[dataRetriever.PeerAccountsUnit.String()] + trieStorageManager := si.trieStorageManagers[dataRetriever.UserAccountsUnit.String()] if accType == ValidatorAccount { trieStorageManager = si.trieStorageManagers[dataRetriever.PeerAccountsUnit.String()] }