Merge pull request #4724 from multiversx/sync-missing-keys-only

add custom error for trie get node

common/constants.go

@@ -768,9 +768,6 @@ const HardforkResolversIdentifier = "hardfork resolver"
 // EpochStartInterceptorsIdentifier represents the identifier that is used in the start-in-epoch process
 const EpochStartInterceptorsIdentifier = "epoch start interceptor"
 
-// GetNodeFromDBErrorString represents the string which is returned when a getting node from DB returns an error
-const GetNodeFromDBErrorString = "getNodeFromDB error"
-
 // TimeoutGettingTrieNodes defines the timeout in trie sync operation if no node is received
 const TimeoutGettingTrieNodes = 2 * time.Minute // to consider syncing a very large trie node of 64MB at ~1MB/s
 

common/interface.go

@@ -79,6 +79,7 @@ type StorageManager interface {
 	SetEpochForPutOperation(uint32)
 	ShouldTakeSnapshot() bool
 	GetBaseTrieStorageManager() StorageManager
+	GetIdentifier() string
 	IsClosed() bool
 	Close() error
 	IsInterfaceNil() bool

common/logging/errorLogging.go

@@ -1,8 +1,8 @@
 package logging
 
 import (
+	"github.com/multiversx/mx-chain-core-go/core"
 	"github.com/multiversx/mx-chain-core-go/core/check"
-	chainErrors "github.com/multiversx/mx-chain-go/errors"
 	logger "github.com/multiversx/mx-chain-logger-go"
 )
 
@@ -24,7 +24,7 @@ func logErrAsLevelExceptAsDebugIfClosingError(logInstance logger.Logger, logLeve
 		return
 	}
 
-	if chainErrors.IsClosingError(err) {
+	if core.IsClosingError(err) {
 		logLevel = logger.LogDebug
 	}
 

dataRetriever/factory/resolverscontainer/metaResolversContainerFactory.go

@@ -8,7 +8,6 @@ import (
 	"github.com/multiversx/mx-chain-go/dataRetriever"
 	"github.com/multiversx/mx-chain-go/dataRetriever/factory/containers"
 	"github.com/multiversx/mx-chain-go/dataRetriever/resolvers"
-	triesFactory "github.com/multiversx/mx-chain-go/trie/factory"
 
 	"github.com/multiversx/mx-chain-core-go/marshal"
 	"github.com/multiversx/mx-chain-go/process/factory"
@@ -142,7 +141,7 @@ func (mrcf *metaResolversContainerFactory) AddShardTrieNodeResolvers(container d
 		identifierTrieNodes := factory.AccountTrieNodesTopic + shardC.CommunicationIdentifier(idx)
 		resolver, err := mrcf.createTrieNodesResolver(
 			identifierTrieNodes,
-			triesFactory.UserAccountTrie,
+			dataRetriever.UserAccountsUnit.String(),
 			idx,
 		)
 		if err != nil {
@@ -295,7 +294,7 @@ func (mrcf *metaResolversContainerFactory) generateTrieNodesResolvers() error {
 	identifierTrieNodes := factory.AccountTrieNodesTopic + core.CommunicationIdentifierBetweenShards(core.MetachainShardId, core.MetachainShardId)
 	resolver, err := mrcf.createTrieNodesResolver(
 		identifierTrieNodes,
-		triesFactory.UserAccountTrie,
+		dataRetriever.UserAccountsUnit.String(),
 		core.MetachainShardId,
 	)
 	if err != nil {
@@ -308,7 +307,7 @@ func (mrcf *metaResolversContainerFactory) generateTrieNodesResolvers() error {
 	identifierTrieNodes = factory.ValidatorTrieNodesTopic + core.CommunicationIdentifierBetweenShards(core.MetachainShardId, core.MetachainShardId)
 	resolver, err = mrcf.createTrieNodesResolver(
 		identifierTrieNodes,
-		triesFactory.PeerAccountTrie,
+		dataRetriever.PeerAccountsUnit.String(),
 		core.MetachainShardId,
 	)
 	if err != nil {

dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go

@@ -18,7 +18,6 @@ import (
 	"github.com/multiversx/mx-chain-go/testscommon/p2pmocks"
 	storageStubs "github.com/multiversx/mx-chain-go/testscommon/storage"
 	trieMock "github.com/multiversx/mx-chain-go/testscommon/trie"
-	triesFactory "github.com/multiversx/mx-chain-go/trie/factory"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -82,8 +81,8 @@ func createStoreForMeta() dataRetriever.StorageService {
 
 func createTriesHolderForMeta() common.TriesHolder {
 	triesHolder := state.NewDataTriesHolder()
-	triesHolder.Put([]byte(triesFactory.UserAccountTrie), &trieMock.TrieStub{})
-	triesHolder.Put([]byte(triesFactory.PeerAccountTrie), &trieMock.TrieStub{})
+	triesHolder.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{})
+	triesHolder.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{})
 	return triesHolder
 }
 

dataRetriever/factory/resolverscontainer/shardResolversContainerFactory.go

@@ -9,7 +9,6 @@ import (
 	"github.com/multiversx/mx-chain-go/dataRetriever/factory/containers"
 	"github.com/multiversx/mx-chain-go/dataRetriever/resolvers"
 	"github.com/multiversx/mx-chain-go/process/factory"
-	triesFactory "github.com/multiversx/mx-chain-go/trie/factory"
 )
 
 var _ dataRetriever.ResolversContainerFactory = (*shardResolversContainerFactory)(nil)
@@ -232,7 +231,7 @@ func (srcf *shardResolversContainerFactory) generateTrieNodesResolvers() error {
 	identifierTrieNodes := factory.AccountTrieNodesTopic + shardC.CommunicationIdentifier(core.MetachainShardId)
 	resolver, err := srcf.createTrieNodesResolver(
 		identifierTrieNodes,
-		triesFactory.UserAccountTrie,
+		dataRetriever.UserAccountsUnit.String(),
 		core.MetachainShardId,
 	)
 	if err != nil {

dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go

@@ -18,7 +18,6 @@ import (
 	"github.com/multiversx/mx-chain-go/testscommon/p2pmocks"
 	storageStubs "github.com/multiversx/mx-chain-go/testscommon/storage"
 	trieMock "github.com/multiversx/mx-chain-go/testscommon/trie"
-	triesFactory "github.com/multiversx/mx-chain-go/trie/factory"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -88,8 +87,8 @@ func createStoreForShard() dataRetriever.StorageService {
 
 func createTriesHolderForShard() common.TriesHolder {
 	triesHolder := state.NewDataTriesHolder()
-	triesHolder.Put([]byte(triesFactory.UserAccountTrie), &trieMock.TrieStub{})
-	triesHolder.Put([]byte(triesFactory.PeerAccountTrie), &trieMock.TrieStub{})
+	triesHolder.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{})
+	triesHolder.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{})
 	return triesHolder
 }
 

epochStart/bootstrap/process.go

@@ -1061,7 +1061,7 @@ func (e *epochStartBootstrap) syncUserAccountsState(rootHash []byte) error {
 	}
 
 	e.mutTrieStorageManagers.RLock()
-	trieStorageManager := e.trieStorageManagers[factory.UserAccountTrie]
+	trieStorageManager := e.trieStorageManagers[dataRetriever.UserAccountsUnit.String()]
 	e.mutTrieStorageManagers.RUnlock()
 
 	argsUserAccountsSyncer := syncer.ArgsNewUserAccountsSyncer{
@@ -1132,7 +1132,7 @@ func (e *epochStartBootstrap) createStorageService(
 
 func (e *epochStartBootstrap) syncValidatorAccountsState(rootHash []byte) error {
 	e.mutTrieStorageManagers.RLock()
-	peerTrieStorageManager := e.trieStorageManagers[factory.PeerAccountTrie]
+	peerTrieStorageManager := e.trieStorageManagers[dataRetriever.PeerAccountsUnit.String()]
 	e.mutTrieStorageManagers.RUnlock()
 
 	argsValidatorAccountsSyncer := syncer.ArgsNewValidatorAccountsSyncer{

epochStart/shardchain/trigger.go

@@ -22,7 +22,6 @@ import (
 	"github.com/multiversx/mx-chain-go/common"
 	"github.com/multiversx/mx-chain-go/dataRetriever"
 	"github.com/multiversx/mx-chain-go/epochStart"
-	"github.com/multiversx/mx-chain-go/errors"
 	"github.com/multiversx/mx-chain-go/process"
 	"github.com/multiversx/mx-chain-go/storage"
 	"github.com/multiversx/mx-chain-logger-go"
@@ -997,7 +996,7 @@ func (t *trigger) SetProcessed(header data.HeaderHandler, _ data.BodyHandler) {
 	errNotCritical = t.shardHdrStorage.Put([]byte(epochStartIdentifier), shardHdrBuff)
 	if errNotCritical != nil {
 		logLevel := logger.LogWarning
-		if errors.IsClosingError(errNotCritical) {
+		if core.IsClosingError(errNotCritical) {
 			logLevel = logger.LogDebug
 		}
 		log.Log(logLevel, "SetProcessed put to shard header storage error", "error", errNotCritical)

errors/errors.go

@@ -1,6 +1,8 @@
 package errors
 
-import "errors"
+import (
+	"errors"
+)
 
 // ErrAccountsAdapterCreation signals that the accounts adapter cannot be created based on provided data
 var ErrAccountsAdapterCreation = errors.New("error creating accounts adapter")
@@ -470,9 +472,6 @@ var ErrNilScheduledTxsExecutionHandler = errors.New("nil scheduled transactions
 // ErrNilScheduledProcessor signals that a nil scheduled processor was provided
 var ErrNilScheduledProcessor = errors.New("nil scheduled processor")
 
-// ErrContextClosing signals that the parent context requested the closing of its children
-var ErrContextClosing = errors.New("context closing")
-
 // ErrNilTxsSender signals that a nil transactions sender has been provided
 var ErrNilTxsSender = errors.New("nil transactions sender has been provided")
 

errors/missingTrieNodeError.go

@@ -1,24 +1,44 @@
 package errors
 
 import (
-	"strings"
+	"encoding/hex"
+	"fmt"
 
-	"github.com/multiversx/mx-chain-go/common"
+	"github.com/multiversx/mx-chain-core-go/core"
 )
 
-// IsGetNodeFromDBError returns true if the provided error is of type getNodeFromDB
-func IsGetNodeFromDBError(err error) bool {
-	if err == nil {
-		return false
-	}
+// GetNodeFromDBErrWithKey defines a custom error for trie get node
+type GetNodeFromDBErrWithKey struct {
+	getErr       error
+	key          []byte
+	dbIdentifier string
+}
 
-	if IsClosingError(err) {
-		return false
+// NewGetNodeFromDBErrWithKey will create a new instance of GetNodeFromDBErrWithKey
+func NewGetNodeFromDBErrWithKey(key []byte, err error, id string) *GetNodeFromDBErrWithKey {
+	return &GetNodeFromDBErrWithKey{
+		getErr:       err,
+		key:          key,
+		dbIdentifier: id,
 	}
+}
 
-	if strings.Contains(err.Error(), common.GetNodeFromDBErrorString) {
-		return true
-	}
+// Error returns the error as string
+func (e *GetNodeFromDBErrWithKey) Error() string {
+	return fmt.Sprintf(
+		"%s: %s for key %v",
+		core.GetNodeFromDBErrorString,
+		e.getErr.Error(),
+		hex.EncodeToString(e.key),
+	)
+}
+
+// GetKey will return the key that generated the error
+func (e *GetNodeFromDBErrWithKey) GetKey() []byte {
+	return e.key
+}
 
-	return false
+// GetIdentifier will return the db identifier corresponding to the db
+func (e *GetNodeFromDBErrWithKey) GetIdentifier() string {
+	return e.dbIdentifier
 }

factory/consensus/consensusComponents.go

@@ -16,6 +16,7 @@ import (
 	"github.com/multiversx/mx-chain-go/consensus/chronology"
 	"github.com/multiversx/mx-chain-go/consensus/spos"
 	"github.com/multiversx/mx-chain-go/consensus/spos/sposFactory"
+	"github.com/multiversx/mx-chain-go/dataRetriever"
 	"github.com/multiversx/mx-chain-go/errors"
 	"github.com/multiversx/mx-chain-go/factory"
 	p2pFactory "github.com/multiversx/mx-chain-go/p2p/factory"
@@ -24,7 +25,6 @@ import (
 	"github.com/multiversx/mx-chain-go/process/sync/storageBootstrap"
 	"github.com/multiversx/mx-chain-go/sharding"
 	"github.com/multiversx/mx-chain-go/state/syncer"
-	trieFactory "github.com/multiversx/mx-chain-go/trie/factory"
 	"github.com/multiversx/mx-chain-go/trie/statistics"
 	"github.com/multiversx/mx-chain-go/trie/storageMarker"
 	"github.com/multiversx/mx-chain-go/update"
@@ -536,7 +536,7 @@ func (ccf *consensusComponentsFactory) createArgsBaseAccountsSyncer(trieStorageM
 }
 
 func (ccf *consensusComponentsFactory) createValidatorAccountsSyncer() (process.AccountsDBSyncer, error) {
-	trieStorageManager, ok := ccf.stateComponents.TrieStorageManagers()[trieFactory.PeerAccountTrie]
+	trieStorageManager, ok := ccf.stateComponents.TrieStorageManagers()[dataRetriever.PeerAccountsUnit.String()]
 	if !ok {
 		return nil, errors.ErrNilTrieStorageManager
 	}
@@ -548,7 +548,7 @@ func (ccf *consensusComponentsFactory) createValidatorAccountsSyncer() (process.
 }
 
 func (ccf *consensusComponentsFactory) createUserAccountsSyncer() (process.AccountsDBSyncer, error) {
-	trieStorageManager, ok := ccf.stateComponents.TrieStorageManagers()[trieFactory.UserAccountTrie]
+	trieStorageManager, ok := ccf.stateComponents.TrieStorageManagers()[dataRetriever.UserAccountsUnit.String()]
 	if !ok {
 		return nil, errors.ErrNilTrieStorageManager
 	}

factory/processing/blockProcessorCreator_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/multiversx/mx-chain-core-go/hashing"
 	"github.com/multiversx/mx-chain-core-go/marshal"
 	"github.com/multiversx/mx-chain-go/common"
+	"github.com/multiversx/mx-chain-go/dataRetriever"
 	dataComp "github.com/multiversx/mx-chain-go/factory/data"
 	"github.com/multiversx/mx-chain-go/factory/mock"
 	processComp "github.com/multiversx/mx-chain-go/factory/processing"
@@ -24,7 +25,6 @@ import (
 	storageManager "github.com/multiversx/mx-chain-go/testscommon/storage"
 	trieMock "github.com/multiversx/mx-chain-go/testscommon/trie"
 	"github.com/multiversx/mx-chain-go/trie"
-	trieFactory "github.com/multiversx/mx-chain-go/trie/factory"
 	vmcommon "github.com/multiversx/mx-chain-vm-common-go"
 	"github.com/stretchr/testify/require"
 )
@@ -104,14 +104,14 @@ func Test_newBlockProcessorCreatorForMeta(t *testing.T) {
 	storageManagerPeer, _ := trie.CreateTrieStorageManager(storageManagerArgs, options)
 
 	trieStorageManagers := make(map[string]common.StorageManager)
-	trieStorageManagers[trieFactory.UserAccountTrie] = storageManagerUser
-	trieStorageManagers[trieFactory.PeerAccountTrie] = storageManagerPeer
+	trieStorageManagers[dataRetriever.UserAccountsUnit.String()] = storageManagerUser
+	trieStorageManagers[dataRetriever.PeerAccountsUnit.String()] = storageManagerPeer
 
 	accounts, err := createAccountAdapter(
 		&mock.MarshalizerMock{},
 		&hashingMocks.HasherMock{},
 		factoryState.NewAccountCreator(),
-		trieStorageManagers[trieFactory.UserAccountTrie],
+		trieStorageManagers[dataRetriever.UserAccountsUnit.String()],
 	)
 	require.Nil(t, err)
 

factory/state/stateComponents.go

@@ -132,7 +132,7 @@ func (scf *stateComponentsFactory) Create() (*stateComponents, error) {
 
 func (scf *stateComponentsFactory) createAccountsAdapters(triesContainer common.TriesHolder) (state.AccountsAdapter, state.AccountsAdapter, state.AccountsRepository, error) {
 	accountFactory := factoryState.NewAccountCreator()
-	merkleTrie := triesContainer.Get([]byte(trieFactory.UserAccountTrie))
+	merkleTrie := triesContainer.Get([]byte(dataRetriever.UserAccountsUnit.String()))
 	storagePruning, err := scf.newStoragePruningManager()
 	if err != nil {
 		return nil, nil, nil, err
@@ -198,7 +198,7 @@ func (scf *stateComponentsFactory) createAccountsAdapters(triesContainer common.
 
 func (scf *stateComponentsFactory) createPeerAdapter(triesContainer common.TriesHolder) (state.AccountsAdapter, error) {
 	accountFactory := factoryState.NewPeerAccountCreator()
-	merkleTrie := triesContainer.Get([]byte(trieFactory.PeerAccountTrie))
+	merkleTrie := triesContainer.Get([]byte(dataRetriever.PeerAccountsUnit.String()))
 	storagePruning, err := scf.newStoragePruningManager()
 	if err != nil {
 		return nil, err