Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

pass storageMarker as argument when SyncAccounts() is called #5188

Merged
merged 3 commits into from
May 4, 2023
Merged

pass storageMarker as argument when SyncAccounts() is called #5188

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
de31e03
pass storageMarker as argument when SyncAccounts() is called
BeniaminDrasovean Apr 24, 2023
3968467
fix after review
BeniaminDrasovean Apr 25, 2023
e4cd99a
add TODOs
BeniaminDrasovean Apr 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions common/interface.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,12 @@ type TrieStats interface {
GetTrieStats(address string, rootHash []byte) (*statistics.TrieStatsDTO, error)
}

// StorageMarker is used to mark the given storer as synced and active
type StorageMarker interface {
MarkStorerAsSyncedAndActive(storer StorageManager)
IsInterfaceNil() bool
}

// KeyBuilder is used for building trie keys as you traverse the trie
type KeyBuilder interface {
BuildKey(keyPart []byte)
Expand Down
4 changes: 2 additions & 2 deletions dataRetriever/storageRequesters/trieNodeRequester_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ import (
"github.com/multiversx/mx-chain-core-go/data/endProcess"
"github.com/multiversx/mx-chain-go/dataRetriever"
"github.com/multiversx/mx-chain-go/dataRetriever/mock"
"github.com/multiversx/mx-chain-go/testscommon"
"github.com/multiversx/mx-chain-go/testscommon/p2pmocks"
"github.com/multiversx/mx-chain-go/testscommon/storageManager"
trieMock "github.com/multiversx/mx-chain-go/testscommon/trie"
"github.com/stretchr/testify/assert"
)
Expand All @@ -23,7 +23,7 @@ func createMockTrieRequesterArguments() ArgTrieRequester {
ResponseTopicName: "",
Marshalizer: &mock.MarshalizerStub{},
TrieDataGetter: &trieMock.TrieStub{},
TrieStorageManager: &testscommon.StorageManagerStub{},
TrieStorageManager: &storageManager.StorageManagerStub{},
ManualEpochStartNotifier: &mock.ManualEpochStartNotifierStub{},
ChanGracefullyClose: make(chan endProcess.ArgEndProcess, 1),
DelayBeforeGracefulClose: 0,
Expand Down
6 changes: 2 additions & 4 deletions epochStart/bootstrap/process.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1076,7 +1076,6 @@ func (e *epochStartBootstrap) syncUserAccountsState(rootHash []byte) error {
MaxHardCapForMissingNodes: e.maxHardCapForMissingNodes,
TrieSyncerVersion: e.trieSyncerVersion,
CheckNodesOnDisk: e.checkNodesOnDisk,
StorageMarker: storageMarker.NewTrieStorageMarker(),
UserAccountsSyncStatisticsHandler: e.trieSyncStatisticsProvider,
AppStatusHandler: e.statusHandler,
},
Expand All @@ -1089,7 +1088,7 @@ func (e *epochStartBootstrap) syncUserAccountsState(rootHash []byte) error {
return err
}

err = accountsDBSyncer.SyncAccounts(rootHash)
err = accountsDBSyncer.SyncAccounts(rootHash, storageMarker.NewTrieStorageMarker())
if err != nil {
return err
}
Expand Down Expand Up @@ -1147,7 +1146,6 @@ func (e *epochStartBootstrap) syncValidatorAccountsState(rootHash []byte) error
MaxHardCapForMissingNodes: e.maxHardCapForMissingNodes,
TrieSyncerVersion: e.trieSyncerVersion,
CheckNodesOnDisk: e.checkNodesOnDisk,
StorageMarker: storageMarker.NewTrieStorageMarker(),
UserAccountsSyncStatisticsHandler: statistics.NewTrieSyncStatistics(),
AppStatusHandler: disabledCommon.NewAppStatusHandler(),
},
Expand All @@ -1157,7 +1155,7 @@ func (e *epochStartBootstrap) syncValidatorAccountsState(rootHash []byte) error
return err
}

err = accountsDBSyncer.SyncAccounts(rootHash)
err = accountsDBSyncer.SyncAccounts(rootHash, storageMarker.NewTrieStorageMarker())
if err != nil {
return err
}
Expand Down
6 changes: 0 additions & 6 deletions epochStart/interface.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,12 +115,6 @@ type PendingMiniBlocksSyncHandler interface {
IsInterfaceNil() bool
}

// AccountsDBSyncer defines the methods for the accounts db syncer
type AccountsDBSyncer interface {
SyncAccounts(rootHash []byte) error
IsInterfaceNil() bool
}

// StartOfEpochMetaSyncer defines the methods to synchronize epoch start meta block from the network when nothing is known
type StartOfEpochMetaSyncer interface {
SyncEpochStartMeta(waitTime time.Duration) (data.MetaHeaderHandler, error)
Expand Down
2 changes: 0 additions & 2 deletions factory/consensus/consensusComponents.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ import (
"github.com/multiversx/mx-chain-go/sharding"
"github.com/multiversx/mx-chain-go/state/syncer"
"github.com/multiversx/mx-chain-go/trie/statistics"
"github.com/multiversx/mx-chain-go/trie/storageMarker"
"github.com/multiversx/mx-chain-go/update"
logger "github.com/multiversx/mx-chain-logger-go"
"github.com/multiversx/mx-chain-storage-go/timecache"
Expand Down Expand Up @@ -529,7 +528,6 @@ func (ccf *consensusComponentsFactory) createArgsBaseAccountsSyncer(trieStorageM
MaxHardCapForMissingNodes: ccf.config.TrieSync.MaxHardCapForMissingNodes,
TrieSyncerVersion: ccf.config.TrieSync.TrieSyncerVersion,
CheckNodesOnDisk: ccf.config.TrieSync.CheckNodesOnDisk,
StorageMarker: storageMarker.NewTrieStorageMarker(),
UserAccountsSyncStatisticsHandler: statistics.NewTrieSyncStatistics(),
AppStatusHandler: disabled.NewAppStatusHandler(),
}
Expand Down
4 changes: 2 additions & 2 deletions factory/state/stateComponentsHandler_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ import (
"github.com/multiversx/mx-chain-go/common"
"github.com/multiversx/mx-chain-go/factory/mock"
stateComp "github.com/multiversx/mx-chain-go/factory/state"
"github.com/multiversx/mx-chain-go/testscommon"
componentsMock "github.com/multiversx/mx-chain-go/testscommon/components"
"github.com/multiversx/mx-chain-go/testscommon/storageManager"
trieMock "github.com/multiversx/mx-chain-go/testscommon/trie"
"github.com/stretchr/testify/require"
)
Expand Down Expand Up @@ -108,7 +108,7 @@ func TestManagedStateComponents_Setters(t *testing.T) {
require.NoError(t, err)

triesContainer := &trieMock.TriesHolderStub{}
triesStorageManagers := map[string]common.StorageManager{"a": &testscommon.StorageManagerStub{}}
triesStorageManagers := map[string]common.StorageManager{"a": &storageManager.StorageManagerStub{}}

err = managedStateComponents.SetTriesContainer(triesContainer)
require.NoError(t, err)
Expand Down
6 changes: 3 additions & 3 deletions integrationTests/mock/accountsDBSyncerStub.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
// AccountsDBSyncerStub -
type AccountsDBSyncerStub struct {
GetSyncedTriesCalled func() map[string]common.Trie
SyncAccountsCalled func(rootHash []byte) error
SyncAccountsCalled func(rootHash []byte, storageMarker common.StorageMarker) error
}

// GetSyncedTries -
Expand All @@ -19,9 +19,9 @@ func (a *AccountsDBSyncerStub) GetSyncedTries() map[string]common.Trie {
}

// SyncAccounts -
func (a *AccountsDBSyncerStub) SyncAccounts(rootHash []byte) error {
func (a *AccountsDBSyncerStub) SyncAccounts(rootHash []byte, storageMarker common.StorageMarker) error {
if a.SyncAccountsCalled != nil {
return a.SyncAccountsCalled(rootHash)
return a.SyncAccountsCalled(rootHash, storageMarker)
}
return nil
}
Expand Down
3 changes: 1 addition & 2 deletions integrationTests/state/stateTrieSync/stateTrieSync_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,7 @@ func testMultipleDataTriesSync(t *testing.T, numAccounts int, numDataTrieLeaves
userAccSyncer, err := syncer.NewUserAccountsSyncer(syncerArgs)
assert.Nil(t, err)

err = userAccSyncer.SyncAccounts(rootHash)
err = userAccSyncer.SyncAccounts(rootHash, storageMarker.NewDisabledStorageMarker())
assert.Nil(t, err)

_ = nRequester.AccntState.RecreateTrie(rootHash)
Expand Down Expand Up @@ -588,7 +588,6 @@ func getUserAccountSyncerArgs(node *integrationTests.TestProcessorNode, version
MaxTrieLevelInMemory: 200,
MaxHardCapForMissingNodes: 5000,
TrieSyncerVersion: version,
StorageMarker: storageMarker.NewTrieStorageMarker(),
UserAccountsSyncStatisticsHandler: statistics.NewTrieSyncStatistics(),
AppStatusHandler: integrationTests.TestAppStatusHandler,
},
Expand Down
9 changes: 5 additions & 4 deletions integrationTests/testProcessorNode.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ import (
stateMock "github.com/multiversx/mx-chain-go/testscommon/state"
statusHandlerMock "github.com/multiversx/mx-chain-go/testscommon/statusHandler"
storageStubs "github.com/multiversx/mx-chain-go/testscommon/storage"
"github.com/multiversx/mx-chain-go/testscommon/storageManager"
trieMock "github.com/multiversx/mx-chain-go/testscommon/trie"
"github.com/multiversx/mx-chain-go/trie/keyBuilder"
"github.com/multiversx/mx-chain-go/update"
Expand Down Expand Up @@ -3184,9 +3185,9 @@ func GetDefaultStateComponents() *testscommon.StateComponentsMock {
AccountsRepo: &stateMock.AccountsRepositoryStub{},
Tries: &trieMock.TriesHolderStub{},
StorageManagers: map[string]common.StorageManager{
"0": &testscommon.StorageManagerStub{},
dataRetriever.UserAccountsUnit.String(): &testscommon.StorageManagerStub{},
dataRetriever.PeerAccountsUnit.String(): &testscommon.StorageManagerStub{},
"0": &storageManager.StorageManagerStub{},
dataRetriever.UserAccountsUnit.String(): &storageManager.StorageManagerStub{},
dataRetriever.PeerAccountsUnit.String(): &storageManager.StorageManagerStub{},
},
}
}
Expand Down Expand Up @@ -3225,7 +3226,7 @@ func getDefaultBootstrapComponents(shardCoordinator sharding.Coordinator) *mainF
return &mainFactoryMocks.BootstrapComponentsStub{
Bootstrapper: &bootstrapMocks.EpochStartBootstrapperStub{
TrieHolder: &trieMock.TriesHolderStub{},
StorageManagers: map[string]common.StorageManager{"0": &testscommon.StorageManagerStub{}},
StorageManagers: map[string]common.StorageManager{"0": &storageManager.StorageManagerStub{}},
BootstrapCalled: nil,
},
BootstrapParams: &bootstrapMocks.BootstrapParamsHandlerMock{},
Expand Down
2 changes: 0 additions & 2 deletions node/nodeRunner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,6 @@ import (
storageFactory "github.com/multiversx/mx-chain-go/storage/factory"
"github.com/multiversx/mx-chain-go/storage/storageunit"
trieStatistics "github.com/multiversx/mx-chain-go/trie/statistics"
"github.com/multiversx/mx-chain-go/trie/storageMarker"
"github.com/multiversx/mx-chain-go/update/trigger"
logger "github.com/multiversx/mx-chain-logger-go"
)
Expand Down Expand Up @@ -674,7 +673,6 @@ func getBaseAccountSyncerArgs(
MaxTrieLevelInMemory: maxTrieLevelInMemory,
MaxHardCapForMissingNodes: config.TrieSync.MaxHardCapForMissingNodes,
TrieSyncerVersion: config.TrieSync.TrieSyncerVersion,
StorageMarker: storageMarker.NewDisabledStorageMarker(),
CheckNodesOnDisk: true,
UserAccountsSyncStatisticsHandler: trieStatistics.NewTrieSyncStatistics(),
AppStatusHandler: disabled.NewAppStatusHandler(),
Expand Down
5 changes: 3 additions & 2 deletions node/nodeTesting_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
dataRetrieverMock "github.com/multiversx/mx-chain-go/testscommon/dataRetriever"
"github.com/multiversx/mx-chain-go/testscommon/p2pmocks"
stateMock "github.com/multiversx/mx-chain-go/testscommon/state"
"github.com/multiversx/mx-chain-go/testscommon/storageManager"
trieMock "github.com/multiversx/mx-chain-go/testscommon/trie"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -397,7 +398,7 @@ func getDefaultCryptoComponents() *factoryMock.CryptoComponentsMock {
PubKeyBytes: []byte("pubKey"),
BlockSig: &mock.SingleSignerMock{},
TxSig: &mock.SingleSignerMock{},
MultiSigContainer: cryptoMocks.NewMultiSignerContainerMock( cryptoMocks.NewMultiSigner()),
MultiSigContainer: cryptoMocks.NewMultiSignerContainerMock(cryptoMocks.NewMultiSigner()),
PeerSignHandler: &mock.PeerSignatureHandler{},
BlKeyGen: &mock.KeyGenMock{},
TxKeyGen: &mock.KeyGenMock{},
Expand All @@ -415,7 +416,7 @@ func getDefaultStateComponents() *testscommon.StateComponentsMock {
AccountsAPI: &stateMock.AccountsStub{},
AccountsRepo: &stateMock.AccountsRepositoryStub{},
Tries: &trieMock.TriesHolderStub{},
StorageManagers: map[string]common.StorageManager{"0": &testscommon.StorageManagerStub{}},
StorageManagers: map[string]common.StorageManager{"0": &storageManager.StorageManagerStub{}},
}
}

Expand Down
3 changes: 2 additions & 1 deletion node/node_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ import (
stateMock "github.com/multiversx/mx-chain-go/testscommon/state"
statusHandlerMock "github.com/multiversx/mx-chain-go/testscommon/statusHandler"
"github.com/multiversx/mx-chain-go/testscommon/storage"
"github.com/multiversx/mx-chain-go/testscommon/storageManager"
trieMock "github.com/multiversx/mx-chain-go/testscommon/trie"
"github.com/multiversx/mx-chain-go/testscommon/txsSenderMock"
"github.com/multiversx/mx-chain-go/vm/systemSmartContracts"
Expand Down Expand Up @@ -4007,7 +4008,7 @@ func getDefaultBootstrapComponents() *mainFactoryMocks.BootstrapComponentsStub {
return &mainFactoryMocks.BootstrapComponentsStub{
Bootstrapper: &bootstrapMocks.EpochStartBootstrapperStub{
TrieHolder: &trieMock.TriesHolderStub{},
StorageManagers: map[string]common.StorageManager{"0": &testscommon.StorageManagerStub{}},
StorageManagers: map[string]common.StorageManager{"0": &storageManager.StorageManagerStub{}},
BootstrapCalled: nil,
},
BootstrapParams: &bootstrapMocks.BootstrapParamsHandlerMock{},
Expand Down
2 changes: 1 addition & 1 deletion process/interface.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1179,7 +1179,7 @@ type InterceptedChunksProcessor interface {

// AccountsDBSyncer defines the methods for the accounts db syncer
type AccountsDBSyncer interface {
SyncAccounts(rootHash []byte) error
SyncAccounts(rootHash []byte, storageMarker common.StorageMarker) error
IsInterfaceNil() bool
}

Expand Down
6 changes: 3 additions & 3 deletions process/mock/accountsDBSyncerStub.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
// AccountsDBSyncerStub -
type AccountsDBSyncerStub struct {
GetSyncedTriesCalled func() map[string]common.Trie
SyncAccountsCalled func(rootHash []byte) error
SyncAccountsCalled func(rootHash []byte, storageMarker common.StorageMarker) error
}

// GetSyncedTries -
Expand All @@ -19,9 +19,9 @@ func (a *AccountsDBSyncerStub) GetSyncedTries() map[string]common.Trie {
}

// SyncAccounts -
func (a *AccountsDBSyncerStub) SyncAccounts(rootHash []byte) error {
func (a *AccountsDBSyncerStub) SyncAccounts(rootHash []byte, storageMarker common.StorageMarker) error {
if a.SyncAccountsCalled != nil {
return a.SyncAccountsCalled(rootHash)
return a.SyncAccountsCalled(rootHash, storageMarker)
}
return nil
}
Expand Down
3 changes: 2 additions & 1 deletion process/sync/baseSync.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/multiversx/mx-chain-go/sharding"
"github.com/multiversx/mx-chain-go/state"
"github.com/multiversx/mx-chain-go/storage"
"github.com/multiversx/mx-chain-go/trie/storageMarker"
logger "github.com/multiversx/mx-chain-logger-go"
)

Expand Down Expand Up @@ -696,7 +697,7 @@ func (boot *baseBootstrap) handleTrieSyncError(err error, ctx context.Context) {

func (boot *baseBootstrap) syncUserAccountsState(key []byte) error {
log.Warn("base sync: started syncUserAccountsState")
return boot.accountsDBSyncer.SyncAccounts(key)
return boot.accountsDBSyncer.SyncAccounts(key, storageMarker.NewDisabledStorageMarker())
}

func (boot *baseBootstrap) cleanNoncesSyncedWithErrorsBehindFinal() {
Expand Down
3 changes: 2 additions & 1 deletion process/sync/metablock.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/multiversx/mx-chain-go/process"
"github.com/multiversx/mx-chain-go/state"
"github.com/multiversx/mx-chain-go/storage"
"github.com/multiversx/mx-chain-go/trie/storageMarker"
)

// MetaBootstrap implements the bootstrap mechanism
Expand Down Expand Up @@ -216,7 +217,7 @@ func (boot *MetaBootstrap) syncAccountsDBs(key []byte, id string) error {

func (boot *MetaBootstrap) syncValidatorAccountsState(key []byte) error {
log.Warn("base sync: started syncValidatorAccountsState")
return boot.validatorStatisticsDBSyncer.SyncAccounts(key)
return boot.validatorStatisticsDBSyncer.SyncAccounts(key, storageMarker.NewDisabledStorageMarker())
}

// Close closes the synchronization loop
Expand Down
6 changes: 3 additions & 3 deletions process/sync/metablock_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1682,7 +1682,7 @@ func TestMetaBootstrap_SyncBlockErrGetNodeDBShouldSyncAccounts(t *testing.T) {
)
accountsSyncCalled := false
args.AccountsDBSyncer = &mock.AccountsDBSyncerStub{
SyncAccountsCalled: func(rootHash []byte) error {
SyncAccountsCalled: func(rootHash []byte, _ common.StorageMarker) error {
accountsSyncCalled = true
return nil
},
Expand Down Expand Up @@ -1736,7 +1736,7 @@ func TestMetaBootstrap_SyncAccountsDBs(t *testing.T) {
args := CreateMetaBootstrapMockArguments()
accountsSyncCalled := false
args.AccountsDBSyncer = &mock.AccountsDBSyncerStub{
SyncAccountsCalled: func(rootHash []byte) error {
SyncAccountsCalled: func(rootHash []byte, _ common.StorageMarker) error {
accountsSyncCalled = true
return nil
},
Expand Down Expand Up @@ -1776,7 +1776,7 @@ func TestMetaBootstrap_SyncAccountsDBs(t *testing.T) {
args := CreateMetaBootstrapMockArguments()
accountsSyncCalled := false
args.ValidatorStatisticsDBSyncer = &mock.AccountsDBSyncerStub{
SyncAccountsCalled: func(rootHash []byte) error {
SyncAccountsCalled: func(rootHash []byte, _ common.StorageMarker) error {
accountsSyncCalled = true
return nil
},
Expand Down
2 changes: 1 addition & 1 deletion process/sync/shardblock_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2135,7 +2135,7 @@ func TestShardBootstrap_SyncBlockGetNodeDBErrorShouldSync(t *testing.T) {

syncCalled := false
args.AccountsDBSyncer = &mock.AccountsDBSyncerStub{
SyncAccountsCalled: func(rootHash []byte) error {
SyncAccountsCalled: func(rootHash []byte, _ common.StorageMarker) error {
syncCalled = true
return nil
}}
Expand Down
3 changes: 2 additions & 1 deletion state/accountsDB.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"github.com/multiversx/mx-chain-go/common/holders"
"github.com/multiversx/mx-chain-go/trie/keyBuilder"
"github.com/multiversx/mx-chain-go/trie/statistics"
"github.com/multiversx/mx-chain-go/trie/storageMarker"
logger "github.com/multiversx/mx-chain-logger-go"
vmcommon "github.com/multiversx/mx-chain-vm-common-go"
)
Expand Down Expand Up @@ -1295,7 +1296,7 @@ func (adb *AccountsDB) syncMissingNodes(missingNodesChan chan []byte, errChan ch
}

for missingNode := range missingNodesChan {
err := syncer.SyncAccounts(missingNode)
err := syncer.SyncAccounts(missingNode, storageMarker.NewDisabledStorageMarker())
if err != nil {
log.Error("could not sync missing node",
"missing node hash", missingNode,
Expand Down
Loading