From d29bfa3f1ef886f0471d3354e05e9aa0cfffbb84 Mon Sep 17 00:00:00 2001 From: Bogdan Rosianu Date: Tue, 25 Oct 2022 17:49:28 +0300 Subject: [PATCH 1/4] fix custom meta DB remover --- storage/factory/pruningStorerFactory.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/storage/factory/pruningStorerFactory.go b/storage/factory/pruningStorerFactory.go index 58fb41ffbbd..5c2911fc1c8 100644 --- a/storage/factory/pruningStorerFactory.go +++ b/storage/factory/pruningStorerFactory.go @@ -12,7 +12,6 @@ import ( "github.com/ElrondNetwork/elrond-go/epochStart" "github.com/ElrondNetwork/elrond-go/storage" "github.com/ElrondNetwork/elrond-go/storage/clean" - "github.com/ElrondNetwork/elrond-go/storage/databaseremover" "github.com/ElrondNetwork/elrond-go/storage/databaseremover/disabled" "github.com/ElrondNetwork/elrond-go/storage/databaseremover/factory" storageDisabled "github.com/ElrondNetwork/elrond-go/storage/disabled" @@ -301,6 +300,9 @@ func (psf *StorageServiceFactory) CreateForShard() (dataRetriever.StorageService return store, err } +// TODO: split in 2 components shardStorageServiceCreator and metStorageServiceCreator that have a base which will contain the +// common storers + // CreateForMeta will return the storage service which contains all storers needed for metachain func (psf *StorageServiceFactory) CreateForMeta() (dataRetriever.StorageService, error) { var metaBlockUnit storage.Storer @@ -322,7 +324,7 @@ func (psf *StorageServiceFactory) CreateForMeta() (dataRetriever.StorageService, // in case of a failure while creating (not opening) disabledCustomDatabaseRemover := disabled.NewDisabledCustomDatabaseRemover() - customDatabaseRemover, err := databaseremover.NewCustomDatabaseRemover(psf.generalConfig.StoragePruning) + customDatabaseRemover, err := factory.CreateCustomDatabaseRemover(psf.generalConfig.StoragePruning) if err != nil { return nil, err } From c0157c42640398f1cdf389e70a94dce2e1eb3055 Mon Sep 17 00:00:00 2001 From: jules01 Date: Tue, 25 Oct 2022 20:02:01 +0300 Subject: [PATCH 2/4] - removed duplicated code --- storage/factory/pruningStorerFactory.go | 328 +++++++----------------- 1 file changed, 97 insertions(+), 231 deletions(-) diff --git a/storage/factory/pruningStorerFactory.go b/storage/factory/pruningStorerFactory.go index 5c2911fc1c8..d7551fa954a 100644 --- a/storage/factory/pruningStorerFactory.go +++ b/storage/factory/pruningStorerFactory.go @@ -111,121 +111,113 @@ func checkArgs(args StorageServiceFactoryArgs) error { return nil } -// CreateForShard will return the storage service which contains all storers needed for a shard -func (psf *StorageServiceFactory) CreateForShard() (dataRetriever.StorageService, error) { - var headerUnit storage.Storer - var peerBlockUnit storage.Storer - var miniBlockUnit storage.Storer - var txUnit storage.Storer - var metachainHeaderUnit storage.Storer - var unsignedTxUnit storage.Storer - var rewardTxUnit storage.Storer - var bootstrapUnit storage.Storer - var receiptsUnit storage.Storer - var userAccountsUnit storage.Storer - var peerAccountsUnit storage.Storer - var userAccountsCheckpointsUnit storage.Storer - var peerAccountsCheckpointsUnit storage.Storer - var scheduledSCRsUnit storage.Storer - var err error - - // TODO: if there will be a differentiation between the creation or opening of a DB, the DBs could be destroyed on a defer - // in case of a failure while creating (not opening). - +func (psf *StorageServiceFactory) createAndAddBaseStorageUnits(store dataRetriever.StorageService, shardID string) error { disabledCustomDatabaseRemover := disabled.NewDisabledCustomDatabaseRemover() customDatabaseRemover, err := factory.CreateCustomDatabaseRemover(psf.generalConfig.StoragePruning) if err != nil { - return nil, err + return err } - txUnitStorerArgs := psf.createPruningStorerArgs(psf.generalConfig.TxStorage, disabledCustomDatabaseRemover) - txUnit, err = psf.createPruningPersister(txUnitStorerArgs) + txUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.TxStorage, disabledCustomDatabaseRemover) + txUnit, err := psf.createPruningPersister(txUnitArgs) if err != nil { - return nil, err + return err } + store.AddStorer(dataRetriever.TransactionUnit, txUnit) - unsignedTxUnitStorerArgs := psf.createPruningStorerArgs(psf.generalConfig.UnsignedTransactionStorage, disabledCustomDatabaseRemover) - unsignedTxUnit, err = psf.createPruningPersister(unsignedTxUnitStorerArgs) + unsignedTxUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.UnsignedTransactionStorage, disabledCustomDatabaseRemover) + unsignedTxUnit, err := psf.createPruningPersister(unsignedTxUnitArgs) if err != nil { - return nil, err + return err } + store.AddStorer(dataRetriever.UnsignedTransactionUnit, unsignedTxUnit) rewardTxUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.RewardTxStorage, disabledCustomDatabaseRemover) - rewardTxUnit, err = psf.createPruningPersister(rewardTxUnitArgs) + rewardTxUnit, err := psf.createPruningPersister(rewardTxUnitArgs) if err != nil { - return nil, err + return err } + store.AddStorer(dataRetriever.RewardTransactionUnit, rewardTxUnit) - miniBlockUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.MiniBlocksStorage, disabledCustomDatabaseRemover) - miniBlockUnit, err = psf.createPruningPersister(miniBlockUnitArgs) + receiptsUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.ReceiptsStorage, disabledCustomDatabaseRemover) + receiptsUnit, err := psf.createPruningPersister(receiptsUnitArgs) if err != nil { - return nil, err + return err } + store.AddStorer(dataRetriever.ReceiptsUnit, receiptsUnit) - peerBlockUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.PeerBlockBodyStorage, disabledCustomDatabaseRemover) - peerBlockUnit, err = psf.createPruningPersister(peerBlockUnitArgs) + scheduledSCRsUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.ScheduledSCRsStorage, disabledCustomDatabaseRemover) + scheduledSCRsUnit, err := psf.createPruningPersister(scheduledSCRsUnitArgs) if err != nil { - return nil, err + return err } + store.AddStorer(dataRetriever.ScheduledSCRsUnit, scheduledSCRsUnit) - headerUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.BlockHeaderStorage, disabledCustomDatabaseRemover) - headerUnit, err = psf.createPruningPersister(headerUnitArgs) + bootstrapUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.BootstrapStorage, disabledCustomDatabaseRemover) + bootstrapUnit, err := psf.createPruningPersister(bootstrapUnitArgs) if err != nil { - return nil, err + return err } + store.AddStorer(dataRetriever.BootstrapUnit, bootstrapUnit) - metaChainHeaderUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.MetaBlockStorage, disabledCustomDatabaseRemover) - metachainHeaderUnit, err = psf.createPruningPersister(metaChainHeaderUnitArgs) + miniBlockUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.MiniBlocksStorage, disabledCustomDatabaseRemover) + miniBlockUnit, err := psf.createPruningPersister(miniBlockUnitArgs) if err != nil { - return nil, err + return err } + store.AddStorer(dataRetriever.MiniBlockUnit, miniBlockUnit) - userAccountsUnit, err = psf.createTriePruningStorer(psf.generalConfig.AccountsTrieStorage, customDatabaseRemover) + metaBlockUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.MetaBlockStorage, disabledCustomDatabaseRemover) + metaBlockUnit, err := psf.createPruningPersister(metaBlockUnitArgs) if err != nil { - return nil, err + return err } + store.AddStorer(dataRetriever.MetaBlockUnit, metaBlockUnit) - peerAccountsUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.PeerAccountsTrieStorage, customDatabaseRemover) - peerAccountsUnit, err = psf.createTrieUnit(psf.generalConfig.PeerAccountsTrieStorage, peerAccountsUnitArgs) + // metaHdrHashNonce is static + metaHdrHashNonceUnitConfig := GetDBFromConfig(psf.generalConfig.MetaHdrNonceHashStorage.DB) + dbPath := psf.pathManager.PathForStatic(shardID, psf.generalConfig.MetaHdrNonceHashStorage.DB.FilePath) + metaHdrHashNonceUnitConfig.FilePath = dbPath + metaHdrHashNonceUnit, err := storageunit.NewStorageUnitFromConf( + GetCacherFromConfig(psf.generalConfig.MetaHdrNonceHashStorage.Cache), + metaHdrHashNonceUnitConfig) if err != nil { - return nil, err + return err } + store.AddStorer(dataRetriever.MetaHdrNonceHashDataUnit, metaHdrHashNonceUnit) - userAccountsCheckpointsUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.AccountsTrieCheckpointsStorage, disabledCustomDatabaseRemover) - userAccountsCheckpointsUnit, err = psf.createPruningPersister(userAccountsCheckpointsUnitArgs) + headerUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.BlockHeaderStorage, disabledCustomDatabaseRemover) + headerUnit, err := psf.createPruningPersister(headerUnitArgs) if err != nil { - return nil, err + return err } + store.AddStorer(dataRetriever.BlockHeaderUnit, headerUnit) - peerAccountsCheckpointsUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.PeerAccountsTrieCheckpointsStorage, disabledCustomDatabaseRemover) - peerAccountsCheckpointsUnit, err = psf.createPruningPersister(peerAccountsCheckpointsUnitArgs) + userAccountsUnit, err := psf.createTriePruningStorer(psf.generalConfig.AccountsTrieStorage, customDatabaseRemover) if err != nil { - return nil, err + return err } + store.AddStorer(dataRetriever.UserAccountsUnit, userAccountsUnit) - // metaHdrHashNonce is static - metaHdrHashNonceUnitConfig := GetDBFromConfig(psf.generalConfig.MetaHdrNonceHashStorage.DB) - shardID := core.GetShardIDString(psf.shardCoordinator.SelfId()) - dbPath := psf.pathManager.PathForStatic(shardID, psf.generalConfig.MetaHdrNonceHashStorage.DB.FilePath) - metaHdrHashNonceUnitConfig.FilePath = dbPath - metaHdrHashNonceUnit, err := storageunit.NewStorageUnitFromConf( - GetCacherFromConfig(psf.generalConfig.MetaHdrNonceHashStorage.Cache), - metaHdrHashNonceUnitConfig) + peerAccountsUnit, err := psf.createTriePruningStorer(psf.generalConfig.PeerAccountsTrieStorage, customDatabaseRemover) if err != nil { - return nil, err + return err } + store.AddStorer(dataRetriever.PeerAccountsUnit, peerAccountsUnit) - // shardHdrHashNonce storer is static - shardHdrHashNonceConfig := GetDBFromConfig(psf.generalConfig.ShardHdrNonceHashStorage.DB) - shardID = core.GetShardIDString(psf.shardCoordinator.SelfId()) - dbPath = psf.pathManager.PathForStatic(shardID, psf.generalConfig.ShardHdrNonceHashStorage.DB.FilePath) + shardID - shardHdrHashNonceConfig.FilePath = dbPath - shardHdrHashNonceUnit, err := storageunit.NewStorageUnitFromConf( - GetCacherFromConfig(psf.generalConfig.ShardHdrNonceHashStorage.Cache), - shardHdrHashNonceConfig) + userAccountsCheckpointsUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.AccountsTrieCheckpointsStorage, disabledCustomDatabaseRemover) + userAccountsCheckpointsUnit, err := psf.createPruningPersister(userAccountsCheckpointsUnitArgs) if err != nil { - return nil, err + return err + } + store.AddStorer(dataRetriever.UserAccountsCheckpointsUnit, userAccountsCheckpointsUnit) + + peerAccountsCheckpointsUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.PeerAccountsTrieCheckpointsStorage, disabledCustomDatabaseRemover) + peerAccountsCheckpointsUnit, err := psf.createPruningPersister(peerAccountsCheckpointsUnitArgs) + if err != nil { + return err } + store.AddStorer(dataRetriever.PeerAccountsCheckpointsUnit, peerAccountsCheckpointsUnit) statusMetricsDbConfig := GetDBFromConfig(psf.generalConfig.StatusMetricsStorage.DB) shardId := core.GetShardIDString(psf.shardCoordinator.SelfId()) @@ -235,52 +227,58 @@ func (psf *StorageServiceFactory) CreateForShard() (dataRetriever.StorageService GetCacherFromConfig(psf.generalConfig.StatusMetricsStorage.Cache), statusMetricsDbConfig) if err != nil { - return nil, err + return err } + store.AddStorer(dataRetriever.StatusMetricsUnit, statusMetricsStorageUnit) trieEpochRootHashStorageUnit, err := psf.createTrieEpochRootHashStorerIfNeeded() if err != nil { - return nil, err + return err } + store.AddStorer(dataRetriever.TrieEpochRootHashUnit, trieEpochRootHashStorageUnit) - bootstrapUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.BootstrapStorage, disabledCustomDatabaseRemover) - bootstrapUnit, err = psf.createPruningPersister(bootstrapUnitArgs) + return nil +} + +// CreateForShard will return the storage service which contains all storers needed for a shard +func (psf *StorageServiceFactory) CreateForShard() (dataRetriever.StorageService, error) { + var peerBlockUnit storage.Storer + var err error + + // TODO: if there will be a differentiation between the creation or opening of a DB, the DBs could be destroyed on a defer + // in case of a failure while creating (not opening). + + disabledCustomDatabaseRemover := disabled.NewDisabledCustomDatabaseRemover() + + peerBlockUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.PeerBlockBodyStorage, disabledCustomDatabaseRemover) + peerBlockUnit, err = psf.createPruningPersister(peerBlockUnitArgs) if err != nil { return nil, err } - receiptsUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.ReceiptsStorage, disabledCustomDatabaseRemover) - receiptsUnit, err = psf.createPruningPersister(receiptsUnitArgs) + shardID := core.GetShardIDString(psf.shardCoordinator.SelfId()) + + // shardHdrHashNonce storer is static + shardHdrHashNonceConfig := GetDBFromConfig(psf.generalConfig.ShardHdrNonceHashStorage.DB) + shardID = core.GetShardIDString(psf.shardCoordinator.SelfId()) + dbPath := psf.pathManager.PathForStatic(shardID, psf.generalConfig.ShardHdrNonceHashStorage.DB.FilePath) + shardID + shardHdrHashNonceConfig.FilePath = dbPath + shardHdrHashNonceUnit, err := storageunit.NewStorageUnitFromConf( + GetCacherFromConfig(psf.generalConfig.ShardHdrNonceHashStorage.Cache), + shardHdrHashNonceConfig) if err != nil { return nil, err } - scheduledSCRsUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.ScheduledSCRsStorage, disabledCustomDatabaseRemover) - scheduledSCRsUnit, err = psf.createPruningPersister(scheduledSCRsUnitArgs) + store := dataRetriever.NewChainStorer() + err = psf.createAndAddBaseStorageUnits(store, shardID) if err != nil { return nil, err } - store := dataRetriever.NewChainStorer() - store.AddStorer(dataRetriever.TransactionUnit, txUnit) - store.AddStorer(dataRetriever.MiniBlockUnit, miniBlockUnit) store.AddStorer(dataRetriever.PeerChangesUnit, peerBlockUnit) - store.AddStorer(dataRetriever.BlockHeaderUnit, headerUnit) - store.AddStorer(dataRetriever.MetaBlockUnit, metachainHeaderUnit) - store.AddStorer(dataRetriever.UnsignedTransactionUnit, unsignedTxUnit) - store.AddStorer(dataRetriever.RewardTransactionUnit, rewardTxUnit) - store.AddStorer(dataRetriever.MetaHdrNonceHashDataUnit, metaHdrHashNonceUnit) hdrNonceHashDataUnit := dataRetriever.ShardHdrNonceHashDataUnit + dataRetriever.UnitType(psf.shardCoordinator.SelfId()) store.AddStorer(hdrNonceHashDataUnit, shardHdrHashNonceUnit) - store.AddStorer(dataRetriever.BootstrapUnit, bootstrapUnit) - store.AddStorer(dataRetriever.StatusMetricsUnit, statusMetricsStorageUnit) - store.AddStorer(dataRetriever.ReceiptsUnit, receiptsUnit) - store.AddStorer(dataRetriever.TrieEpochRootHashUnit, trieEpochRootHashStorageUnit) - store.AddStorer(dataRetriever.UserAccountsUnit, userAccountsUnit) - store.AddStorer(dataRetriever.PeerAccountsUnit, peerAccountsUnit) - store.AddStorer(dataRetriever.UserAccountsCheckpointsUnit, userAccountsCheckpointsUnit) - store.AddStorer(dataRetriever.PeerAccountsCheckpointsUnit, peerAccountsCheckpointsUnit) - store.AddStorer(dataRetriever.ScheduledSCRsUnit, scheduledSCRsUnit) err = psf.setupDbLookupExtensions(store) if err != nil { @@ -305,81 +303,18 @@ func (psf *StorageServiceFactory) CreateForShard() (dataRetriever.StorageService // CreateForMeta will return the storage service which contains all storers needed for metachain func (psf *StorageServiceFactory) CreateForMeta() (dataRetriever.StorageService, error) { - var metaBlockUnit storage.Storer - var headerUnit storage.Storer - var txUnit storage.Storer - var miniBlockUnit storage.Storer - var unsignedTxUnit storage.Storer - var rewardTxUnit storage.Storer - var bootstrapUnit storage.Storer - var receiptsUnit storage.Storer - var userAccountsUnit storage.Storer - var peerAccountsUnit storage.Storer - var userAccountsCheckpointsUnit storage.Storer - var peerAccountsCheckpointsUnit storage.Storer - var scheduledSCRsUnit storage.Storer var err error // TODO: if there will be a differentiation between the creation or opening of a DB, the DBs could be destroyed on a defer // in case of a failure while creating (not opening) - disabledCustomDatabaseRemover := disabled.NewDisabledCustomDatabaseRemover() - customDatabaseRemover, err := factory.CreateCustomDatabaseRemover(psf.generalConfig.StoragePruning) - if err != nil { - return nil, err - } - - metaBlockUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.MetaBlockStorage, disabledCustomDatabaseRemover) - metaBlockUnit, err = psf.createPruningPersister(metaBlockUnitArgs) - if err != nil { - return nil, err - } - - headerUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.BlockHeaderStorage, disabledCustomDatabaseRemover) - headerUnit, err = psf.createPruningPersister(headerUnitArgs) - if err != nil { - return nil, err - } - - userAccountsUnit, err = psf.createTriePruningStorer(psf.generalConfig.AccountsTrieStorage, customDatabaseRemover) - if err != nil { - return nil, err - } - - peerAccountsUnit, err = psf.createTriePruningStorer(psf.generalConfig.PeerAccountsTrieStorage, customDatabaseRemover) - if err != nil { - return nil, err - } - - userAccountsCheckpointsUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.AccountsTrieCheckpointsStorage, disabledCustomDatabaseRemover) - userAccountsCheckpointsUnit, err = psf.createPruningPersister(userAccountsCheckpointsUnitArgs) - if err != nil { - return nil, err - } - - peerAccountsCheckpointsUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.PeerAccountsTrieCheckpointsStorage, disabledCustomDatabaseRemover) - peerAccountsCheckpointsUnit, err = psf.createPruningPersister(peerAccountsCheckpointsUnitArgs) - if err != nil { - return nil, err - } - - // metaHdrHashNonce is static - metaHdrHashNonceUnitConfig := GetDBFromConfig(psf.generalConfig.MetaHdrNonceHashStorage.DB) shardID := core.GetShardIDString(core.MetachainShardId) - dbPath := psf.pathManager.PathForStatic(shardID, psf.generalConfig.MetaHdrNonceHashStorage.DB.FilePath) - metaHdrHashNonceUnitConfig.FilePath = dbPath - metaHdrHashNonceUnit, err := storageunit.NewStorageUnitFromConf( - GetCacherFromConfig(psf.generalConfig.MetaHdrNonceHashStorage.Cache), - metaHdrHashNonceUnitConfig) - if err != nil { - return nil, err - } shardHdrHashNonceUnits := make([]*storageunit.Unit, psf.shardCoordinator.NumberOfShards()) for i := uint32(0); i < psf.shardCoordinator.NumberOfShards(); i++ { shardHdrHashNonceConfig := GetDBFromConfig(psf.generalConfig.ShardHdrNonceHashStorage.DB) shardID = core.GetShardIDString(core.MetachainShardId) - dbPath = psf.pathManager.PathForStatic(shardID, psf.generalConfig.ShardHdrNonceHashStorage.DB.FilePath) + fmt.Sprintf("%d", i) + dbPath := psf.pathManager.PathForStatic(shardID, psf.generalConfig.ShardHdrNonceHashStorage.DB.FilePath) + fmt.Sprintf("%d", i) shardHdrHashNonceConfig.FilePath = dbPath shardHdrHashNonceUnits[i], err = storageunit.NewStorageUnitFromConf( GetCacherFromConfig(psf.generalConfig.ShardHdrNonceHashStorage.Cache), @@ -389,85 +324,16 @@ func (psf *StorageServiceFactory) CreateForMeta() (dataRetriever.StorageService, } } - statusMetricsDbConfig := GetDBFromConfig(psf.generalConfig.StatusMetricsStorage.DB) - shardId := core.GetShardIDString(psf.shardCoordinator.SelfId()) - dbPath = psf.pathManager.PathForStatic(shardId, psf.generalConfig.StatusMetricsStorage.DB.FilePath) - statusMetricsDbConfig.FilePath = dbPath - statusMetricsStorageUnit, err := storageunit.NewStorageUnitFromConf( - GetCacherFromConfig(psf.generalConfig.StatusMetricsStorage.Cache), - statusMetricsDbConfig) - if err != nil { - return nil, err - } - - trieEpochRootHashStorageUnit, err := psf.createTrieEpochRootHashStorerIfNeeded() - if err != nil { - return nil, err - } - - txUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.TxStorage, disabledCustomDatabaseRemover) - txUnit, err = psf.createPruningPersister(txUnitArgs) - if err != nil { - return nil, err - } - - unsignedTxUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.UnsignedTransactionStorage, disabledCustomDatabaseRemover) - unsignedTxUnit, err = psf.createPruningPersister(unsignedTxUnitArgs) - if err != nil { - return nil, err - } - - rewardTxUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.RewardTxStorage, disabledCustomDatabaseRemover) - rewardTxUnit, err = psf.createPruningPersister(rewardTxUnitArgs) - if err != nil { - return nil, err - } - - miniBlockUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.MiniBlocksStorage, disabledCustomDatabaseRemover) - miniBlockUnit, err = psf.createPruningPersister(miniBlockUnitArgs) - if err != nil { - return nil, err - } - - bootstrapUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.BootstrapStorage, disabledCustomDatabaseRemover) - bootstrapUnit, err = psf.createPruningPersister(bootstrapUnitArgs) - if err != nil { - return nil, err - } - - receiptsUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.ReceiptsStorage, disabledCustomDatabaseRemover) - receiptsUnit, err = psf.createPruningPersister(receiptsUnitArgs) - if err != nil { - return nil, err - } - - scheduledSCRsUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.ScheduledSCRsStorage, disabledCustomDatabaseRemover) - scheduledSCRsUnit, err = pruning.NewPruningStorer(scheduledSCRsUnitArgs) + store := dataRetriever.NewChainStorer() + err = psf.createAndAddBaseStorageUnits(store, shardID) if err != nil { return nil, err } - store := dataRetriever.NewChainStorer() - store.AddStorer(dataRetriever.MetaBlockUnit, metaBlockUnit) - store.AddStorer(dataRetriever.BlockHeaderUnit, headerUnit) - store.AddStorer(dataRetriever.MetaHdrNonceHashDataUnit, metaHdrHashNonceUnit) - store.AddStorer(dataRetriever.TransactionUnit, txUnit) - store.AddStorer(dataRetriever.UnsignedTransactionUnit, unsignedTxUnit) - store.AddStorer(dataRetriever.MiniBlockUnit, miniBlockUnit) - store.AddStorer(dataRetriever.RewardTransactionUnit, rewardTxUnit) for i := uint32(0); i < psf.shardCoordinator.NumberOfShards(); i++ { hdrNonceHashDataUnit := dataRetriever.ShardHdrNonceHashDataUnit + dataRetriever.UnitType(i) store.AddStorer(hdrNonceHashDataUnit, shardHdrHashNonceUnits[i]) } - store.AddStorer(dataRetriever.BootstrapUnit, bootstrapUnit) - store.AddStorer(dataRetriever.StatusMetricsUnit, statusMetricsStorageUnit) - store.AddStorer(dataRetriever.ReceiptsUnit, receiptsUnit) - store.AddStorer(dataRetriever.TrieEpochRootHashUnit, trieEpochRootHashStorageUnit) - store.AddStorer(dataRetriever.UserAccountsUnit, userAccountsUnit) - store.AddStorer(dataRetriever.PeerAccountsUnit, peerAccountsUnit) - store.AddStorer(dataRetriever.UserAccountsCheckpointsUnit, userAccountsCheckpointsUnit) - store.AddStorer(dataRetriever.PeerAccountsCheckpointsUnit, peerAccountsCheckpointsUnit) - store.AddStorer(dataRetriever.ScheduledSCRsUnit, scheduledSCRsUnit) err = psf.setupDbLookupExtensions(store) if err != nil { From b992f86c18dd490f4548493b9376023164ca5a12 Mon Sep 17 00:00:00 2001 From: jules01 Date: Thu, 27 Oct 2022 10:37:45 +0300 Subject: [PATCH 3/4] - made the peerAccountsUnit creation different between shard and meta --- storage/factory/pruningStorerFactory.go | 52 +++++++++++++++---------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/storage/factory/pruningStorerFactory.go b/storage/factory/pruningStorerFactory.go index d7551fa954a..fd262085e46 100644 --- a/storage/factory/pruningStorerFactory.go +++ b/storage/factory/pruningStorerFactory.go @@ -111,12 +111,12 @@ func checkArgs(args StorageServiceFactoryArgs) error { return nil } -func (psf *StorageServiceFactory) createAndAddBaseStorageUnits(store dataRetriever.StorageService, shardID string) error { +func (psf *StorageServiceFactory) createAndAddBaseStorageUnits( + store dataRetriever.StorageService, + customDatabaseRemover storage.CustomDatabaseRemoverHandler, + shardID string, +) error { disabledCustomDatabaseRemover := disabled.NewDisabledCustomDatabaseRemover() - customDatabaseRemover, err := factory.CreateCustomDatabaseRemover(psf.generalConfig.StoragePruning) - if err != nil { - return err - } txUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.TxStorage, disabledCustomDatabaseRemover) txUnit, err := psf.createPruningPersister(txUnitArgs) @@ -199,12 +199,6 @@ func (psf *StorageServiceFactory) createAndAddBaseStorageUnits(store dataRetriev } store.AddStorer(dataRetriever.UserAccountsUnit, userAccountsUnit) - peerAccountsUnit, err := psf.createTriePruningStorer(psf.generalConfig.PeerAccountsTrieStorage, customDatabaseRemover) - if err != nil { - return err - } - store.AddStorer(dataRetriever.PeerAccountsUnit, peerAccountsUnit) - userAccountsCheckpointsUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.AccountsTrieCheckpointsStorage, disabledCustomDatabaseRemover) userAccountsCheckpointsUnit, err := psf.createPruningPersister(userAccountsCheckpointsUnitArgs) if err != nil { @@ -242,16 +236,11 @@ func (psf *StorageServiceFactory) createAndAddBaseStorageUnits(store dataRetriev // CreateForShard will return the storage service which contains all storers needed for a shard func (psf *StorageServiceFactory) CreateForShard() (dataRetriever.StorageService, error) { - var peerBlockUnit storage.Storer - var err error - // TODO: if there will be a differentiation between the creation or opening of a DB, the DBs could be destroyed on a defer // in case of a failure while creating (not opening). disabledCustomDatabaseRemover := disabled.NewDisabledCustomDatabaseRemover() - - peerBlockUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.PeerBlockBodyStorage, disabledCustomDatabaseRemover) - peerBlockUnit, err = psf.createPruningPersister(peerBlockUnitArgs) + customDatabaseRemover, err := factory.CreateCustomDatabaseRemover(psf.generalConfig.StoragePruning) if err != nil { return nil, err } @@ -271,12 +260,25 @@ func (psf *StorageServiceFactory) CreateForShard() (dataRetriever.StorageService } store := dataRetriever.NewChainStorer() - err = psf.createAndAddBaseStorageUnits(store, shardID) + err = psf.createAndAddBaseStorageUnits(store, customDatabaseRemover, shardID) + if err != nil { + return nil, err + } + + peerAccountsUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.PeerAccountsTrieStorage, customDatabaseRemover) + peerAccountsUnit, err := psf.createTrieUnit(psf.generalConfig.PeerAccountsTrieStorage, peerAccountsUnitArgs) if err != nil { return nil, err } + store.AddStorer(dataRetriever.PeerAccountsUnit, peerAccountsUnit) + peerBlockUnitArgs := psf.createPruningStorerArgs(psf.generalConfig.PeerBlockBodyStorage, disabledCustomDatabaseRemover) + peerBlockUnit, err := psf.createPruningPersister(peerBlockUnitArgs) + if err != nil { + return nil, err + } store.AddStorer(dataRetriever.PeerChangesUnit, peerBlockUnit) + hdrNonceHashDataUnit := dataRetriever.ShardHdrNonceHashDataUnit + dataRetriever.UnitType(psf.shardCoordinator.SelfId()) store.AddStorer(hdrNonceHashDataUnit, shardHdrHashNonceUnit) @@ -303,11 +305,13 @@ func (psf *StorageServiceFactory) CreateForShard() (dataRetriever.StorageService // CreateForMeta will return the storage service which contains all storers needed for metachain func (psf *StorageServiceFactory) CreateForMeta() (dataRetriever.StorageService, error) { - var err error - // TODO: if there will be a differentiation between the creation or opening of a DB, the DBs could be destroyed on a defer // in case of a failure while creating (not opening) + customDatabaseRemover, err := factory.CreateCustomDatabaseRemover(psf.generalConfig.StoragePruning) + if err != nil { + return nil, err + } shardID := core.GetShardIDString(core.MetachainShardId) shardHdrHashNonceUnits := make([]*storageunit.Unit, psf.shardCoordinator.NumberOfShards()) @@ -325,11 +329,17 @@ func (psf *StorageServiceFactory) CreateForMeta() (dataRetriever.StorageService, } store := dataRetriever.NewChainStorer() - err = psf.createAndAddBaseStorageUnits(store, shardID) + err = psf.createAndAddBaseStorageUnits(store, customDatabaseRemover, shardID) if err != nil { return nil, err } + peerAccountsUnit, err := psf.createTriePruningStorer(psf.generalConfig.PeerAccountsTrieStorage, customDatabaseRemover) + if err != nil { + return nil, err + } + store.AddStorer(dataRetriever.PeerAccountsUnit, peerAccountsUnit) + for i := uint32(0); i < psf.shardCoordinator.NumberOfShards(); i++ { hdrNonceHashDataUnit := dataRetriever.ShardHdrNonceHashDataUnit + dataRetriever.UnitType(i) store.AddStorer(hdrNonceHashDataUnit, shardHdrHashNonceUnits[i]) From a85aca1832b396bc3bc62817942f7299c457ad88 Mon Sep 17 00:00:00 2001 From: Bogdan Rosianu Date: Thu, 27 Oct 2022 11:00:51 +0300 Subject: [PATCH 4/4] fix golangci lint --- storage/factory/pruningStorerFactory.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/storage/factory/pruningStorerFactory.go b/storage/factory/pruningStorerFactory.go index fd262085e46..f4e84d1f19a 100644 --- a/storage/factory/pruningStorerFactory.go +++ b/storage/factory/pruningStorerFactory.go @@ -249,7 +249,6 @@ func (psf *StorageServiceFactory) CreateForShard() (dataRetriever.StorageService // shardHdrHashNonce storer is static shardHdrHashNonceConfig := GetDBFromConfig(psf.generalConfig.ShardHdrNonceHashStorage.DB) - shardID = core.GetShardIDString(psf.shardCoordinator.SelfId()) dbPath := psf.pathManager.PathForStatic(shardID, psf.generalConfig.ShardHdrNonceHashStorage.DB.FilePath) + shardID shardHdrHashNonceConfig.FilePath = dbPath shardHdrHashNonceUnit, err := storageunit.NewStorageUnitFromConf( @@ -300,9 +299,6 @@ func (psf *StorageServiceFactory) CreateForShard() (dataRetriever.StorageService return store, err } -// TODO: split in 2 components shardStorageServiceCreator and metStorageServiceCreator that have a base which will contain the -// common storers - // CreateForMeta will return the storage service which contains all storers needed for metachain func (psf *StorageServiceFactory) CreateForMeta() (dataRetriever.StorageService, error) { // TODO: if there will be a differentiation between the creation or opening of a DB, the DBs could be destroyed on a defer