Feat/elrond-go-storage

api/shared/logging/logging.go

@@ -8,6 +8,7 @@ import (
 )
 
 var log = logger.GetOrCreate("api/shared/logging")
+
 const thresholdMinAPICallDurationToLog = 200 * time.Millisecond
 
 // LogAPIActionDurationIfNeeded will log the duration of an action triggered by an API call if it's above a threshold

common/constants.go

@@ -75,15 +75,6 @@ const ConnectionTopic = "connection"
 // ValidatorInfoTopic is the topic used for validatorInfo signaling
 const ValidatorInfoTopic = "validatorInfo"
 
-// PathShardPlaceholder represents the placeholder for the shard ID in paths
-const PathShardPlaceholder = "[S]"
-
-// PathEpochPlaceholder represents the placeholder for the epoch number in paths
-const PathEpochPlaceholder = "[E]"
-
-// PathIdentifierPlaceholder represents the placeholder for the identifier in paths
-const PathIdentifierPlaceholder = "[I]"
-
 // MetricCurrentRound is the metric for monitoring the current round of a node
 const MetricCurrentRound = "erd_current_round"
 
@@ -301,7 +292,7 @@ const MetricCreatedProposedBlock = "erd_consensus_created_proposed_block"
 // MetricRedundancyLevel is the metric that specifies the redundancy level of the current node
 const MetricRedundancyLevel = "erd_redundancy_level"
 
-// MetricRedundancyIsMainActive is the metrics that specifies data about the redundancy main machine
+// MetricRedundancyIsMainActive is the metric that specifies data about the redundancy main machine
 const MetricRedundancyIsMainActive = "erd_redundancy_is_main_active"
 
 // MetricValueNA represents the value to be used when a metric is not available/applicable
@@ -683,27 +674,12 @@ const WrongConfiguration = "wrongConfiguration"
 // ImportComplete signals that a node restart will be done because the import did complete
 const ImportComplete = "importComplete"
 
-// MaxRetriesToCreateDB represents the maximum number of times to try to create DB if it failed
-const MaxRetriesToCreateDB = 10
-
-// SleepTimeBetweenCreateDBRetries represents the number of seconds to sleep between DB creates
-const SleepTimeBetweenCreateDBRetries = 5 * time.Second
-
 // DefaultStatsPath is the default path where the node stats are logged
 const DefaultStatsPath = "stats"
 
 // DefaultDBPath is the default path for nodes databases
 const DefaultDBPath = "db"
 
-// DefaultEpochString is the default folder root name for node per epoch databases
-const DefaultEpochString = "Epoch"
-
-// DefaultStaticDbString is the default name for the static databases (not changing with epoch)
-const DefaultStaticDbString = "Static"
-
-// DefaultShardString is the default folder root name for per shard databases
-const DefaultShardString = "Shard"
-
 // MetachainShardName is the string identifier of the metachain shard
 const MetachainShardName = "metachain"
 

consensus/broadcast/delayedBroadcast.go

@@ -18,7 +18,7 @@ import (
 	"github.com/ElrondNetwork/elrond-go/process/factory"
 	"github.com/ElrondNetwork/elrond-go/sharding"
 	"github.com/ElrondNetwork/elrond-go/storage"
-	"github.com/ElrondNetwork/elrond-go/storage/lrucache"
+	"github.com/ElrondNetwork/elrond-go/storage/cache"
 )
 
 const prefixHeaderAlarm = "header_"
@@ -98,7 +98,7 @@ func NewDelayedBlockBroadcaster(args *ArgsDelayedBlockBroadcaster) (*delayedBloc
 		return nil, spos.ErrNilAlarmScheduler
 	}
 
-	cacheHeaders, err := lrucache.NewCache(sizeHeadersCache)
+	cacheHeaders, err := cache.NewLRUCache(sizeHeadersCache)
 	if err != nil {
 		return nil, err
 	}

dataRetriever/constants.go

@@ -3,9 +3,6 @@ package dataRetriever
 // TxPoolNumSendersToPreemptivelyEvict instructs tx pool eviction algorithm to remove this many senders when eviction takes place
 const TxPoolNumSendersToPreemptivelyEvict = uint32(100)
 
-// TxPoolNumTxsToPreemptivelyEvict instructs tx pool eviction algorithm to remove this many transactions when eviction takes place
-const TxPoolNumTxsToPreemptivelyEvict = uint32(1000)
-
 // UnsignedTxPoolName defines the name of the unsigned transactions pool
 const UnsignedTxPoolName = "uTxPool"
 

dataRetriever/factory/dataPoolFactory.go

@@ -18,12 +18,10 @@ import (
 	"github.com/ElrondNetwork/elrond-go/process"
 	"github.com/ElrondNetwork/elrond-go/sharding"
 	"github.com/ElrondNetwork/elrond-go/storage"
+	"github.com/ElrondNetwork/elrond-go/storage/cache"
 	"github.com/ElrondNetwork/elrond-go/storage/disabled"
 	"github.com/ElrondNetwork/elrond-go/storage/factory"
-	"github.com/ElrondNetwork/elrond-go/storage/lrucache/capacity"
-	"github.com/ElrondNetwork/elrond-go/storage/storageCacherAdapter"
-	"github.com/ElrondNetwork/elrond-go/storage/storageUnit"
-	"github.com/ElrondNetwork/elrond-go/storage/timecache"
+	"github.com/ElrondNetwork/elrond-go/storage/storageunit"
 	trieFactory "github.com/ElrondNetwork/elrond-go/trie/factory"
 )
 
@@ -88,18 +86,18 @@ func NewDataPoolFromConfig(args ArgsDataPool) (dataRetriever.PoolsHolder, error)
 	}
 
 	cacherCfg := factory.GetCacherFromConfig(mainConfig.TxBlockBodyDataPool)
-	txBlockBody, err := storageUnit.NewCache(cacherCfg)
+	txBlockBody, err := storageunit.NewCache(cacherCfg)
 	if err != nil {
 		return nil, fmt.Errorf("%w while creating the cache for the miniblocks", err)
 	}
 
 	cacherCfg = factory.GetCacherFromConfig(mainConfig.PeerBlockBodyDataPool)
-	peerChangeBlockBody, err := storageUnit.NewCache(cacherCfg)
+	peerChangeBlockBody, err := storageunit.NewCache(cacherCfg)
 	if err != nil {
 		return nil, fmt.Errorf("%w while creating the cache for the peer mini block body", err)
 	}
 
-	cacher, err := capacity.NewCapacityLRU(
+	cacher, err := cache.NewCapacityLRU(
 		int(mainConfig.TrieSyncStorage.Capacity),
 		int64(mainConfig.TrieSyncStorage.SizeInBytes),
 	)
@@ -113,24 +111,24 @@ func NewDataPoolFromConfig(args ArgsDataPool) (dataRetriever.PoolsHolder, error)
 	}
 
 	tnf := trieFactory.NewTrieNodeFactory()
-	adaptedTrieNodesStorage, err := storageCacherAdapter.NewStorageCacherAdapter(cacher, trieSyncDB, tnf, args.Marshalizer)
+	adaptedTrieNodesStorage, err := storageunit.NewStorageCacherAdapter(cacher, trieSyncDB, tnf, args.Marshalizer)
 	if err != nil {
 		return nil, fmt.Errorf("%w while creating the adapter for the trie nodes", err)
 	}
 
 	cacherCfg = factory.GetCacherFromConfig(mainConfig.TrieNodesChunksDataPool)
-	trieNodesChunks, err := storageUnit.NewCache(cacherCfg)
+	trieNodesChunks, err := storageunit.NewCache(cacherCfg)
 	if err != nil {
 		return nil, fmt.Errorf("%w while creating the cache for the trie chunks", err)
 	}
 
 	cacherCfg = factory.GetCacherFromConfig(mainConfig.SmartContractDataPool)
-	smartContracts, err := storageUnit.NewCache(cacherCfg)
+	smartContracts, err := storageunit.NewCache(cacherCfg)
 	if err != nil {
 		return nil, fmt.Errorf("%w while creating the cache for the smartcontract results", err)
 	}
 
-	peerAuthPool, err := timecache.NewTimeCacher(timecache.ArgTimeCacher{
+	peerAuthPool, err := cache.NewTimeCacher(cache.ArgTimeCacher{
 		DefaultSpan: time.Duration(mainConfig.HeartbeatV2.PeerAuthenticationTimeBetweenSendsInSec) * time.Second * peerAuthExpiryMultiplier,
 		CacheExpiry: peerAuthenticationCacheRefresh,
 	})
@@ -139,7 +137,7 @@ func NewDataPoolFromConfig(args ArgsDataPool) (dataRetriever.PoolsHolder, error)
 	}
 
 	cacherCfg = factory.GetCacherFromConfig(mainConfig.HeartbeatV2.HeartbeatPool)
-	heartbeatPool, err := storageUnit.NewCache(cacherCfg)
+	heartbeatPool, err := storageunit.NewCache(cacherCfg)
 	if err != nil {
 		return nil, fmt.Errorf("%w while creating the cache for the heartbeat messages", err)
 	}
@@ -180,7 +178,7 @@ func createTrieSyncDB(args ArgsDataPool) (storage.Persister, error) {
 
 	dbCfg := factory.GetDBFromConfig(mainConfig.TrieSyncStorage.DB)
 	shardId := core.GetShardIDString(args.ShardCoordinator.SelfId())
-	argDB := storageUnit.ArgDB{
+	argDB := storageunit.ArgDB{
 		DBType:            dbCfg.Type,
 		Path:              args.PathManager.PathForStatic(shardId, mainConfig.TrieSyncStorage.DB.FilePath),
 		BatchDelaySeconds: dbCfg.BatchDelaySeconds,
@@ -197,7 +195,7 @@ func createTrieSyncDB(args ArgsDataPool) (storage.Persister, error) {
 		argDB.Path = filePath
 	}
 
-	db, err := storageUnit.NewDB(argDB)
+	db, err := storageunit.NewDB(argDB)
 	if err != nil {
 		return nil, fmt.Errorf("%w while creating the db for the trie nodes", err)
 	}

dataRetriever/requestHandlers/requestHandler_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/ElrondNetwork/elrond-go/common"
 	"github.com/ElrondNetwork/elrond-go/dataRetriever"
 	"github.com/ElrondNetwork/elrond-go/dataRetriever/mock"
-	"github.com/ElrondNetwork/elrond-go/storage/timecache"
+	cache "github.com/ElrondNetwork/elrond-go/storage/cache"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -203,14 +203,14 @@ func TestResolverRequestHandler_RequestTransactionShouldRequest4TimesIfDifferent
 	}
 
 	timeSpan := time.Second
-	cache := timecache.NewTimeCache(timeSpan)
+	timeCache := cache.NewTimeCache(timeSpan)
 	rrh, _ := NewResolverRequestHandler(
 		&mock.ResolversFinderStub{
 			CrossShardResolverCalled: func(baseTopic string, crossShard uint32) (resolver dataRetriever.Resolver, e error) {
 				return txResolver, nil
 			},
 		},
-		cache,
+		timeCache,
 		&mock.WhiteListHandlerStub{},
 		1,
 		0,

dataRetriever/shardedData/shardedData.go

@@ -9,8 +9,8 @@ import (
 	logger "github.com/ElrondNetwork/elrond-go-logger"
 	"github.com/ElrondNetwork/elrond-go/dataRetriever"
 	"github.com/ElrondNetwork/elrond-go/storage"
-	"github.com/ElrondNetwork/elrond-go/storage/immunitycache"
-	"github.com/ElrondNetwork/elrond-go/storage/storageUnit"
+	"github.com/ElrondNetwork/elrond-go/storage/cache"
+	"github.com/ElrondNetwork/elrond-go/storage/storageunit"
 )
 
 var log = logger.GetOrCreate("dataretriever/shardeddata")
@@ -32,7 +32,7 @@ type shardedData struct {
 	// Each key represents a destination shard id and the value will contain all
 	// data hashes that have that shard as destination
 	shardedDataStore map[string]*shardStore
-	configPrototype  immunitycache.CacheConfig
+	configPrototype  cache.CacheConfig
 
 	mutAddedDataHandlers sync.RWMutex
 	addedDataHandlers    []func(key []byte, value interface{})
@@ -44,15 +44,15 @@ type shardStore struct {
 }
 
 // NewShardedData is responsible for creating an empty pool of data
-func NewShardedData(name string, config storageUnit.CacheConfig) (*shardedData, error) {
+func NewShardedData(name string, config storageunit.CacheConfig) (*shardedData, error) {
 	log.Debug("NewShardedData", "name", name, "config", config.String())
 
-	configPrototype := immunitycache.CacheConfig{
+	configPrototype := cache.CacheConfig{
 		Name:                        untitledCacheName,
 		NumChunks:                   config.Shards,
 		MaxNumItems:                 config.Capacity,
 		MaxNumBytes:                 uint32(config.SizeInBytes),
-		NumItemsToPreemptivelyEvict: dataRetriever.TxPoolNumTxsToPreemptivelyEvict,
+		NumItemsToPreemptivelyEvict: storage.TxPoolNumTxsToPreemptivelyEvict,
 	}
 
 	err := configPrototype.Verify()
@@ -128,14 +128,14 @@ func (sd *shardedData) addShardStoreNoLock(cacheID string) *shardStore {
 func (sd *shardedData) newShardStore(cacheID string) (*shardStore, error) {
 	config := sd.configPrototype
 	config.Name = fmt.Sprintf("%s:%s", sd.name, cacheID)
-	cache, err := immunitycache.NewImmunityCache(config)
+	newImmunityCache, err := cache.NewImmunityCache(config)
 	if err != nil {
 		return nil, err
 	}
 
 	return &shardStore{
 		cacheID: cacheID,
-		cache:   cache,
+		cache:   newImmunityCache,
 	}, nil
 }
 
@@ -263,8 +263,8 @@ func (sd *shardedData) Keys() [][]byte {
 
 	keys := make([][]byte, 0)
 	for _, shard := range sd.shardedDataStore {
-		cache := shard.cache
-		keys = append(keys, cache.Keys()...)
+		c := shard.cache
+		keys = append(keys, c.Keys()...)
 	}
 
 	return keys
@@ -278,8 +278,8 @@ func (sd *shardedData) GetCounts() counting.CountsWithSize {
 	counts := counting.NewConcurrentShardedCountsWithSize()
 
 	for cacheID, shard := range sd.shardedDataStore {
-		cache := shard.cache
-		counts.PutCounts(cacheID, int64(cache.Len()), int64(cache.NumBytes()))
+		c := shard.cache
+		counts.PutCounts(cacheID, int64(c.Len()), int64(c.NumBytes()))
 	}
 
 	return counts

dataRetriever/shardedData/shardedData_test.go

@@ -10,13 +10,13 @@ import (
 
 	"github.com/ElrondNetwork/elrond-go-core/core/check"
 	"github.com/ElrondNetwork/elrond-go-core/data/transaction"
-	"github.com/ElrondNetwork/elrond-go/storage/storageUnit"
+	"github.com/ElrondNetwork/elrond-go/storage/storageunit"
 	"github.com/stretchr/testify/assert"
 )
 
 var timeoutWaitForWaitGroups = time.Second * 2
 
-var defaultTestConfig = storageUnit.CacheConfig{
+var defaultTestConfig = storageunit.CacheConfig{
 	Capacity:    75000,
 	SizeInBytes: 104857600,
 	Shards:      1,
@@ -25,7 +25,7 @@ var defaultTestConfig = storageUnit.CacheConfig{
 func TestNewShardedData_BadConfigShouldErr(t *testing.T) {
 	t.Parallel()
 
-	cacheConfigBad := storageUnit.CacheConfig{
+	cacheConfigBad := storageunit.CacheConfig{
 		Capacity: 0,
 	}
 

dataRetriever/txpool/argShardedTxPool.go

@@ -6,13 +6,13 @@ import (
 
 	"github.com/ElrondNetwork/elrond-go-core/core/check"
 	"github.com/ElrondNetwork/elrond-go/dataRetriever"
-	"github.com/ElrondNetwork/elrond-go/storage/storageUnit"
+	"github.com/ElrondNetwork/elrond-go/storage/storageunit"
 	"github.com/ElrondNetwork/elrond-go/storage/txcache"
 )
 
 // ArgShardedTxPool is the argument for ShardedTxPool's constructor
 type ArgShardedTxPool struct {
-	Config         storageUnit.CacheConfig
+	Config         storageunit.CacheConfig
 	TxGasHandler   txcache.TxGasHandler
 	NumberOfShards uint32
 	SelfShardID    uint32

dataRetriever/txpool/memorytests/memory_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/ElrondNetwork/elrond-go-core/data/transaction"
 	"github.com/ElrondNetwork/elrond-go/dataRetriever"
 	"github.com/ElrondNetwork/elrond-go/dataRetriever/txpool"
-	"github.com/ElrondNetwork/elrond-go/storage/storageUnit"
+	"github.com/ElrondNetwork/elrond-go/storage/storageunit"
 	"github.com/ElrondNetwork/elrond-go/testscommon/txcachemocks"
 	"github.com/stretchr/testify/require"
 )
@@ -101,7 +101,7 @@ type memoryAssertion struct {
 }
 
 func newPool() dataRetriever.ShardedDataCacherNotifier {
-	config := storageUnit.CacheConfig{
+	config := storageunit.CacheConfig{
 		Capacity:             600000,
 		SizePerSender:        60000,
 		SizeInBytes:          400 * core.MegabyteSize,
@@ -229,7 +229,9 @@ func pprofHeap(scenario *scenario, step string) {
 
 	defer func() {
 		errClose := file.Close()
-		panic(fmt.Sprintf("cannot close file: %s", errClose.Error()))
+		if errClose != nil {
+			panic(fmt.Sprintf("cannot close file: %s", errClose.Error()))
+		}
 	}()
 
 	err = pprof.WriteHeapProfile(file)

dataRetriever/txpool/shardedTxPool.go

@@ -66,7 +66,7 @@ func NewShardedTxPool(args ArgShardedTxPool) (*shardedTxPool, error) {
 		NumChunks:                   args.Config.Shards,
 		MaxNumBytes:                 uint32(halfOfSizeInBytes) / numCrossTxCaches,
 		MaxNumItems:                 halfOfCapacity / numCrossTxCaches,
-		NumItemsToPreemptivelyEvict: dataRetriever.TxPoolNumTxsToPreemptivelyEvict,
+		NumItemsToPreemptivelyEvict: storage.TxPoolNumTxsToPreemptivelyEvict,
 	}
 
 	shardedTxPoolObject := &shardedTxPool{