From 337c066790478e32013e4189a3a644b699edcdd6 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 7 Dec 2023 13:06:12 +0200 Subject: [PATCH] Applied PR suggestions regarding cache size --- cmd/scaffold.go | 3 ++- consensus/integration/nodes_test.go | 3 ++- integration/testnet/container.go | 3 ++- storage/badger/all.go | 3 ++- storage/badger/protocol_state.go | 16 +++++++++++++--- storage/badger/protocol_state_test.go | 8 ++++---- 6 files changed, 25 insertions(+), 11 deletions(-) diff --git a/cmd/scaffold.go b/cmd/scaffold.go index b192d7c9a28..f878c512ea5 100644 --- a/cmd/scaffold.go +++ b/cmd/scaffold.go @@ -1005,7 +1005,8 @@ func (fnb *FlowNodeBuilder) initStorage() error { setups := bstorage.NewEpochSetups(fnb.Metrics.Cache, fnb.DB) epochCommits := bstorage.NewEpochCommits(fnb.Metrics.Cache, fnb.DB) commits := bstorage.NewCommits(fnb.Metrics.Cache, fnb.DB) - protocolState := bstorage.NewProtocolState(fnb.Metrics.Cache, setups, epochCommits, fnb.DB, bstorage.DefaultCacheSize) + protocolState := bstorage.NewProtocolState(fnb.Metrics.Cache, setups, epochCommits, fnb.DB, + bstorage.DefaultProtocolStateCacheSize, bstorage.DefaultProtocolStateByBlockIDCacheSize) versionBeacons := bstorage.NewVersionBeacons(fnb.DB) fnb.Storage = Storage{ diff --git a/consensus/integration/nodes_test.go b/consensus/integration/nodes_test.go index c44775cd14b..ef5501d189d 100644 --- a/consensus/integration/nodes_test.go +++ b/consensus/integration/nodes_test.go @@ -377,7 +377,8 @@ func createNode( qcsDB := storage.NewQuorumCertificates(metricsCollector, db, storage.DefaultCacheSize) setupsDB := storage.NewEpochSetups(metricsCollector, db) commitsDB := storage.NewEpochCommits(metricsCollector, db) - protocolStateDB := storage.NewProtocolState(metricsCollector, setupsDB, commitsDB, db, storage.DefaultCacheSize) + protocolStateDB := storage.NewProtocolState(metricsCollector, setupsDB, commitsDB, db, + storage.DefaultProtocolStateCacheSize, storage.DefaultProtocolStateByBlockIDCacheSize) versionBeaconDB := storage.NewVersionBeacons(db) protocolStateEvents := events.NewDistributor() diff --git a/integration/testnet/container.go b/integration/testnet/container.go index ab51f4a44e4..4e181df97e1 100644 --- a/integration/testnet/container.go +++ b/integration/testnet/container.go @@ -395,7 +395,8 @@ func (c *Container) OpenState() (*state.State, error) { qcs := storage.NewQuorumCertificates(metrics, db, storage.DefaultCacheSize) setups := storage.NewEpochSetups(metrics, db) commits := storage.NewEpochCommits(metrics, db) - protocolState := storage.NewProtocolState(metrics, setups, commits, db, storage.DefaultCacheSize) + protocolState := storage.NewProtocolState(metrics, setups, commits, db, + storage.DefaultProtocolStateCacheSize, storage.DefaultProtocolStateByBlockIDCacheSize) versionBeacons := storage.NewVersionBeacons(db) return state.OpenState( diff --git a/storage/badger/all.go b/storage/badger/all.go index 9ee0aa24002..8d1a718044c 100644 --- a/storage/badger/all.go +++ b/storage/badger/all.go @@ -19,7 +19,8 @@ func InitAll(metrics module.CacheMetrics, db *badger.DB) *storage.All { qcs := NewQuorumCertificates(metrics, db, DefaultCacheSize) setups := NewEpochSetups(metrics, db) epochCommits := NewEpochCommits(metrics, db) - protocolState := NewProtocolState(metrics, setups, epochCommits, db, DefaultCacheSize) + protocolState := NewProtocolState(metrics, setups, epochCommits, db, + DefaultProtocolStateCacheSize, DefaultProtocolStateByBlockIDCacheSize) versionBeacons := NewVersionBeacons(db) commits := NewCommits(metrics, db) diff --git a/storage/badger/protocol_state.go b/storage/badger/protocol_state.go index c1ce172d7ef..b5d588cbd30 100644 --- a/storage/badger/protocol_state.go +++ b/storage/badger/protocol_state.go @@ -15,6 +15,15 @@ import ( "github.com/onflow/flow-go/storage/badger/transaction" ) +// DefaultProtocolStateCacheSize is the default size for primary protocol state cache. +// Minimally, we have 3 entries per epoch (one on epoch Switchover, one on receiving the Epoch Setup and one when seeing the Epoch Commit event). +// Lets be generous and assume we have 20 different Protocol States per epoch. +var DefaultProtocolStateCacheSize uint = 20 + +// DefaultProtocolStateByBlockIDCacheSize is the default value for secondary byBlockIdCache. +// We want to be able to cover a broad interval of views without cache misses, so we use a bigger value. +var DefaultProtocolStateByBlockIDCacheSize uint = 1000 + // ProtocolState implements persistent storage for storing Protocol States. // Protocol state uses an embedded cache without storing capabilities(store happens on first retrieval) to avoid unnecessary // operations and to speed up access to frequently used Protocol State. @@ -59,7 +68,8 @@ func NewProtocolState(collector module.CacheMetrics, epochSetups storage.EpochSetups, epochCommits storage.EpochCommits, db *badger.DB, - cacheSize uint, + stateCacheSize uint, + stateByBlockIDCacheSize uint, ) *ProtocolState { retrieveByProtocolStateID := func(protocolStateID flow.Identifier) func(tx *badger.Txn) (*flow.RichProtocolStateEntry, error) { var protocolStateEntry flow.ProtocolStateEntry @@ -100,11 +110,11 @@ func NewProtocolState(collector module.CacheMetrics, return &ProtocolState{ db: db, cache: newCache[flow.Identifier, *flow.RichProtocolStateEntry](collector, metrics.ResourceProtocolState, - withLimit[flow.Identifier, *flow.RichProtocolStateEntry](cacheSize), + withLimit[flow.Identifier, *flow.RichProtocolStateEntry](stateCacheSize), withStore(noopStore[flow.Identifier, *flow.RichProtocolStateEntry]), withRetrieve(retrieveByProtocolStateID)), byBlockIdCache: newCache[flow.Identifier, flow.Identifier](collector, metrics.ResourceProtocolStateByBlockID, - withLimit[flow.Identifier, flow.Identifier](cacheSize), + withLimit[flow.Identifier, flow.Identifier](stateByBlockIDCacheSize), withStore(storeByBlockID), withRetrieve(retrieveByBlockID)), } diff --git a/storage/badger/protocol_state_test.go b/storage/badger/protocol_state_test.go index 701d4e282e9..857b7fd6900 100644 --- a/storage/badger/protocol_state_test.go +++ b/storage/badger/protocol_state_test.go @@ -22,7 +22,7 @@ func TestProtocolStateStorage(t *testing.T) { setups := NewEpochSetups(metrics, db) commits := NewEpochCommits(metrics, db) - store := NewProtocolState(metrics, setups, commits, db, DefaultCacheSize) + store := NewProtocolState(metrics, setups, commits, db, DefaultProtocolStateCacheSize, DefaultProtocolStateByBlockIDCacheSize) expected := unittest.ProtocolStateFixture(unittest.WithNextEpochProtocolState()) protocolStateID := expected.ID() @@ -73,7 +73,7 @@ func TestProtocolStateStoreInvalidProtocolState(t *testing.T) { metrics := metrics.NewNoopCollector() setups := NewEpochSetups(metrics, db) commits := NewEpochCommits(metrics, db) - store := NewProtocolState(metrics, setups, commits, db, DefaultCacheSize) + store := NewProtocolState(metrics, setups, commits, db, DefaultProtocolStateCacheSize, DefaultProtocolStateByBlockIDCacheSize) invalid := unittest.ProtocolStateFixture().ProtocolStateEntry // swap first and second elements to break canonical order invalid.CurrentEpoch.ActiveIdentities[0], invalid.CurrentEpoch.ActiveIdentities[1] = invalid.CurrentEpoch.ActiveIdentities[1], invalid.CurrentEpoch.ActiveIdentities[0] @@ -99,7 +99,7 @@ func TestProtocolStateMergeParticipants(t *testing.T) { setups := NewEpochSetups(metrics, db) commits := NewEpochCommits(metrics, db) - store := NewProtocolState(metrics, setups, commits, db, DefaultCacheSize) + store := NewProtocolState(metrics, setups, commits, db, DefaultProtocolStateCacheSize, DefaultProtocolStateByBlockIDCacheSize) stateEntry := unittest.ProtocolStateFixture() // change address of participant in current epoch, so we can distinguish it from the one in previous epoch @@ -148,7 +148,7 @@ func TestProtocolStateRootSnapshot(t *testing.T) { setups := NewEpochSetups(metrics, db) commits := NewEpochCommits(metrics, db) - store := NewProtocolState(metrics, setups, commits, db, DefaultCacheSize) + store := NewProtocolState(metrics, setups, commits, db, DefaultProtocolStateCacheSize, DefaultProtocolStateByBlockIDCacheSize) expected := unittest.RootProtocolStateFixture() protocolStateID := expected.ID()