From cf86037ea9c5b5c3638fd7f0ed80d8d031a25009 Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Fri, 31 Mar 2023 16:30:02 +0200 Subject: [PATCH] storage layer test refactor --- engine/access/access_test.go | 84 +++++++++++------------ module/builder/collection/builder_test.go | 29 ++++---- state/cluster/badger/mutator_test.go | 8 +-- state/cluster/badger/snapshot_test.go | 6 +- state/protocol/badger/mutator_test.go | 34 +++++---- state/protocol/badger/state_test.go | 5 +- state/protocol/util/testing.go | 58 ++++++++-------- storage/util/testing.go | 40 ++--------- 8 files changed, 118 insertions(+), 146 deletions(-) diff --git a/engine/access/access_test.go b/engine/access/access_test.go index 483d6442acd..6c16f01fc00 100644 --- a/engine/access/access_test.go +++ b/engine/access/access_test.go @@ -39,7 +39,8 @@ import ( "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/mocknetwork" protocol "github.com/onflow/flow-go/state/protocol/mock" - storage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/storage" + bstorage "github.com/onflow/flow-go/storage/badger" "github.com/onflow/flow-go/storage/badger/operation" "github.com/onflow/flow-go/storage/util" "github.com/onflow/flow-go/utils/unittest" @@ -109,23 +110,20 @@ func (suite *Suite) SetupTest() { } func (suite *Suite) RunTest( - f func(handler *access.Handler, db *badger.DB, blocks *storage.Blocks, headers *storage.Headers, results *storage.ExecutionResults), + f func(handler *access.Handler, db *badger.DB, all *storage.All), ) { unittest.RunWithBadgerDB(suite.T(), func(db *badger.DB) { - headers, _, _, _, _, blocks, _, _, _, _, results := util.StorageLayer(suite.T(), db) - transactions := storage.NewTransactions(suite.metrics, db) - collections := storage.NewCollections(db, transactions) - receipts := storage.NewExecutionReceipts(suite.metrics, db, results, storage.DefaultCacheSize) + all := util.StorageLayer(suite.T(), db) suite.backend = backend.New(suite.state, suite.collClient, nil, - blocks, - headers, - collections, - transactions, - receipts, - results, + all.Blocks, + all.Headers, + all.Collections, + all.Transactions, + all.Receipts, + all.Results, suite.chainID, suite.metrics, nil, @@ -138,12 +136,12 @@ func (suite *Suite) RunTest( ) handler := access.NewHandler(suite.backend, suite.chainID.Chain(), access.WithBlockSignerDecoder(suite.signerIndicesDecoder)) - f(handler, db, blocks, headers, results) + f(handler, db, all) }) } func (suite *Suite) TestSendAndGetTransaction() { - suite.RunTest(func(handler *access.Handler, _ *badger.DB, _ *storage.Blocks, _ *storage.Headers, _ *storage.ExecutionResults) { + suite.RunTest(func(handler *access.Handler, _ *badger.DB, _ *storage.All) { referenceBlock := unittest.BlockHeaderFixture() transaction := unittest.TransactionFixture() transaction.SetReferenceBlockID(referenceBlock.ID()) @@ -196,7 +194,7 @@ func (suite *Suite) TestSendAndGetTransaction() { } func (suite *Suite) TestSendExpiredTransaction() { - suite.RunTest(func(handler *access.Handler, _ *badger.DB, _ *storage.Blocks, _ *storage.Headers, _ *storage.ExecutionResults) { + suite.RunTest(func(handler *access.Handler, _ *badger.DB, _ *storage.All) { referenceBlock := unittest.BlockHeaderFixture() // create latest block that is past the expiry window @@ -251,8 +249,8 @@ func (suite *Suite) TestSendTransactionToRandomCollectionNode() { // create storage metrics := metrics.NewNoopCollector() - transactions := storage.NewTransactions(metrics, db) - collections := storage.NewCollections(db, transactions) + transactions := bstorage.NewTransactions(metrics, db) + collections := bstorage.NewCollections(db, transactions) // create collection node cluster count := 2 @@ -349,7 +347,7 @@ func (suite *Suite) TestSendTransactionToRandomCollectionNode() { } func (suite *Suite) TestGetBlockByIDAndHeight() { - suite.RunTest(func(handler *access.Handler, db *badger.DB, blocks *storage.Blocks, _ *storage.Headers, _ *storage.ExecutionResults) { + suite.RunTest(func(handler *access.Handler, db *badger.DB, all *storage.All) { // test block1 get by ID block1 := unittest.BlockFixture() @@ -357,8 +355,8 @@ func (suite *Suite) TestGetBlockByIDAndHeight() { block2 := unittest.BlockFixture() block2.Header.Height = 2 - require.NoError(suite.T(), blocks.Store(&block1)) - require.NoError(suite.T(), blocks.Store(&block2)) + require.NoError(suite.T(), all.Blocks.Store(&block1)) + require.NoError(suite.T(), all.Blocks.Store(&block2)) // the follower logic should update height index on the block storage when a block is finalized err := db.Update(operation.IndexBlockHeight(block2.Header.Height, block2.ID())) @@ -473,7 +471,7 @@ func (suite *Suite) TestGetBlockByIDAndHeight() { } func (suite *Suite) TestGetExecutionResultByBlockID() { - suite.RunTest(func(handler *access.Handler, db *badger.DB, blocks *storage.Blocks, _ *storage.Headers, executionResults *storage.ExecutionResults) { + suite.RunTest(func(handler *access.Handler, db *badger.DB, all *storage.All) { // test block1 get by ID nonexistingID := unittest.IdentifierFixture() @@ -483,8 +481,8 @@ func (suite *Suite) TestGetExecutionResultByBlockID() { unittest.WithExecutionResultBlockID(blockID), unittest.WithServiceEvents(2)) - require.NoError(suite.T(), executionResults.Store(er)) - require.NoError(suite.T(), executionResults.Index(blockID, er.ID())) + require.NoError(suite.T(), all.Results.Store(er)) + require.NoError(suite.T(), all.Results.Index(blockID, er.ID())) assertResp := func(resp *accessproto.ExecutionResultForBlockIDResponse, err error, executionResult *flow.ExecutionResult) { require.NoError(suite.T(), err) @@ -555,9 +553,9 @@ func (suite *Suite) TestGetExecutionResultByBlockID() { // is reported as sealed func (suite *Suite) TestGetSealedTransaction() { unittest.RunWithBadgerDB(suite.T(), func(db *badger.DB) { - headers, _, _, _, _, blocks, _, _, _, _, _ := util.StorageLayer(suite.T(), db) - results := storage.NewExecutionResults(suite.metrics, db) - receipts := storage.NewExecutionReceipts(suite.metrics, db, results, storage.DefaultCacheSize) + all := util.StorageLayer(suite.T(), db) + results := bstorage.NewExecutionResults(suite.metrics, db) + receipts := bstorage.NewExecutionReceipts(suite.metrics, db, results, bstorage.DefaultCacheSize) enIdentities := unittest.IdentityListFixture(2, unittest.WithRole(flow.RoleExecution)) enNodeIDs := flow.IdentifierList(enIdentities.NodeIDs()) @@ -594,8 +592,8 @@ func (suite *Suite) TestGetSealedTransaction() { // initialize storage metrics := metrics.NewNoopCollector() - transactions := storage.NewTransactions(metrics, db) - collections := storage.NewCollections(db, transactions) + transactions := bstorage.NewTransactions(metrics, db) + collections := bstorage.NewCollections(db, transactions) collectionsToMarkFinalized, err := stdmap.NewTimes(100) require.NoError(suite.T(), err) collectionsToMarkExecuted, err := stdmap.NewTimes(100) @@ -606,8 +604,8 @@ func (suite *Suite) TestGetSealedTransaction() { backend := backend.New(suite.state, suite.collClient, nil, - blocks, - headers, + all.Blocks, + all.Headers, collections, transactions, receipts, @@ -625,19 +623,19 @@ func (suite *Suite) TestGetSealedTransaction() { handler := access.NewHandler(backend, suite.chainID.Chain()) - rpcEngBuilder, err := rpc.NewBuilder(suite.log, suite.state, rpc.Config{}, nil, nil, blocks, headers, collections, transactions, receipts, + rpcEngBuilder, err := rpc.NewBuilder(suite.log, suite.state, rpc.Config{}, nil, nil, all.Blocks, all.Headers, collections, transactions, receipts, results, suite.chainID, metrics, metrics, 0, 0, false, false, nil, nil) require.NoError(suite.T(), err) rpcEng, err := rpcEngBuilder.WithLegacy().Build() require.NoError(suite.T(), err) // create the ingest engine - ingestEng, err := ingestion.New(suite.log, suite.net, suite.state, suite.me, suite.request, blocks, headers, collections, + ingestEng, err := ingestion.New(suite.log, suite.net, suite.state, suite.me, suite.request, all.Blocks, all.Headers, collections, transactions, results, receipts, metrics, collectionsToMarkFinalized, collectionsToMarkExecuted, blocksToMarkExecuted, rpcEng) require.NoError(suite.T(), err) // 1. Assume that follower engine updated the block storage and the protocol state. The block is reported as sealed - err = blocks.Store(&block) + err = all.Blocks.Store(&block) require.NoError(suite.T(), err) suite.snapshot.On("Head").Return(block.Header, nil).Twice() @@ -683,11 +681,11 @@ func (suite *Suite) TestGetSealedTransaction() { // the correct block id func (suite *Suite) TestExecuteScript() { unittest.RunWithBadgerDB(suite.T(), func(db *badger.DB) { - headers, _, _, _, _, blocks, _, _, _, _, _ := util.StorageLayer(suite.T(), db) - transactions := storage.NewTransactions(suite.metrics, db) - collections := storage.NewCollections(db, transactions) - results := storage.NewExecutionResults(suite.metrics, db) - receipts := storage.NewExecutionReceipts(suite.metrics, db, results, storage.DefaultCacheSize) + all := util.StorageLayer(suite.T(), db) + transactions := bstorage.NewTransactions(suite.metrics, db) + collections := bstorage.NewCollections(db, transactions) + results := bstorage.NewExecutionResults(suite.metrics, db) + receipts := bstorage.NewExecutionReceipts(suite.metrics, db, results, bstorage.DefaultCacheSize) identities := unittest.IdentityListFixture(2, unittest.WithRole(flow.RoleExecution)) suite.snapshot.On("Identities", mock.Anything).Return(identities, nil) @@ -699,8 +697,8 @@ func (suite *Suite) TestExecuteScript() { suite.backend = backend.New(suite.state, suite.collClient, nil, - blocks, - headers, + all.Blocks, + all.Headers, collections, transactions, receipts, @@ -731,14 +729,14 @@ func (suite *Suite) TestExecuteScript() { suite.net.On("Register", channels.ReceiveReceipts, mock.Anything).Return(conduit, nil). Once() // create the ingest engine - ingestEng, err := ingestion.New(suite.log, suite.net, suite.state, suite.me, suite.request, blocks, headers, collections, + ingestEng, err := ingestion.New(suite.log, suite.net, suite.state, suite.me, suite.request, all.Blocks, all.Headers, collections, transactions, results, receipts, metrics, collectionsToMarkFinalized, collectionsToMarkExecuted, blocksToMarkExecuted, nil) require.NoError(suite.T(), err) // create a block and a seal pointing to that block lastBlock := unittest.BlockFixture() lastBlock.Header.Height = 2 - err = blocks.Store(&lastBlock) + err = all.Blocks.Store(&lastBlock) require.NoError(suite.T(), err) err = db.Update(operation.IndexBlockHeight(lastBlock.Header.Height, lastBlock.ID())) require.NoError(suite.T(), err) @@ -755,7 +753,7 @@ func (suite *Suite) TestExecuteScript() { // create another block as a predecessor of the block created earlier prevBlock := unittest.BlockFixture() prevBlock.Header.Height = lastBlock.Header.Height - 1 - err = blocks.Store(&prevBlock) + err = all.Blocks.Store(&prevBlock) require.NoError(suite.T(), err) err = db.Update(operation.IndexBlockHeight(prevBlock.Header.Height, prevBlock.ID())) require.NoError(suite.T(), err) diff --git a/module/builder/collection/builder_test.go b/module/builder/collection/builder_test.go index 84988ce762d..fb2484444ae 100644 --- a/module/builder/collection/builder_test.go +++ b/module/builder/collection/builder_test.go @@ -26,7 +26,8 @@ import ( "github.com/onflow/flow-go/state/protocol/events" "github.com/onflow/flow-go/state/protocol/inmem" "github.com/onflow/flow-go/state/protocol/util" - storage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/storage" + bstorage "github.com/onflow/flow-go/storage/badger" "github.com/onflow/flow-go/storage/badger/operation" "github.com/onflow/flow-go/storage/badger/procedure" sutil "github.com/onflow/flow-go/storage/util" @@ -43,9 +44,9 @@ type BuilderSuite struct { genesis *model.Block chainID flow.ChainID - headers *storage.Headers - payloads *storage.ClusterPayloads - blocks *storage.Blocks + headers storage.Headers + payloads storage.ClusterPayloads + blocks storage.Blocks state cluster.MutableState @@ -73,11 +74,11 @@ func (suite *BuilderSuite) SetupTest() { metrics := metrics.NewNoopCollector() tracer := trace.NewNoopTracer() - headers, _, seals, index, conPayloads, blocks, qcs, setups, commits, statuses, results := sutil.StorageLayer(suite.T(), suite.db) + all := sutil.StorageLayer(suite.T(), suite.db) consumer := events.NewNoop() - suite.headers = headers - suite.blocks = blocks - suite.payloads = storage.NewClusterPayloads(metrics, suite.db) + suite.headers = all.Headers + suite.blocks = all.Blocks + suite.payloads = bstorage.NewClusterPayloads(metrics, suite.db) clusterQC := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(suite.genesis.ID())) clusterStateRoot, err := clusterkv.NewStateRoot(suite.genesis, clusterQC) @@ -98,10 +99,10 @@ func (suite *BuilderSuite) SetupTest() { rootSnapshot, err := inmem.SnapshotFromBootstrapState(root, result, seal, unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(root.ID()))) require.NoError(suite.T(), err) - state, err := pbadger.Bootstrap(metrics, suite.db, headers, seals, results, blocks, qcs, setups, commits, statuses, rootSnapshot) + state, err := pbadger.Bootstrap(metrics, suite.db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) require.NoError(suite.T(), err) - suite.protoState, err = pbadger.NewFollowerState(state, index, conPayloads, tracer, consumer, util.MockBlockTimer()) + suite.protoState, err = pbadger.NewFollowerState(state, all.Index, all.Payloads, tracer, consumer, util.MockBlockTimer()) require.NoError(suite.T(), err) // add some transactions to transaction pool @@ -979,10 +980,10 @@ func benchmarkBuildOn(b *testing.B, size int) { metrics := metrics.NewNoopCollector() tracer := trace.NewNoopTracer() - headers, _, _, _, _, blocks, _, _, _, _, _ := sutil.StorageLayer(suite.T(), suite.db) - suite.headers = headers - suite.blocks = blocks - suite.payloads = storage.NewClusterPayloads(metrics, suite.db) + all := sutil.StorageLayer(suite.T(), suite.db) + suite.headers = all.Headers + suite.blocks = all.Blocks + suite.payloads = bstorage.NewClusterPayloads(metrics, suite.db) qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(suite.genesis.ID())) stateRoot, err := clusterkv.NewStateRoot(suite.genesis, qc) diff --git a/state/cluster/badger/mutator_test.go b/state/cluster/badger/mutator_test.go index 3b0eb86ec29..b039fb75d7e 100644 --- a/state/cluster/badger/mutator_test.go +++ b/state/cluster/badger/mutator_test.go @@ -62,14 +62,14 @@ func (suite *MutatorSuite) SetupTest() { metrics := metrics.NewNoopCollector() tracer := trace.NewNoopTracer() - headers, _, seals, index, conPayloads, blocks, qcs, setups, commits, statuses, results := util.StorageLayer(suite.T(), suite.db) + all := util.StorageLayer(suite.T(), suite.db) colPayloads := storage.NewClusterPayloads(metrics, suite.db) clusterStateRoot, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture()) suite.NoError(err) clusterState, err := Bootstrap(suite.db, clusterStateRoot) suite.Assert().Nil(err) - suite.state, err = NewMutableState(clusterState, tracer, headers, colPayloads) + suite.state, err = NewMutableState(clusterState, tracer, all.Headers, colPayloads) suite.Assert().Nil(err) consumer := events.NewNoop() @@ -86,10 +86,10 @@ func (suite *MutatorSuite) SetupTest() { suite.protoGenesis = genesis.Header - state, err := pbadger.Bootstrap(metrics, suite.db, headers, seals, results, blocks, qcs, setups, commits, statuses, rootSnapshot) + state, err := pbadger.Bootstrap(metrics, suite.db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) require.NoError(suite.T(), err) - suite.protoState, err = pbadger.NewFollowerState(state, index, conPayloads, tracer, consumer, protocolutil.MockBlockTimer()) + suite.protoState, err = pbadger.NewFollowerState(state, all.Index, all.Payloads, tracer, consumer, protocolutil.MockBlockTimer()) require.NoError(suite.T(), err) } diff --git a/state/cluster/badger/snapshot_test.go b/state/cluster/badger/snapshot_test.go index 6c299b58839..b17a24e8d6e 100644 --- a/state/cluster/badger/snapshot_test.go +++ b/state/cluster/badger/snapshot_test.go @@ -55,20 +55,20 @@ func (suite *SnapshotSuite) SetupTest() { metrics := metrics.NewNoopCollector() tracer := trace.NewNoopTracer() - headers, _, seals, _, _, blocks, qcs, setups, commits, statuses, results := util.StorageLayer(suite.T(), suite.db) + all := util.StorageLayer(suite.T(), suite.db) colPayloads := storage.NewClusterPayloads(metrics, suite.db) clusterStateRoot, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture()) suite.Assert().Nil(err) clusterState, err := Bootstrap(suite.db, clusterStateRoot) suite.Assert().Nil(err) - suite.state, err = NewMutableState(clusterState, tracer, headers, colPayloads) + suite.state, err = NewMutableState(clusterState, tracer, all.Headers, colPayloads) suite.Assert().Nil(err) participants := unittest.IdentityListFixture(5, unittest.WithAllRoles()) root := unittest.RootSnapshotFixture(participants) - suite.protoState, err = pbadger.Bootstrap(metrics, suite.db, headers, seals, results, blocks, qcs, setups, commits, statuses, root) + suite.protoState, err = pbadger.Bootstrap(metrics, suite.db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, root) require.NoError(suite.T(), err) suite.Require().Nil(err) diff --git a/state/protocol/badger/mutator_test.go b/state/protocol/badger/mutator_test.go index 19a5d59fc1b..e3660492216 100644 --- a/state/protocol/badger/mutator_test.go +++ b/state/protocol/badger/mutator_test.go @@ -90,7 +90,7 @@ func TestExtendValid(t *testing.T) { unittest.RunWithBadgerDB(t, func(db *badger.DB) { metrics := metrics.NewNoopCollector() tracer := trace.NewNoopTracer() - headers, _, seals, index, payloads, blocks, qcs, setups, commits, statuses, results := storeutil.StorageLayer(t, db) + all := storeutil.StorageLayer(t, db) distributor := events.NewDistributor() consumer := mockprotocol.NewConsumer(t) @@ -101,10 +101,11 @@ func TestExtendValid(t *testing.T) { rootSnapshot, err := inmem.SnapshotFromBootstrapState(block, result, seal, qc) require.NoError(t, err) - state, err := protocol.Bootstrap(metrics, db, headers, seals, results, blocks, qcs, setups, commits, statuses, rootSnapshot) + state, err := protocol.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) require.NoError(t, err) - fullState, err := protocol.NewFullConsensusState(state, index, payloads, tracer, consumer, util.MockBlockTimer(), util.MockReceiptValidator(), util.MockSealValidator(seals)) + fullState, err := protocol.NewFullConsensusState(state, all.Index, all.Payloads, tracer, consumer, util.MockBlockTimer(), + util.MockReceiptValidator(), util.MockSealValidator(all.Seals)) require.NoError(t, err) // insert block1 on top of the root block @@ -627,12 +628,13 @@ func TestExtendEpochTransitionValid(t *testing.T) { metrics.On("CurrentDKGPhase3FinalView", dkgPhase3FinalView).Once() tracer := trace.NewNoopTracer() - headers, _, seals, index, payloads, blocks, qcs, setups, commits, statuses, results := storeutil.StorageLayer(t, db) - protoState, err := protocol.Bootstrap(metrics, db, headers, seals, results, blocks, qcs, setups, commits, statuses, rootSnapshot) + all := storeutil.StorageLayer(t, db) + protoState, err := protocol.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) require.NoError(t, err) receiptValidator := util.MockReceiptValidator() - sealValidator := util.MockSealValidator(seals) - state, err := protocol.NewFullConsensusState(protoState, index, payloads, tracer, consumer, util.MockBlockTimer(), receiptValidator, sealValidator) + sealValidator := util.MockSealValidator(all.Seals) + state, err := protocol.NewFullConsensusState(protoState, all.Index, all.Payloads, tracer, consumer, + util.MockBlockTimer(), receiptValidator, sealValidator) require.NoError(t, err) head, err := rootSnapshot.Head() @@ -1762,7 +1764,7 @@ func TestExtendInvalidSealsInBlock(t *testing.T) { unittest.RunWithBadgerDB(t, func(db *badger.DB) { metrics := metrics.NewNoopCollector() tracer := trace.NewNoopTracer() - headers, _, seals, index, payloads, blocks, qcs, setups, commits, statuses, results := storeutil.StorageLayer(t, db) + all := storeutil.StorageLayer(t, db) // create a event consumer to test epoch transition events distributor := events.NewDistributor() @@ -1772,7 +1774,7 @@ func TestExtendInvalidSealsInBlock(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) - state, err := protocol.Bootstrap(metrics, db, headers, seals, results, blocks, qcs, setups, commits, statuses, rootSnapshot) + state, err := protocol.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) require.NoError(t, err) head, err := rootSnapshot.Head() @@ -1798,18 +1800,19 @@ func TestExtendInvalidSealsInBlock(t *testing.T) { if candidate.ID() == block3.ID() { return nil } - seal, _ := seals.HighestInFork(candidate.Header.ParentID) + seal, _ := all.Seals.HighestInFork(candidate.Header.ParentID) return seal }, func(candidate *flow.Block) error { if candidate.ID() == block3.ID() { return engine.NewInvalidInputError("") } - _, err := seals.HighestInFork(candidate.Header.ParentID) + _, err := all.Seals.HighestInFork(candidate.Header.ParentID) return err }). Times(3) - fullState, err := protocol.NewFullConsensusState(state, index, payloads, tracer, consumer, util.MockBlockTimer(), util.MockReceiptValidator(), sealValidator) + fullState, err := protocol.NewFullConsensusState(state, all.Index, all.Payloads, tracer, consumer, + util.MockBlockTimer(), util.MockReceiptValidator(), sealValidator) require.NoError(t, err) err = fullState.Extend(context.Background(), block1) @@ -2229,7 +2232,7 @@ func TestHeaderInvalidTimestamp(t *testing.T) { unittest.RunWithBadgerDB(t, func(db *badger.DB) { metrics := metrics.NewNoopCollector() tracer := trace.NewNoopTracer() - headers, _, seals, index, payloads, blocks, qcs, setups, commits, statuses, results := storeutil.StorageLayer(t, db) + all := storeutil.StorageLayer(t, db) // create a event consumer to test epoch transition events distributor := events.NewDistributor() @@ -2241,13 +2244,14 @@ func TestHeaderInvalidTimestamp(t *testing.T) { rootSnapshot, err := inmem.SnapshotFromBootstrapState(block, result, seal, qc) require.NoError(t, err) - state, err := protocol.Bootstrap(metrics, db, headers, seals, results, blocks, qcs, setups, commits, statuses, rootSnapshot) + state, err := protocol.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) require.NoError(t, err) blockTimer := &mockprotocol.BlockTimer{} blockTimer.On("Validate", mock.Anything, mock.Anything).Return(realprotocol.NewInvalidBlockTimestamp("")) - fullState, err := protocol.NewFullConsensusState(state, index, payloads, tracer, consumer, blockTimer, util.MockReceiptValidator(), util.MockSealValidator(seals)) + fullState, err := protocol.NewFullConsensusState(state, all.Index, all.Payloads, tracer, consumer, blockTimer, + util.MockReceiptValidator(), util.MockSealValidator(all.Seals)) require.NoError(t, err) extend := unittest.BlockWithParentFixture(block.Header) diff --git a/state/protocol/badger/state_test.go b/state/protocol/badger/state_test.go index b67d146c195..987dae1ca48 100644 --- a/state/protocol/badger/state_test.go +++ b/state/protocol/badger/state_test.go @@ -8,7 +8,6 @@ import ( "time" "github.com/dgraph-io/badger/v2" - "github.com/stretchr/testify/assert" testmock "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -510,8 +509,8 @@ func bootstrap(t *testing.T, rootSnapshot protocol.Snapshot, f func(*bprotocol.S defer os.RemoveAll(dir) db := unittest.BadgerDB(t, dir) defer db.Close() - headers, _, seals, _, _, blocks, qcs, setups, commits, statuses, results := storutil.StorageLayer(t, db) - state, err := bprotocol.Bootstrap(metrics, db, headers, seals, results, blocks, qcs, setups, commits, statuses, rootSnapshot) + all := storutil.StorageLayer(t, db) + state, err := bprotocol.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) f(state, err) } diff --git a/state/protocol/util/testing.go b/state/protocol/util/testing.go index 9309faec8bd..a1a0be77744 100644 --- a/state/protocol/util/testing.go +++ b/state/protocol/util/testing.go @@ -65,8 +65,8 @@ func MockSealValidator(sealsDB storage.Seals) module.SealValidator { func RunWithBootstrapState(t testing.TB, rootSnapshot protocol.Snapshot, f func(*badger.DB, *pbadger.State)) { unittest.RunWithBadgerDB(t, func(db *badger.DB) { metrics := metrics.NewNoopCollector() - headers, _, seals, _, _, blocks, qcs, setups, commits, statuses, results := util.StorageLayer(t, db) - state, err := pbadger.Bootstrap(metrics, db, headers, seals, results, blocks, qcs, setups, commits, statuses, rootSnapshot) + all := util.StorageLayer(t, db) + state, err := pbadger.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) require.NoError(t, err) f(db, state) }) @@ -77,13 +77,13 @@ func RunWithFullProtocolState(t testing.TB, rootSnapshot protocol.Snapshot, f fu metrics := metrics.NewNoopCollector() tracer := trace.NewNoopTracer() consumer := events.NewNoop() - headers, _, seals, index, payloads, blocks, qcs, setups, commits, statuses, results := util.StorageLayer(t, db) - state, err := pbadger.Bootstrap(metrics, db, headers, seals, results, blocks, qcs, setups, commits, statuses, rootSnapshot) + all := util.StorageLayer(t, db) + state, err := pbadger.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) require.NoError(t, err) receiptValidator := MockReceiptValidator() - sealValidator := MockSealValidator(seals) + sealValidator := MockSealValidator(all.Seals) mockTimer := MockBlockTimer() - fullState, err := pbadger.NewFullConsensusState(state, index, payloads, tracer, consumer, mockTimer, receiptValidator, sealValidator) + fullState, err := pbadger.NewFullConsensusState(state, all.Index, all.Payloads, tracer, consumer, mockTimer, receiptValidator, sealValidator) require.NoError(t, err) f(db, fullState) }) @@ -93,13 +93,13 @@ func RunWithFullProtocolStateAndMetrics(t testing.TB, rootSnapshot protocol.Snap unittest.RunWithBadgerDB(t, func(db *badger.DB) { tracer := trace.NewNoopTracer() consumer := events.NewNoop() - headers, _, seals, index, payloads, blocks, qcs, setups, commits, statuses, results := util.StorageLayer(t, db) - state, err := pbadger.Bootstrap(metrics, db, headers, seals, results, blocks, qcs, setups, commits, statuses, rootSnapshot) + all := util.StorageLayer(t, db) + state, err := pbadger.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) require.NoError(t, err) receiptValidator := MockReceiptValidator() - sealValidator := MockSealValidator(seals) + sealValidator := MockSealValidator(all.Seals) mockTimer := MockBlockTimer() - fullState, err := pbadger.NewFullConsensusState(state, index, payloads, tracer, consumer, mockTimer, receiptValidator, sealValidator) + fullState, err := pbadger.NewFullConsensusState(state, all.Index, all.Payloads, tracer, consumer, mockTimer, receiptValidator, sealValidator) require.NoError(t, err) f(db, fullState) }) @@ -110,12 +110,12 @@ func RunWithFullProtocolStateAndValidator(t testing.TB, rootSnapshot protocol.Sn metrics := metrics.NewNoopCollector() tracer := trace.NewNoopTracer() consumer := events.NewNoop() - headers, _, seals, index, payloads, blocks, qcs, setups, commits, statuses, results := util.StorageLayer(t, db) - state, err := pbadger.Bootstrap(metrics, db, headers, seals, results, blocks, qcs, setups, commits, statuses, rootSnapshot) + all := util.StorageLayer(t, db) + state, err := pbadger.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) require.NoError(t, err) - sealValidator := MockSealValidator(seals) + sealValidator := MockSealValidator(all.Seals) mockTimer := MockBlockTimer() - fullState, err := pbadger.NewFullConsensusState(state, index, payloads, tracer, consumer, mockTimer, validator, sealValidator) + fullState, err := pbadger.NewFullConsensusState(state, all.Index, all.Payloads, tracer, consumer, mockTimer, validator, sealValidator) require.NoError(t, err) f(db, fullState) }) @@ -126,11 +126,11 @@ func RunWithFollowerProtocolState(t testing.TB, rootSnapshot protocol.Snapshot, metrics := metrics.NewNoopCollector() tracer := trace.NewNoopTracer() consumer := events.NewNoop() - headers, _, seals, index, payloads, blocks, qcs, setups, commits, statuses, results := util.StorageLayer(t, db) - state, err := pbadger.Bootstrap(metrics, db, headers, seals, results, blocks, qcs, setups, commits, statuses, rootSnapshot) + all := util.StorageLayer(t, db) + state, err := pbadger.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) require.NoError(t, err) mockTimer := MockBlockTimer() - followerState, err := pbadger.NewFollowerState(state, index, payloads, tracer, consumer, mockTimer) + followerState, err := pbadger.NewFollowerState(state, all.Index, all.Payloads, tracer, consumer, mockTimer) require.NoError(t, err) f(db, followerState) }) @@ -140,13 +140,13 @@ func RunWithFullProtocolStateAndConsumer(t testing.TB, rootSnapshot protocol.Sna unittest.RunWithBadgerDB(t, func(db *badger.DB) { metrics := metrics.NewNoopCollector() tracer := trace.NewNoopTracer() - headers, _, seals, index, payloads, blocks, qcs, setups, commits, statuses, results := util.StorageLayer(t, db) - state, err := pbadger.Bootstrap(metrics, db, headers, seals, results, blocks, qcs, setups, commits, statuses, rootSnapshot) + all := util.StorageLayer(t, db) + state, err := pbadger.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) require.NoError(t, err) receiptValidator := MockReceiptValidator() - sealValidator := MockSealValidator(seals) + sealValidator := MockSealValidator(all.Seals) mockTimer := MockBlockTimer() - fullState, err := pbadger.NewFullConsensusState(state, index, payloads, tracer, consumer, mockTimer, receiptValidator, sealValidator) + fullState, err := pbadger.NewFullConsensusState(state, all.Index, all.Payloads, tracer, consumer, mockTimer, receiptValidator, sealValidator) require.NoError(t, err) f(db, fullState) }) @@ -155,13 +155,13 @@ func RunWithFullProtocolStateAndConsumer(t testing.TB, rootSnapshot protocol.Sna func RunWithFullProtocolStateAndMetricsAndConsumer(t testing.TB, rootSnapshot protocol.Snapshot, metrics module.ComplianceMetrics, consumer protocol.Consumer, f func(*badger.DB, *pbadger.ParticipantState)) { unittest.RunWithBadgerDB(t, func(db *badger.DB) { tracer := trace.NewNoopTracer() - headers, _, seals, index, payloads, blocks, qcs, setups, commits, statuses, results := util.StorageLayer(t, db) - state, err := pbadger.Bootstrap(metrics, db, headers, seals, results, blocks, qcs, setups, commits, statuses, rootSnapshot) + all := util.StorageLayer(t, db) + state, err := pbadger.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) require.NoError(t, err) receiptValidator := MockReceiptValidator() - sealValidator := MockSealValidator(seals) + sealValidator := MockSealValidator(all.Seals) mockTimer := MockBlockTimer() - fullState, err := pbadger.NewFullConsensusState(state, index, payloads, tracer, consumer, mockTimer, receiptValidator, sealValidator) + fullState, err := pbadger.NewFullConsensusState(state, all.Index, all.Payloads, tracer, consumer, mockTimer, receiptValidator, sealValidator) require.NoError(t, err) f(db, fullState) }) @@ -172,12 +172,12 @@ func RunWithFollowerProtocolStateAndHeaders(t testing.TB, rootSnapshot protocol. metrics := metrics.NewNoopCollector() tracer := trace.NewNoopTracer() consumer := events.NewNoop() - headers, _, seals, index, payloads, blocks, qcs, setups, commits, statuses, results := util.StorageLayer(t, db) - state, err := pbadger.Bootstrap(metrics, db, headers, seals, results, blocks, qcs, setups, commits, statuses, rootSnapshot) + all := util.StorageLayer(t, db) + state, err := pbadger.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) require.NoError(t, err) mockTimer := MockBlockTimer() - followerState, err := pbadger.NewFollowerState(state, index, payloads, tracer, consumer, mockTimer) + followerState, err := pbadger.NewFollowerState(state, all.Index, all.Payloads, tracer, consumer, mockTimer) require.NoError(t, err) - f(db, followerState, headers, index) + f(db, followerState, all.Headers, all.Index) }) } diff --git a/storage/util/testing.go b/storage/util/testing.go index e8294c6f61c..89e7e523364 100644 --- a/storage/util/testing.go +++ b/storage/util/testing.go @@ -9,44 +9,14 @@ import ( "github.com/stretchr/testify/require" "github.com/onflow/flow-go/module/metrics" - storage "github.com/onflow/flow-go/storage/badger" - "github.com/onflow/flow-go/utils/unittest" + "github.com/onflow/flow-go/storage" + bstorage "github.com/onflow/flow-go/storage/badger" ) -func StorageLayer(t testing.TB, db *badger.DB) ( - *storage.Headers, - *storage.Guarantees, - *storage.Seals, - *storage.Index, - *storage.Payloads, - *storage.Blocks, - *storage.QuorumCertificates, - *storage.EpochSetups, - *storage.EpochCommits, - *storage.EpochStatuses, - *storage.ExecutionResults, -) { +func StorageLayer(_ testing.TB, db *badger.DB) *storage.All { metrics := metrics.NewNoopCollector() - headers := storage.NewHeaders(metrics, db) - guarantees := storage.NewGuarantees(metrics, db, storage.DefaultCacheSize) - seals := storage.NewSeals(metrics, db) - results := storage.NewExecutionResults(metrics, db) - receipts := storage.NewExecutionReceipts(metrics, db, results, storage.DefaultCacheSize) - index := storage.NewIndex(metrics, db) - payloads := storage.NewPayloads(db, index, guarantees, seals, receipts, results) - blocks := storage.NewBlocks(db, headers, payloads) - qcs := storage.NewQuorumCertificates(metrics, db, storage.DefaultCacheSize) - setups := storage.NewEpochSetups(metrics, db) - commits := storage.NewEpochCommits(metrics, db) - statuses := storage.NewEpochStatuses(metrics, db) - return headers, guarantees, seals, index, payloads, blocks, qcs, setups, commits, statuses, results -} - -func RunWithStorageLayer(t testing.TB, f func(*badger.DB, *storage.Headers, *storage.Guarantees, *storage.Seals, *storage.Index, *storage.Payloads, *storage.Blocks, *storage.EpochSetups, *storage.EpochCommits, *storage.EpochStatuses)) { - unittest.RunWithBadgerDB(t, func(db *badger.DB) { - headers, guarantees, seals, index, payloads, blocks, _, setups, commits, statuses, _ := StorageLayer(t, db) - f(db, headers, guarantees, seals, index, payloads, blocks, setups, commits, statuses) - }) + all := bstorage.InitAll(metrics, db) + return all } func CreateFiles(t *testing.T, dir string, names ...string) {