From 76767e9cfa3a745dc415b33c51ecb7fbf79423b3 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 8 Aug 2023 12:05:15 -0300 Subject: [PATCH 001/128] feat(trie): Add v2 runtime functions --- lib/runtime/wazero/imports.go | 185 +++++++++++++++++++++++++++++++-- lib/runtime/wazero/instance.go | 6 ++ 2 files changed, 185 insertions(+), 6 deletions(-) diff --git a/lib/runtime/wazero/imports.go b/lib/runtime/wazero/imports.go index c4e344544e..25db775147 100644 --- a/lib/runtime/wazero/imports.go +++ b/lib/runtime/wazero/imports.go @@ -859,6 +859,56 @@ func ext_trie_blake2_256_root_version_1(ctx context.Context, m api.Module, dataS return ptr } +func ext_trie_blake2_256_root_version_2(ctx context.Context, m api.Module, dataSpan uint64, version uint32) uint32 { + rtCtx := ctx.Value(runtimeContextKey).(*runtime.Context) + if rtCtx == nil { + panic("nil runtime context") + } + + data := read(m, dataSpan) + + t := trie.NewEmptyTrie() + + type kv struct { + Key, Value []byte + } + + // this function is expecting an array of (key, value) tuples + var kvs []kv + if err := scale.Unmarshal(data, &kvs); err != nil { + logger.Errorf("failed scale decoding data: %s", err) + return 0 + } + + for _, kv := range kvs { + //TODO: use version parameter here + err := t.Put(kv.Key, kv.Value) + if err != nil { + logger.Errorf("failed putting key 0x%x and value 0x%x into trie: %s", + kv.Key, kv.Value, err) + return 0 + } + } + + // allocate memory for value and copy value to memory + ptr, err := rtCtx.Allocator.Allocate(32) + if err != nil { + logger.Errorf("failed allocating: %s", err) + return 0 + } + + //TODO: use version parameter here + hash, err := t.Hash() + if err != nil { + logger.Errorf("failed computing trie Merkle root hash: %s", err) + return 0 + } + + logger.Debugf("root hash is %s", hash) + m.Memory().Write(ptr, hash[:]) + return ptr +} + func ext_trie_blake2_256_ordered_root_version_1(ctx context.Context, m api.Module, dataSpan uint64) uint32 { rtCtx := ctx.Value(runtimeContextKey).(*runtime.Context) if rtCtx == nil { @@ -913,8 +963,57 @@ func ext_trie_blake2_256_ordered_root_version_1(ctx context.Context, m api.Modul func ext_trie_blake2_256_ordered_root_version_2( ctx context.Context, m api.Module, dataSpan uint64, version uint32) uint32 { - // TODO: update to use state trie version 1 (#2418) - return ext_trie_blake2_256_ordered_root_version_1(ctx, m, dataSpan) + rtCtx := ctx.Value(runtimeContextKey).(*runtime.Context) + if rtCtx == nil { + panic("nil runtime context") + } + + data := read(m, dataSpan) + + t := trie.NewEmptyTrie() + var values [][]byte + err := scale.Unmarshal(data, &values) + if err != nil { + logger.Errorf("failed scale decoding data: %s", err) + return 0 + } + + for i, value := range values { + key, err := scale.Marshal(big.NewInt(int64(i))) + if err != nil { + logger.Errorf("failed scale encoding value index %d: %s", i, err) + return 0 + } + logger.Tracef( + "put key=0x%x and value=0x%x", + key, value) + + //TODO: use version parameter here + err = t.Put(key, value) + if err != nil { + logger.Errorf("failed putting key 0x%x and value 0x%x into trie: %s", + key, value, err) + return 0 + } + } + + // allocate memory for value and copy value to memory + ptr, err := rtCtx.Allocator.Allocate(32) + if err != nil { + logger.Errorf("failed allocating: %s", err) + return 0 + } + + //TODO: use version parameter here + hash, err := t.Hash() + if err != nil { + logger.Errorf("failed computing trie Merkle root hash: %s", err) + return 0 + } + + logger.Debugf("root hash is %s", hash) + m.Memory().Write(ptr, hash[:]) + return ptr } func ext_trie_blake2_256_verify_proof_version_1( @@ -949,6 +1048,39 @@ func ext_trie_blake2_256_verify_proof_version_1( return 1 } +func ext_trie_blake2_256_verify_proof_version_2( + ctx context.Context, m api.Module, rootSpan uint32, proofSpan, keySpan, valueSpan uint64, version uint32) uint32 { + rtCtx := ctx.Value(runtimeContextKey).(*runtime.Context) + if rtCtx == nil { + panic("nil runtime context") + } + + toDecProofs := read(m, proofSpan) + var encodedProofNodes [][]byte + err := scale.Unmarshal(toDecProofs, &encodedProofNodes) + if err != nil { + logger.Errorf("failed scale decoding proof data: %s", err) + return uint32(0) + } + + key := read(m, keySpan) + value := read(m, valueSpan) + + trieRoot, ok := m.Memory().Read(rootSpan, 32) + if !ok { + panic("read overflow") + } + + //TODO: use trie version parameter here + err = proof.Verify(encodedProofNodes, trieRoot, key, value) + if err != nil { + logger.Errorf("failed proof verification: %s", err) + return 0 + } + + return 1 +} + func ext_misc_print_hex_version_1(ctx context.Context, m api.Module, dataSpan uint64) { data := read(m, dataSpan) logger.Debugf("data: 0x%x", data) @@ -1273,8 +1405,30 @@ func ext_default_child_storage_root_version_1( //export ext_default_child_storage_root_version_2 func ext_default_child_storage_root_version_2(ctx context.Context, m api.Module, childStorageKey uint64, stateVersion uint32) (ptrSize uint64) { - // TODO: Implement this after we have storage trie version 1 implemented #2418 - return ext_default_child_storage_root_version_1(ctx, m, childStorageKey) + rtCtx := ctx.Value(runtimeContextKey).(*runtime.Context) + if rtCtx == nil { + panic("nil runtime context") + } + storage := rtCtx.Storage + child, err := storage.GetChild(read(m, childStorageKey)) + if err != nil { + logger.Errorf("failed to retrieve child: %s", err) + return 0 + } + + //TODO: version this call + childRoot, err := child.Hash() + if err != nil { + logger.Errorf("failed to encode child root: %s", err) + return 0 + } + childRootSlice := childRoot[:] + + ret, err := write(m, rtCtx.Allocator, scale.MustMarshal(&childRootSlice)) + if err != nil { + panic(err) + } + return ret } func ext_default_child_storage_storage_kill_version_1(ctx context.Context, m api.Module, childStorageKeySpan uint64) { @@ -2267,8 +2421,27 @@ func ext_storage_root_version_1(ctx context.Context, m api.Module) uint64 { } func ext_storage_root_version_2(ctx context.Context, m api.Module, version uint32) uint64 { //skipcq: RVV-B0012 - // TODO: update to use state trie version 1 (#2418) - return ext_storage_root_version_1(ctx, m) + rtCtx := ctx.Value(runtimeContextKey).(*runtime.Context) + if rtCtx == nil { + panic("nil runtime context") + } + storage := rtCtx.Storage + + //TODO: user version parameter + root, err := storage.Root() + if err != nil { + logger.Errorf("failed to get storage root: %s", err) + panic(err) + } + + logger.Debugf("root hash is: %s", root) + + rootSpan, err := write(m, rtCtx.Allocator, root[:]) + if err != nil { + logger.Errorf("failed to allocate: %s", err) + panic(err) + } + return rootSpan } func ext_storage_set_version_1(ctx context.Context, m api.Module, keySpan, valueSpan uint64) { diff --git a/lib/runtime/wazero/instance.go b/lib/runtime/wazero/instance.go index 9d329341ee..de0ba4bf41 100644 --- a/lib/runtime/wazero/instance.go +++ b/lib/runtime/wazero/instance.go @@ -207,6 +207,9 @@ func NewInstance(code []byte, cfg Config) (instance *Instance, err error) { WithFunc(ext_trie_blake2_256_root_version_1). Export("ext_trie_blake2_256_root_version_1"). NewFunctionBuilder(). + WithFunc(ext_trie_blake2_256_root_version_2). + Export("ext_trie_blake2_256_root_version_2"). + NewFunctionBuilder(). WithFunc(ext_trie_blake2_256_ordered_root_version_1). Export("ext_trie_blake2_256_ordered_root_version_1"). NewFunctionBuilder(). @@ -216,6 +219,9 @@ func NewInstance(code []byte, cfg Config) (instance *Instance, err error) { WithFunc(ext_trie_blake2_256_verify_proof_version_1). Export("ext_trie_blake2_256_verify_proof_version_1"). NewFunctionBuilder(). + WithFunc(ext_trie_blake2_256_verify_proof_version_2). + Export("ext_trie_blake2_256_verify_proof_version_2"). + NewFunctionBuilder(). WithFunc(ext_misc_print_hex_version_1). Export("ext_misc_print_hex_version_1"). NewFunctionBuilder(). From f4ad7387bb738f6321db18d2458372bf599fd0d1 Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 10 Aug 2023 12:03:16 -0300 Subject: [PATCH 002/128] chore(trie): version trie functions --- dot/core/helpers_test.go | 4 +- dot/rpc/modules/childstate_test.go | 10 ++--- dot/state/base_test.go | 2 +- dot/state/storage_notify_test.go | 7 +-- dot/state/storage_test.go | 10 ++--- dot/state/test_helpers.go | 2 +- lib/runtime/interfaces.go | 2 +- lib/runtime/storage/trie.go | 4 +- lib/runtime/storage/trie_test.go | 22 +++++----- lib/runtime/wasmer/exports_test.go | 16 +++---- lib/runtime/wasmer/helpers.go | 5 ++- lib/runtime/wasmer/imports.go | 8 ++-- lib/runtime/wasmer/imports_test.go | 68 ++++++++++++++--------------- lib/runtime/wazero/imports.go | 12 ++--- lib/runtime/wazero/imports_test.go | 68 ++++++++++++++--------------- lib/runtime/wazero/instance_test.go | 16 +++---- lib/trie/child_storage.go | 4 +- lib/trie/database_test.go | 4 +- lib/trie/helpers_test.go | 2 +- lib/trie/mem_test.go | 4 +- lib/trie/proof/generate_test.go | 3 +- lib/trie/proof/proof_test.go | 3 +- lib/trie/trie.go | 2 +- lib/trie/trie_endtoend_test.go | 44 +++++++++---------- lib/trie/trie_test.go | 4 +- lib/trie/version.go | 26 +++++++++-- lib/trie/version_test.go | 2 +- 27 files changed, 189 insertions(+), 165 deletions(-) diff --git a/dot/core/helpers_test.go b/dot/core/helpers_test.go index f3802c850a..737ab367fc 100644 --- a/dot/core/helpers_test.go +++ b/dot/core/helpers_test.go @@ -108,9 +108,9 @@ func createTestService(t *testing.T, genesisFilePath string, cfgRuntime, err := wazero_runtime.NewRuntimeFromGenesis(rtCfg) require.NoError(t, err) - cfgRuntime.Context.Storage.Put(aliceBalanceKey, encodedAccountInfo) + cfgRuntime.Context.Storage.Put(aliceBalanceKey, encodedAccountInfo, trie.V0) // this key is System.UpgradedToDualRefCount -> set to true since all accounts have been upgraded to v0.9 format - cfgRuntime.Context.Storage.Put(common.UpgradedToDualRefKey, []byte{1}) + cfgRuntime.Context.Storage.Put(common.UpgradedToDualRefKey, []byte{1}, trie.V0) cfgBlockState.StoreRuntime(cfgBlockState.BestBlockHash(), cfgRuntime) diff --git a/dot/rpc/modules/childstate_test.go b/dot/rpc/modules/childstate_test.go index c84df630ce..533e6cb57e 100644 --- a/dot/rpc/modules/childstate_test.go +++ b/dot/rpc/modules/childstate_test.go @@ -24,13 +24,13 @@ func createTestTrieState(t *testing.T) (*trie.Trie, common.Hash) { _, genesisTrie, _ := newWestendLocalGenesisWithTrieAndHeader(t) tr := rtstorage.NewTrieState(&genesisTrie) - tr.Put([]byte(":first_key"), []byte(":value1")) - tr.Put([]byte(":second_key"), []byte(":second_value")) + tr.Put([]byte(":first_key"), []byte(":value1"), trie.V0) + tr.Put([]byte(":second_key"), []byte(":second_value"), trie.V0) childTr := trie.NewEmptyTrie() - childTr.Put([]byte(":child_first"), []byte(":child_first_value")) - childTr.Put([]byte(":child_second"), []byte(":child_second_value")) - childTr.Put([]byte(":another_child"), []byte("value")) + childTr.Put([]byte(":child_first"), []byte(":child_first_value"), trie.V0) + childTr.Put([]byte(":child_second"), []byte(":child_second_value"), trie.V0) + childTr.Put([]byte(":another_child"), []byte("value"), trie.V0) err := tr.SetChild([]byte(":child_storage_key"), childTr) require.NoError(t, err) diff --git a/dot/state/base_test.go b/dot/state/base_test.go index a6bedcde74..1286373e33 100644 --- a/dot/state/base_test.go +++ b/dot/state/base_test.go @@ -23,7 +23,7 @@ func TestTrie_StoreAndLoadFromDB(t *testing.T) { for keyString, value := range kv { key := []byte(keyString) - tt.Put(key, value) + tt.Put(key, value, trie.V0) } err := tt.WriteDirty(db) diff --git a/dot/state/storage_notify_test.go b/dot/state/storage_notify_test.go index 4751e6137f..1f2f0da551 100644 --- a/dot/state/storage_notify_test.go +++ b/dot/state/storage_notify_test.go @@ -13,6 +13,7 @@ import ( "github.com/ChainSafe/chaindb" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/trie" "github.com/dgraph-io/badger/v4/pb" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" @@ -49,7 +50,7 @@ func TestStorageState_RegisterStorageObserver(t *testing.T) { ss.RegisterStorageObserver(mockobs) defer ss.UnregisterStorageObserver(mockobs) - ts.Put([]byte("mackcom"), []byte("wuz here")) + ts.Put([]byte("mackcom"), []byte("wuz here"), trie.V0) err = ss.StoreTrie(ts, nil) require.NoError(t, err) @@ -86,7 +87,7 @@ func TestStorageState_RegisterStorageObserver_Multi(t *testing.T) { key1 := []byte("key1") value1 := []byte("value1") - ts.Put(key1, value1) + ts.Put(key1, value1, trie.V0) err = ss.StoreTrie(ts, nil) require.NoError(t, err) @@ -125,7 +126,7 @@ func TestStorageState_RegisterStorageObserver_Multi_Filter(t *testing.T) { ss.RegisterStorageObserver(mockobs) } - ts.Put(key1, value1) + ts.Put(key1, value1, trie.V0) err = ss.StoreTrie(ts, nil) require.NoError(t, err) diff --git a/dot/state/storage_test.go b/dot/state/storage_test.go index 1af0d3c768..fd0e597f0a 100644 --- a/dot/state/storage_test.go +++ b/dot/state/storage_test.go @@ -56,7 +56,7 @@ func TestStorage_GetStorageByBlockHash(t *testing.T) { key := []byte("testkey") value := []byte("testvalue") - ts.Put(key, value) + ts.Put(key, value, trie.V0) root, err := ts.Root() require.NoError(t, err) @@ -88,7 +88,7 @@ func TestStorage_TrieState(t *testing.T) { storage := newTestStorageState(t) ts, err := storage.TrieState(&trie.EmptyHash) require.NoError(t, err) - ts.Put([]byte("noot"), []byte("washere")) + ts.Put([]byte("noot"), []byte("washere"), trie.V0) root, err := ts.Root() require.NoError(t, err) @@ -120,7 +120,7 @@ func TestStorage_LoadFromDB(t *testing.T) { } for _, kv := range trieKV { - ts.Put(kv.key, kv.value) + ts.Put(kv.key, kv.value, trie.V0) } root, err := ts.Root() @@ -157,7 +157,7 @@ func TestStorage_StoreTrie_NotSyncing(t *testing.T) { key := []byte("testkey") value := []byte("testvalue") - ts.Put(key, value) + ts.Put(key, value, trie.V0) err = storage.StoreTrie(ts, nil) require.NoError(t, err) @@ -188,7 +188,7 @@ func TestGetStorageChildAndGetStorageFromChild(t *testing.T) { } testChildTrie := trie.NewTrie(trieRoot, dbGetter) - testChildTrie.Put([]byte("keyInsidechild"), []byte("voila")) + testChildTrie.Put([]byte("keyInsidechild"), []byte("voila"), trie.V0) err = genTrie.SetChild([]byte("keyToChild"), testChildTrie) require.NoError(t, err) diff --git a/dot/state/test_helpers.go b/dot/state/test_helpers.go index 82c2d60858..56255b3198 100644 --- a/dot/state/test_helpers.go +++ b/dot/state/test_helpers.go @@ -244,7 +244,7 @@ func generateBlockWithRandomTrie(t *testing.T, serv *Service, rand := time.Now().UnixNano() key := []byte("testKey" + fmt.Sprint(rand)) value := []byte("testValue" + fmt.Sprint(rand)) - err = trieState.Put(key, value) + err = trieState.Put(key, value, trie.V0) require.NoError(t, err) trieStateRoot, err := trieState.Root() diff --git a/lib/runtime/interfaces.go b/lib/runtime/interfaces.go index 2f813a21f6..9a46e5a650 100644 --- a/lib/runtime/interfaces.go +++ b/lib/runtime/interfaces.go @@ -11,7 +11,7 @@ import ( // Storage runtime interface. type Storage interface { - Put(key []byte, value []byte) (err error) + Put(key []byte, value []byte, version trie.Version) (err error) Get(key []byte) []byte Root() (common.Hash, error) SetChild(keyToChild []byte, child *trie.Trie) error diff --git a/lib/runtime/storage/trie.go b/lib/runtime/storage/trie.go index b88e8fb56a..0c6f1a9619 100644 --- a/lib/runtime/storage/trie.go +++ b/lib/runtime/storage/trie.go @@ -70,10 +70,10 @@ func (s *TrieState) RollbackStorageTransaction() { } // Put puts a key-value pair in the trie -func (s *TrieState) Put(key, value []byte) (err error) { +func (s *TrieState) Put(key, value []byte, version trie.Version) (err error) { s.lock.Lock() defer s.lock.Unlock() - return s.t.Put(key, value) + return s.t.Put(key, value, version) } // Get gets a value from the trie diff --git a/lib/runtime/storage/trie_test.go b/lib/runtime/storage/trie_test.go index 01e0e2f732..73e528db89 100644 --- a/lib/runtime/storage/trie_test.go +++ b/lib/runtime/storage/trie_test.go @@ -27,7 +27,7 @@ var testCases = []string{ func TestTrieState_SetGet(t *testing.T) { testFunc := func(ts *TrieState) { for _, tc := range testCases { - ts.Put([]byte(tc), []byte(tc)) + ts.Put([]byte(tc), []byte(tc), trie.V0) } for _, tc := range testCases { @@ -43,7 +43,7 @@ func TestTrieState_SetGet(t *testing.T) { func TestTrieState_Delete(t *testing.T) { testFunc := func(ts *TrieState) { for _, tc := range testCases { - ts.Put([]byte(tc), []byte(tc)) + ts.Put([]byte(tc), []byte(tc), trie.V0) } ts.Delete([]byte(testCases[0])) @@ -58,7 +58,7 @@ func TestTrieState_Delete(t *testing.T) { func TestTrieState_Root(t *testing.T) { testFunc := func(ts *TrieState) { for _, tc := range testCases { - ts.Put([]byte(tc), []byte(tc)) + ts.Put([]byte(tc), []byte(tc), trie.V0) } expected := ts.MustRoot() @@ -79,7 +79,7 @@ func TestTrieState_ClearPrefix(t *testing.T) { } for i, key := range keys { - ts.Put([]byte(key), []byte{byte(i)}) + ts.Put([]byte(key), []byte{byte(i)}, trie.V0) } ts.ClearPrefix([]byte("noo")) @@ -105,7 +105,7 @@ func TestTrieState_ClearPrefixInChild(t *testing.T) { } for i, key := range keys { - child.Put([]byte(key), []byte{byte(i)}) + child.Put([]byte(key), []byte{byte(i)}, trie.V0) } keyToChild := []byte("keytochild") @@ -131,7 +131,7 @@ func TestTrieState_NextKey(t *testing.T) { ts := &TrieState{t: trie.NewEmptyTrie()} for _, tc := range testCases { - ts.Put([]byte(tc), []byte(tc)) + ts.Put([]byte(tc), []byte(tc), trie.V0) } sort.Slice(testCases, func(i, j int) bool { @@ -152,12 +152,12 @@ func TestTrieState_CommitStorageTransaction(t *testing.T) { ts := &TrieState{t: trie.NewEmptyTrie()} for _, tc := range testCases { - ts.Put([]byte(tc), []byte(tc)) + ts.Put([]byte(tc), []byte(tc), trie.V0) } ts.BeginStorageTransaction() testValue := []byte("noot") - ts.Put([]byte(testCases[0]), testValue) + ts.Put([]byte(testCases[0]), testValue, trie.V0) ts.CommitStorageTransaction() val := ts.Get([]byte(testCases[0])) @@ -168,12 +168,12 @@ func TestTrieState_RollbackStorageTransaction(t *testing.T) { ts := &TrieState{t: trie.NewEmptyTrie()} for _, tc := range testCases { - ts.Put([]byte(tc), []byte(tc)) + ts.Put([]byte(tc), []byte(tc), trie.V0) } ts.BeginStorageTransaction() testValue := []byte("noot") - ts.Put([]byte(testCases[0]), testValue) + ts.Put([]byte(testCases[0]), testValue, trie.V0) ts.RollbackStorageTransaction() val := ts.Get([]byte(testCases[0])) @@ -191,7 +191,7 @@ func TestTrieState_DeleteChildLimit(t *testing.T) { } for i, key := range keys { - child.Put([]byte(key), []byte{byte(i)}) + child.Put([]byte(key), []byte{byte(i)}, trie.V0) } keyToChild := []byte("keytochild") diff --git a/lib/runtime/wasmer/exports_test.go b/lib/runtime/wasmer/exports_test.go index 7a9f937a6e..b02e3f1ece 100644 --- a/lib/runtime/wasmer/exports_test.go +++ b/lib/runtime/wasmer/exports_test.go @@ -204,9 +204,9 @@ func TestWestendRuntime_ValidateTransaction(t *testing.T) { encBal, err := scale.Marshal(accInfo) require.NoError(t, err) - rt.ctx.Storage.Put(aliceBalanceKey, encBal) + rt.ctx.Storage.Put(aliceBalanceKey, encBal, trie.V0) // this key is System.UpgradedToDualRefCount -> set to true since all accounts have been upgraded to v0.9 format - rt.ctx.Storage.Put(common.UpgradedToDualRefKey, []byte{1}) + rt.ctx.Storage.Put(common.UpgradedToDualRefKey, []byte{1}, trie.V0) genesisHeader := &types.Header{ Number: 0, @@ -237,7 +237,7 @@ func TestInstance_GrandpaAuthorities_NodeRuntime(t *testing.T) { require.NoError(t, err) key := common.MustHexToBytes(genesis.GrandpaAuthoritiesKeyHex) - tt.Put(key, value) + tt.Put(key, value, trie.V0) rt := NewTestInstanceWithTrie(t, runtime.WESTEND_RUNTIME_v0929, tt) @@ -265,7 +265,7 @@ func TestInstance_GrandpaAuthorities_PolkadotRuntime(t *testing.T) { require.NoError(t, err) key := common.MustHexToBytes(genesis.GrandpaAuthoritiesKeyHex) - tt.Put(key, value) + tt.Put(key, value, trie.V0) rt := NewTestInstanceWithTrie(t, runtime.POLKADOT_RUNTIME_v0929, tt) @@ -312,13 +312,13 @@ func TestInstance_BabeGenerateKeyOwnershipProof(t *testing.T) { randomnessValue, err := common.HexToHash("0x01") require.NoError(t, err) key := common.MustHexToBytes(genesis.BABERandomnessKeyHex) - tt.Put(key, randomnessValue[:]) + tt.Put(key, randomnessValue[:], trie.V0) authorityValue, err := common.HexToBytes("0x08eea1eabcac7d2c8a6459b7322cf997874482bfc3d2ec7a80888a3a7d714103640100000000000000b64994460e59b30364cad3c92e3df6052f9b0ebbb8f88460c194dc5794d6d7170100000000000000") //nolint:lll require.NoError(t, err) key = common.MustHexToBytes(genesis.BABEAuthoritiesKeyHex) - tt.Put(key, authorityValue) + tt.Put(key, authorityValue, trie.V0) rt := NewTestInstanceWithTrie(t, testCase.targetRuntime, tt) @@ -399,13 +399,13 @@ func TestInstance_BabeConfiguration_WestendRuntime_WithAuthorities(t *testing.T) randomnessValue, err := common.HexToHash("0x01") require.NoError(t, err) key := common.MustHexToBytes(genesis.BABERandomnessKeyHex) - tt.Put(key, randomnessValue[:]) + tt.Put(key, randomnessValue[:], trie.V0) authorityValue, err := common.HexToBytes("0x08eea1eabcac7d2c8a6459b7322cf997874482bfc3d2ec7a80888a3a7d714103640100000000000000b64994460e59b30364cad3c92e3df6052f9b0ebbb8f88460c194dc5794d6d7170100000000000000") //nolint:lll require.NoError(t, err) key = common.MustHexToBytes(genesis.BABEAuthoritiesKeyHex) - tt.Put(key, authorityValue) + tt.Put(key, authorityValue, trie.V0) rt := NewTestInstanceWithTrie(t, runtime.WESTEND_RUNTIME_v0929, tt) diff --git a/lib/runtime/wasmer/helpers.go b/lib/runtime/wasmer/helpers.go index 1cfbf372de..9c95f220e2 100644 --- a/lib/runtime/wasmer/helpers.go +++ b/lib/runtime/wasmer/helpers.go @@ -12,6 +12,7 @@ import ( "github.com/ChainSafe/gossamer/lib/common/types" "github.com/ChainSafe/gossamer/lib/runtime" + "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/pkg/scale" "github.com/wasmerio/go-ext-wasm/wasmer" ) @@ -208,7 +209,7 @@ func toWasmMemoryFixedSizeOptional(context wasmer.InstanceContext, data []byte) return toWasmMemory(context, encodedOptionalFixedSize) } -func storageAppend(storage runtime.Storage, key, valueToAppend []byte) (err error) { +func storageAppend(storage runtime.Storage, key, valueToAppend []byte, version trie.Version) (err error) { // this function assumes the item in storage is a SCALE encoded array of items // the valueToAppend is a new item, so it appends the item and increases the length prefix by 1 currentValue := storage.Get(key) @@ -259,7 +260,7 @@ func storageAppend(storage runtime.Storage, key, valueToAppend []byte) (err erro } } - err = storage.Put(key, value) + err = storage.Put(key, value, version) if err != nil { return fmt.Errorf("putting key and value in storage: %w", err) } diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index 02ad2826a2..a0a0568a57 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -820,7 +820,7 @@ func ext_trie_blake2_256_root_version_1(context unsafe.Pointer, dataSpan C.int64 } for _, kv := range kvs { - err := t.Put(kv.Key, kv.Value) + err := t.Put(kv.Key, kv.Value, trie.V0) if err != nil { logger.Errorf("failed putting key 0x%x and value 0x%x into trie: %s", kv.Key, kv.Value, err) @@ -873,7 +873,7 @@ func ext_trie_blake2_256_ordered_root_version_1(context unsafe.Pointer, dataSpan "put key=0x%x and value=0x%x", key, value) - err = t.Put(key, value) + err = t.Put(key, value, trie.V0) if err != nil { logger.Errorf("failed putting key 0x%x and value 0x%x into trie: %s", key, value, err) @@ -1916,7 +1916,7 @@ func ext_storage_append_version_1(context unsafe.Pointer, keySpan, valueSpan C.i cp := make([]byte, len(valueAppend)) copy(cp, valueAppend) - err := storageAppend(storage, key, cp) + err := storageAppend(storage, key, cp, trie.V0) if err != nil { logger.Errorf("failed appending to storage: %s", err) } @@ -2156,7 +2156,7 @@ func ext_storage_set_version_1(context unsafe.Pointer, keySpan, valueSpan C.int6 logger.Debugf( "key 0x%x has value 0x%x", key, value) - err := storage.Put(key, cp) + err := storage.Put(key, cp, trie.V0) panicOnError(err) } diff --git a/lib/runtime/wasmer/imports_test.go b/lib/runtime/wasmer/imports_test.go index 7f060d496c..a936da44af 100644 --- a/lib/runtime/wasmer/imports_test.go +++ b/lib/runtime/wasmer/imports_test.go @@ -207,7 +207,7 @@ func Test_ext_storage_clear_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") - inst.ctx.Storage.Put(testkey, []byte{1}) + inst.ctx.Storage.Put(testkey, []byte{1}, trie.V0) enc, err := scale.Marshal(testkey) require.NoError(t, err) @@ -393,10 +393,10 @@ func Test_ext_storage_clear_prefix_version_1_hostAPI(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("static") - inst.ctx.Storage.Put(testkey, []byte("Inverse")) + inst.ctx.Storage.Put(testkey, []byte("Inverse"), trie.V0) testkey2 := []byte("even-keeled") - inst.ctx.Storage.Put(testkey2, []byte("Future-proofed")) + inst.ctx.Storage.Put(testkey2, []byte("Future-proofed"), trie.V0) enc, err := scale.Marshal(testkey[:3]) require.NoError(t, err) @@ -416,10 +416,10 @@ func Test_ext_storage_clear_prefix_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") - inst.ctx.Storage.Put(testkey, []byte{1}) + inst.ctx.Storage.Put(testkey, []byte{1}, trie.V0) testkey2 := []byte("spaghet") - inst.ctx.Storage.Put(testkey2, []byte{2}) + inst.ctx.Storage.Put(testkey2, []byte{2}, trie.V0) enc, err := scale.Marshal(testkey[:3]) require.NoError(t, err) @@ -439,20 +439,20 @@ func Test_ext_storage_clear_prefix_version_2(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") - inst.ctx.Storage.Put(testkey, []byte{1}) + inst.ctx.Storage.Put(testkey, []byte{1}, trie.V0) testkey2 := []byte("noot1") - inst.ctx.Storage.Put(testkey2, []byte{1}) + inst.ctx.Storage.Put(testkey2, []byte{1}, trie.V0) testkey3 := []byte("noot2") - inst.ctx.Storage.Put(testkey3, []byte{1}) + inst.ctx.Storage.Put(testkey3, []byte{1}, trie.V0) testkey4 := []byte("noot3") - inst.ctx.Storage.Put(testkey4, []byte{1}) + inst.ctx.Storage.Put(testkey4, []byte{1}, trie.V0) testkey5 := []byte("spaghet") testValue5 := []byte{2} - inst.ctx.Storage.Put(testkey5, testValue5) + inst.ctx.Storage.Put(testkey5, testValue5, trie.V0) enc, err := scale.Marshal(testkey[:3]) require.NoError(t, err) @@ -513,7 +513,7 @@ func Test_ext_storage_get_version_1(t *testing.T) { testkey := []byte("noot") testvalue := []byte{1, 2} - inst.ctx.Storage.Put(testkey, testvalue) + inst.ctx.Storage.Put(testkey, testvalue, trie.V0) enc, err := scale.Marshal(testkey) require.NoError(t, err) @@ -559,7 +559,7 @@ func Test_ext_storage_exists_version_1(t *testing.T) { instance := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) if testCase.value != nil { - instance.ctx.Storage.Put(testCase.key, testCase.value) + instance.ctx.Storage.Put(testCase.key, testCase.value, trie.V0) } encodedKey, err := scale.Marshal(testCase.key) @@ -582,10 +582,10 @@ func Test_ext_storage_next_key_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") - inst.ctx.Storage.Put(testkey, []byte{1}) + inst.ctx.Storage.Put(testkey, []byte{1}, trie.V0) nextkey := []byte("oot") - inst.ctx.Storage.Put(nextkey, []byte{1}) + inst.ctx.Storage.Put(nextkey, []byte{1}, trie.V0) enc, err := scale.Marshal(testkey) require.NoError(t, err) @@ -606,7 +606,7 @@ func Test_ext_storage_read_version_1(t *testing.T) { testkey := []byte("noot") testvalue := []byte("washere") - inst.ctx.Storage.Put(testkey, testvalue) + inst.ctx.Storage.Put(testkey, testvalue, trie.V0) testoffset := uint32(2) testBufferSize := uint32(100) @@ -635,7 +635,7 @@ func Test_ext_storage_read_version_1_again(t *testing.T) { testkey := []byte("noot") testvalue := []byte("_was_here_") - inst.ctx.Storage.Put(testkey, testvalue) + inst.ctx.Storage.Put(testkey, testvalue, trie.V0) testoffset := uint32(8) testBufferSize := uint32(5) @@ -665,7 +665,7 @@ func Test_ext_storage_read_version_1_OffsetLargerThanValue(t *testing.T) { testkey := []byte("noot") testvalue := []byte("washere") - inst.ctx.Storage.Put(testkey, testvalue) + inst.ctx.Storage.Put(testkey, testvalue, trie.V0) testoffset := uint32(len(testvalue)) testBufferSize := uint32(8) @@ -1574,8 +1574,8 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_all(t *testing. inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`)) - tr.Put([]byte(`key1`), []byte(`value1`)) + tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) + tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) err := inst.ctx.Storage.SetChild(testChildKey, tr) require.NoError(t, err) @@ -1608,8 +1608,8 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_1(t *testing.T) inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`)) - tr.Put([]byte(`key1`), []byte(`value1`)) + tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) + tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) err := inst.ctx.Storage.SetChild(testChildKey, tr) require.NoError(t, err) @@ -1642,8 +1642,8 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_none(t *testing inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`)) - tr.Put([]byte(`key1`), []byte(`value1`)) + tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) + tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) err := inst.ctx.Storage.SetChild(testChildKey, tr) require.NoError(t, err) @@ -1673,9 +1673,9 @@ func Test_ext_default_child_storage_storage_kill_version_3(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`)) - tr.Put([]byte(`key1`), []byte(`value1`)) - tr.Put([]byte(`key3`), []byte(`value3`)) + tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) + tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) + tr.Put([]byte(`key3`), []byte(`value3`), trie.V0) err := inst.ctx.Storage.SetChild(testChildKey, tr) require.NoError(t, err) @@ -1856,8 +1856,8 @@ func Test_ext_trie_blake2_256_root_version_1(t *testing.T) { require.NoError(t, err) tt := trie.NewEmptyTrie() - tt.Put([]byte("noot"), []byte("was")) - tt.Put([]byte("here"), []byte("??")) + tt.Put([]byte("noot"), []byte("was"), trie.V0) + tt.Put([]byte("here"), []byte("??"), trie.V0) expected := tt.MustHash() require.Equal(t, expected[:], hash) @@ -1875,17 +1875,17 @@ func Test_ext_trie_blake2_256_verify_proof_version_1(t *testing.T) { require.NoError(t, err) otherTrie := trie.NewEmptyTrie() - otherTrie.Put([]byte("simple"), []byte("cat")) + otherTrie.Put([]byte("simple"), []byte("cat"), trie.V0) otherHash, err := otherTrie.Hash() require.NoError(t, err) tr := trie.NewEmptyTrie() - tr.Put([]byte("do"), []byte("verb")) - tr.Put([]byte("domain"), []byte("website")) - tr.Put([]byte("other"), []byte("random")) - tr.Put([]byte("otherwise"), []byte("randomstuff")) - tr.Put([]byte("cat"), []byte("another animal")) + tr.Put([]byte("do"), []byte("verb"), trie.V0) + tr.Put([]byte("domain"), []byte("website"), trie.V0) + tr.Put([]byte("other"), []byte("random"), trie.V0) + tr.Put([]byte("otherwise"), []byte("randomstuff"), trie.V0) + tr.Put([]byte("cat"), []byte("another animal"), trie.V0) err = tr.WriteDirty(memdb) require.NoError(t, err) diff --git a/lib/runtime/wazero/imports.go b/lib/runtime/wazero/imports.go index c4e344544e..0de05b4cdf 100644 --- a/lib/runtime/wazero/imports.go +++ b/lib/runtime/wazero/imports.go @@ -833,7 +833,7 @@ func ext_trie_blake2_256_root_version_1(ctx context.Context, m api.Module, dataS } for _, kv := range kvs { - err := t.Put(kv.Key, kv.Value) + err := t.Put(kv.Key, kv.Value, trie.V0) if err != nil { logger.Errorf("failed putting key 0x%x and value 0x%x into trie: %s", kv.Key, kv.Value, err) @@ -885,7 +885,7 @@ func ext_trie_blake2_256_ordered_root_version_1(ctx context.Context, m api.Modul "put key=0x%x and value=0x%x", key, value) - err = t.Put(key, value) + err = t.Put(key, value, trie.V0) if err != nil { logger.Errorf("failed putting key 0x%x and value 0x%x into trie: %s", key, value, err) @@ -1917,7 +1917,7 @@ func ext_offchain_http_request_add_header_version_1( return ptr } -func storageAppend(storage runtime.Storage, key, valueToAppend []byte) (err error) { +func storageAppend(storage runtime.Storage, key, valueToAppend []byte, version trie.Version) (err error) { // this function assumes the item in storage is a SCALE encoded array of items // the valueToAppend is a new item, so it appends the item and increases the length prefix by 1 currentValue := storage.Get(key) @@ -1968,7 +1968,7 @@ func storageAppend(storage runtime.Storage, key, valueToAppend []byte) (err erro } } - err = storage.Put(key, value) + err = storage.Put(key, value, version) if err != nil { return fmt.Errorf("putting key and value in storage: %w", err) } @@ -1992,7 +1992,7 @@ func ext_storage_append_version_1(ctx context.Context, m api.Module, keySpan, va cp := make([]byte, len(valueAppend)) copy(cp, valueAppend) - err := storageAppend(storage, key, cp) + err := storageAppend(storage, key, cp, trie.V0) if err != nil { logger.Errorf("failed appending to storage: %s", err) } @@ -2287,7 +2287,7 @@ func ext_storage_set_version_1(ctx context.Context, m api.Module, keySpan, value logger.Debugf( "key 0x%x has value 0x%x", key, value) - err := storage.Put(key, cp) + err := storage.Put(key, cp, trie.V0) if err != nil { panic(err) } diff --git a/lib/runtime/wazero/imports_test.go b/lib/runtime/wazero/imports_test.go index c9d95f0a89..815970dae5 100644 --- a/lib/runtime/wazero/imports_test.go +++ b/lib/runtime/wazero/imports_test.go @@ -562,8 +562,8 @@ func Test_ext_trie_blake2_256_root_version_1(t *testing.T) { require.NoError(t, err) tt := trie.NewEmptyTrie() - tt.Put([]byte("noot"), []byte("was")) - tt.Put([]byte("here"), []byte("??")) + tt.Put([]byte("noot"), []byte("was"), trie.V0) + tt.Put([]byte("here"), []byte("??"), trie.V0) expected := tt.MustHash() require.Equal(t, expected[:], hash) @@ -597,17 +597,17 @@ func Test_ext_trie_blake2_256_verify_proof_version_1(t *testing.T) { require.NoError(t, err) otherTrie := trie.NewEmptyTrie() - otherTrie.Put([]byte("simple"), []byte("cat")) + otherTrie.Put([]byte("simple"), []byte("cat"), trie.V0) otherHash, err := otherTrie.Hash() require.NoError(t, err) tr := trie.NewEmptyTrie() - tr.Put([]byte("do"), []byte("verb")) - tr.Put([]byte("domain"), []byte("website")) - tr.Put([]byte("other"), []byte("random")) - tr.Put([]byte("otherwise"), []byte("randomstuff")) - tr.Put([]byte("cat"), []byte("another animal")) + tr.Put([]byte("do"), []byte("verb"), trie.V0) + tr.Put([]byte("domain"), []byte("website"), trie.V0) + tr.Put([]byte("other"), []byte("random"), trie.V0) + tr.Put([]byte("otherwise"), []byte("randomstuff"), trie.V0) + tr.Put([]byte("cat"), []byte("another animal"), trie.V0) err = tr.WriteDirty(memdb) require.NoError(t, err) @@ -1023,8 +1023,8 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_all(t *testing. inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`)) - tr.Put([]byte(`key1`), []byte(`value1`)) + tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) + tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) err := inst.Context.Storage.SetChild(testChildKey, tr) require.NoError(t, err) @@ -1056,8 +1056,8 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_1(t *testing.T) inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`)) - tr.Put([]byte(`key1`), []byte(`value1`)) + tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) + tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) err := inst.Context.Storage.SetChild(testChildKey, tr) require.NoError(t, err) @@ -1089,8 +1089,8 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_none(t *testing inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`)) - tr.Put([]byte(`key1`), []byte(`value1`)) + tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) + tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) err := inst.Context.Storage.SetChild(testChildKey, tr) require.NoError(t, err) @@ -1119,9 +1119,9 @@ func Test_ext_default_child_storage_storage_kill_version_3(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`)) - tr.Put([]byte(`key1`), []byte(`value1`)) - tr.Put([]byte(`key3`), []byte(`value3`)) + tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) + tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) + tr.Put([]byte(`key3`), []byte(`value3`), trie.V0) err := inst.Context.Storage.SetChild(testChildKey, tr) require.NoError(t, err) @@ -1524,7 +1524,7 @@ func Test_ext_storage_clear_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") - inst.Context.Storage.Put(testkey, []byte{1}) + inst.Context.Storage.Put(testkey, []byte{1}, trie.V0) enc, err := scale.Marshal(testkey) require.NoError(t, err) @@ -1540,10 +1540,10 @@ func Test_ext_storage_clear_prefix_version_1_hostAPI(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("static") - inst.Context.Storage.Put(testkey, []byte("Inverse")) + inst.Context.Storage.Put(testkey, []byte("Inverse"), trie.V0) testkey2 := []byte("even-keeled") - inst.Context.Storage.Put(testkey2, []byte("Future-proofed")) + inst.Context.Storage.Put(testkey2, []byte("Future-proofed"), trie.V0) enc, err := scale.Marshal(testkey[:3]) require.NoError(t, err) @@ -1562,10 +1562,10 @@ func Test_ext_storage_clear_prefix_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") - inst.Context.Storage.Put(testkey, []byte{1}) + inst.Context.Storage.Put(testkey, []byte{1}, trie.V0) testkey2 := []byte("spaghet") - inst.Context.Storage.Put(testkey2, []byte{2}) + inst.Context.Storage.Put(testkey2, []byte{2}, trie.V0) enc, err := scale.Marshal(testkey[:3]) require.NoError(t, err) @@ -1584,20 +1584,20 @@ func Test_ext_storage_clear_prefix_version_2(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") - inst.Context.Storage.Put(testkey, []byte{1}) + inst.Context.Storage.Put(testkey, []byte{1}, trie.V0) testkey2 := []byte("noot1") - inst.Context.Storage.Put(testkey2, []byte{1}) + inst.Context.Storage.Put(testkey2, []byte{1}, trie.V0) testkey3 := []byte("noot2") - inst.Context.Storage.Put(testkey3, []byte{1}) + inst.Context.Storage.Put(testkey3, []byte{1}, trie.V0) testkey4 := []byte("noot3") - inst.Context.Storage.Put(testkey4, []byte{1}) + inst.Context.Storage.Put(testkey4, []byte{1}, trie.V0) testkey5 := []byte("spaghet") testValue5 := []byte{2} - inst.Context.Storage.Put(testkey5, testValue5) + inst.Context.Storage.Put(testkey5, testValue5, trie.V0) enc, err := scale.Marshal(testkey[:3]) require.NoError(t, err) @@ -1657,7 +1657,7 @@ func Test_ext_storage_get_version_1(t *testing.T) { testkey := []byte("noot") testvalue := []byte{1, 2} - inst.Context.Storage.Put(testkey, testvalue) + inst.Context.Storage.Put(testkey, testvalue, trie.V0) enc, err := scale.Marshal(testkey) require.NoError(t, err) @@ -1700,7 +1700,7 @@ func Test_ext_storage_exists_version_1(t *testing.T) { instance := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) if testCase.value != nil { - instance.Context.Storage.Put(testCase.key, testCase.value) + instance.Context.Storage.Put(testCase.key, testCase.value, trie.V0) } encodedKey, err := scale.Marshal(testCase.key) @@ -1722,10 +1722,10 @@ func Test_ext_storage_next_key_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") - inst.Context.Storage.Put(testkey, []byte{1}) + inst.Context.Storage.Put(testkey, []byte{1}, trie.V0) nextkey := []byte("oot") - inst.Context.Storage.Put(nextkey, []byte{1}) + inst.Context.Storage.Put(nextkey, []byte{1}, trie.V0) enc, err := scale.Marshal(testkey) require.NoError(t, err) @@ -1745,7 +1745,7 @@ func Test_ext_storage_read_version_1(t *testing.T) { testkey := []byte("noot") testvalue := []byte("washere") - inst.Context.Storage.Put(testkey, testvalue) + inst.Context.Storage.Put(testkey, testvalue, trie.V0) testoffset := uint32(2) testBufferSize := uint32(100) @@ -1773,7 +1773,7 @@ func Test_ext_storage_read_version_1_again(t *testing.T) { testkey := []byte("noot") testvalue := []byte("_was_here_") - inst.Context.Storage.Put(testkey, testvalue) + inst.Context.Storage.Put(testkey, testvalue, trie.V0) testoffset := uint32(8) testBufferSize := uint32(5) @@ -1802,7 +1802,7 @@ func Test_ext_storage_read_version_1_OffsetLargerThanValue(t *testing.T) { testkey := []byte("noot") testvalue := []byte("washere") - inst.Context.Storage.Put(testkey, testvalue) + inst.Context.Storage.Put(testkey, testvalue, trie.V0) testoffset := uint32(len(testvalue)) testBufferSize := uint32(8) diff --git a/lib/runtime/wazero/instance_test.go b/lib/runtime/wazero/instance_test.go index 204266e1c8..b54ce2151c 100644 --- a/lib/runtime/wazero/instance_test.go +++ b/lib/runtime/wazero/instance_test.go @@ -222,9 +222,9 @@ func TestWestendRuntime_ValidateTransaction(t *testing.T) { encBal, err := scale.Marshal(accInfo) require.NoError(t, err) - rt.Context.Storage.Put(aliceBalanceKey, encBal) + rt.Context.Storage.Put(aliceBalanceKey, encBal, trie.V0) // this key is System.UpgradedToDualRefCount -> set to true since all accounts have been upgraded to v0.9 format - rt.Context.Storage.Put(common.UpgradedToDualRefKey, []byte{1}) + rt.Context.Storage.Put(common.UpgradedToDualRefKey, []byte{1}, trie.V0) genesisHeader := &types.Header{ Number: 0, @@ -255,7 +255,7 @@ func TestInstance_GrandpaAuthorities_NodeRuntime(t *testing.T) { require.NoError(t, err) key := common.MustHexToBytes(genesis.GrandpaAuthoritiesKeyHex) - tt.Put(key, value) + tt.Put(key, value, trie.V0) rt := NewTestInstanceWithTrie(t, runtime.WESTEND_RUNTIME_v0929, tt) @@ -283,7 +283,7 @@ func TestInstance_GrandpaAuthorities_PolkadotRuntime(t *testing.T) { require.NoError(t, err) key := common.MustHexToBytes(genesis.GrandpaAuthoritiesKeyHex) - tt.Put(key, value) + tt.Put(key, value, trie.V0) rt := NewTestInstanceWithTrie(t, runtime.POLKADOT_RUNTIME_v0929, tt) @@ -326,13 +326,13 @@ func TestInstance_BabeGenerateKeyOwnershipProof(t *testing.T) { randomnessValue, err := common.HexToHash("0x01") require.NoError(t, err) key := common.MustHexToBytes(genesis.BABERandomnessKeyHex) - tt.Put(key, randomnessValue[:]) + tt.Put(key, randomnessValue[:], trie.V0) authorityValue, err := common.HexToBytes("0x08eea1eabcac7d2c8a6459b7322cf997874482bfc3d2ec7a80888a3a7d714103640100000000000000b64994460e59b30364cad3c92e3df6052f9b0ebbb8f88460c194dc5794d6d7170100000000000000") //nolint:lll require.NoError(t, err) key = common.MustHexToBytes(genesis.BABEAuthoritiesKeyHex) - tt.Put(key, authorityValue) + tt.Put(key, authorityValue, trie.V0) rt := NewTestInstanceWithTrie(t, testCase.targetRuntime, tt) @@ -409,13 +409,13 @@ func TestInstance_BabeConfiguration_WestendRuntime_WithAuthorities(t *testing.T) randomnessValue, err := common.HexToHash("0x01") require.NoError(t, err) key := common.MustHexToBytes(genesis.BABERandomnessKeyHex) - tt.Put(key, randomnessValue[:]) + tt.Put(key, randomnessValue[:], trie.V0) authorityValue, err := common.HexToBytes("0x08eea1eabcac7d2c8a6459b7322cf997874482bfc3d2ec7a80888a3a7d714103640100000000000000b64994460e59b30364cad3c92e3df6052f9b0ebbb8f88460c194dc5794d6d7170100000000000000") //nolint:lll require.NoError(t, err) key = common.MustHexToBytes(genesis.BABEAuthoritiesKeyHex) - tt.Put(key, authorityValue) + tt.Put(key, authorityValue, trie.V0) rt := NewTestInstanceWithTrie(t, runtime.WESTEND_RUNTIME_v0929, tt) diff --git a/lib/trie/child_storage.go b/lib/trie/child_storage.go index 78506eaa8c..444c713da0 100644 --- a/lib/trie/child_storage.go +++ b/lib/trie/child_storage.go @@ -28,7 +28,7 @@ func (t *Trie) SetChild(keyToChild []byte, child *Trie) error { copy(key, ChildStorageKeyPrefix) copy(key[len(ChildStorageKeyPrefix):], keyToChild) - err = t.Put(key, childHash.ToBytes()) + err = t.Put(key, childHash.ToBytes(), V0) if err != nil { return fmt.Errorf("putting child trie root hash %s in trie: %w", childHash, err) } @@ -63,7 +63,7 @@ func (t *Trie) PutIntoChild(keyToChild, key, value []byte) error { return err } - err = child.Put(key, value) + err = child.Put(key, value, V0) if err != nil { return fmt.Errorf("putting into child trie located at key 0x%x: %w", keyToChild, err) } diff --git a/lib/trie/database_test.go b/lib/trie/database_test.go index 2300adb415..0737e3703d 100644 --- a/lib/trie/database_test.go +++ b/lib/trie/database_test.go @@ -54,7 +54,7 @@ func Test_Trie_WriteDirty_Put(t *testing.T) { for keyString, value := range keyValues { key := []byte(keyString) - trie.Put(key, value) + trie.Put(key, value, V0) err := trie.WriteDirty(db) require.NoError(t, err) @@ -75,7 +75,7 @@ func Test_Trie_WriteDirty_Put(t *testing.T) { newValue := make([]byte, len(existingValue)) copy(newValue, existingValue) newValue = append(newValue, 99) - trie.Put(existingKey, newValue) + trie.Put(existingKey, newValue, V0) err = trie.WriteDirty(db) require.NoError(t, err) diff --git a/lib/trie/helpers_test.go b/lib/trie/helpers_test.go index cbae542394..b36385e098 100644 --- a/lib/trie/helpers_test.go +++ b/lib/trie/helpers_test.go @@ -93,7 +93,7 @@ func makeSeededTrie(t *testing.T, size int) ( for keyString, value := range keyValues { key := []byte(keyString) - trie.Put(key, value) + trie.Put(key, value, V0) } return trie, keyValues diff --git a/lib/trie/mem_test.go b/lib/trie/mem_test.go index 56096571bd..757da3185f 100644 --- a/lib/trie/mem_test.go +++ b/lib/trie/mem_test.go @@ -93,7 +93,7 @@ func populateTrieAtPrefix(trie *Trie, for keyString, value := range kv { key := append(prefix, []byte(keyString)...) //skipcq: CRT-D0001 - trie.Put(key, value) + trie.Put(key, value, V0) } } @@ -113,6 +113,6 @@ func mutateTrieLeavesAtPrefix(trie *Trie, } } - trie.Put(key, newValue) + trie.Put(key, newValue, V0) } } diff --git a/lib/trie/proof/generate_test.go b/lib/trie/proof/generate_test.go index 49f519b3c0..6517ed8bd0 100644 --- a/lib/trie/proof/generate_test.go +++ b/lib/trie/proof/generate_test.go @@ -828,6 +828,7 @@ func Test_lenCommonPrefix(t *testing.T) { // so the code is kept to this inefficient-looking append, // which is in the end quite performant still. func Benchmark_walkRoot(b *testing.B) { + stateVersion := trie.V0 trie := trie.NewEmptyTrie() // Build a deep trie. @@ -838,7 +839,7 @@ func Benchmark_walkRoot(b *testing.B) { const trieValueSize = 10 value := make([]byte, trieValueSize) - trie.Put(key, value) + trie.Put(key, value, stateVersion) } longestKeyLE := make([]byte, trieDepth) diff --git a/lib/trie/proof/proof_test.go b/lib/trie/proof/proof_test.go index 8f813b4f61..f808c999b4 100644 --- a/lib/trie/proof/proof_test.go +++ b/lib/trie/proof/proof_test.go @@ -16,6 +16,7 @@ import ( func Test_Generate_Verify(t *testing.T) { t.Parallel() + stateVersion := trie.V0 keys := []string{ "cat", @@ -29,7 +30,7 @@ func Test_Generate_Verify(t *testing.T) { for i, key := range keys { value := fmt.Sprintf("%x-%d", key, i) - trie.Put([]byte(key), []byte(value)) + trie.Put([]byte(key), []byte(value), stateVersion) } rootHash, err := trie.Hash() diff --git a/lib/trie/trie.go b/lib/trie/trie.go index ea2d1dd4ae..5f1067d5ba 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -355,7 +355,7 @@ func findNextKeyChild(children []*Node, startIndex byte, // Put inserts a value into the trie at the // key specified in little Endian format. -func (t *Trie) Put(keyLE, value []byte) (err error) { +func (t *Trie) Put(keyLE, value []byte, version Version) (err error) { pendingDeltas := tracking.New() defer func() { const success = true diff --git a/lib/trie/trie_endtoend_test.go b/lib/trie/trie_endtoend_test.go index 3ca189ebe5..a6d87e9360 100644 --- a/lib/trie/trie_endtoend_test.go +++ b/lib/trie/trie_endtoend_test.go @@ -40,7 +40,7 @@ func buildSmallTrie() *Trie { } for _, test := range tests { - trie.Put(test.key, test.value) + trie.Put(test.key, test.value, V0) } return trie @@ -50,7 +50,7 @@ func runTests(t *testing.T, trie *Trie, tests []keyValues) { for _, test := range tests { switch test.op { case put: - trie.Put(test.key, test.value) + trie.Put(test.key, test.value, V0) case get: val := trie.Get(test.key) assert.Equal(t, test.value, val) @@ -104,7 +104,7 @@ func TestPutAndGetOddKeyLengths(t *testing.T) { func Fuzz_Trie_PutAndGet_Single(f *testing.F) { f.Fuzz(func(t *testing.T, key, value []byte) { trie := NewEmptyTrie() - trie.Put(key, value) + trie.Put(key, value, V0) retrievedValue := trie.Get(key) assert.Equal(t, value, retrievedValue) }) @@ -119,7 +119,7 @@ func Test_Trie_PutAndGet_Multiple(t *testing.T) { keyValues := generateKeyValues(t, generator, numberOfKeyValuePairs) for keyString, value := range keyValues { key := []byte(keyString) - trie.Put(key, value) + trie.Put(key, value, V0) // Check value is inserted correctly. retrievedValue := trie.Get(key) @@ -296,7 +296,7 @@ func TestTrieDiff(t *testing.T) { } for _, test := range tests { - trie.Put(test.key, test.value) + trie.Put(test.key, test.value, V0) } newTrie := trie.Snapshot() @@ -312,7 +312,7 @@ func TestTrieDiff(t *testing.T) { } for _, test := range tests { - newTrie.Put(test.key, test.value) + newTrie.Put(test.key, test.value, V0) } deletedNodeHashes := newTrie.deltas.Deleted() @@ -349,7 +349,7 @@ func TestDelete(t *testing.T) { for keyString, value := range kv { key := []byte(keyString) - trie.Put(key, value) + trie.Put(key, value, V0) } dcTrie := trie.DeepCopy() @@ -432,7 +432,7 @@ func TestClearPrefix(t *testing.T) { trie := NewEmptyTrie() for _, test := range tests { - trie.Put(test.key, test.value) + trie.Put(test.key, test.value, V0) } dcTrie := trie.DeepCopy() @@ -517,7 +517,7 @@ func TestClearPrefix_Small(t *testing.T) { "other", } for _, key := range keys { - ssTrie.Put([]byte(key), []byte(key)) + ssTrie.Put([]byte(key), []byte(key), V0) } ssTrie.ClearPrefix([]byte("noo")) @@ -598,8 +598,8 @@ func TestTrie_ClearPrefixVsDelete(t *testing.T) { trieClearPrefix := NewEmptyTrie() for _, test := range testCase { - trieDelete.Put(test.key, test.value) - trieClearPrefix.Put(test.key, test.value) + trieDelete.Put(test.key, test.value, V0) + trieClearPrefix.Put(test.key, test.value, V0) } prefixedKeys := trieDelete.GetKeysWithPrefix(prefix) @@ -627,7 +627,7 @@ func TestSnapshot(t *testing.T) { expectedTrie := NewEmptyTrie() for _, test := range tests { - expectedTrie.Put(test.key, test.value) + expectedTrie.Put(test.key, test.value, V0) } // put all keys except first @@ -636,11 +636,11 @@ func TestSnapshot(t *testing.T) { if i == 0 { continue } - parentTrie.Put(test.key, test.value) + parentTrie.Put(test.key, test.value, V0) } newTrie := parentTrie.Snapshot() - newTrie.Put(tests[0].key, tests[0].value) + newTrie.Put(tests[0].key, tests[0].value, V0) require.Equal(t, expectedTrie.MustHash(), newTrie.MustHash()) require.NotEqual(t, parentTrie.MustHash(), newTrie.MustHash()) @@ -667,7 +667,7 @@ func Test_Trie_NextKey_Random(t *testing.T) { for _, key := range sortedKeys { value := []byte{1} - trie.Put(key, value) + trie.Put(key, value, V0) } for i, key := range sortedKeys { @@ -691,7 +691,7 @@ func Benchmark_Trie_Hash(b *testing.B) { trie := NewEmptyTrie() for keyString, value := range kv { key := []byte(keyString) - trie.Put(key, value) + trie.Put(key, value, V0) } b.StartTimer() @@ -732,7 +732,7 @@ func TestTrie_ConcurrentSnapshotWrites(t *testing.T) { switch op { case put: - expectedTries[i].Put(k, k) + expectedTries[i].Put(k, k, V0) case del: expectedTries[i].Delete(k) case clearPrefix: @@ -763,7 +763,7 @@ func TestTrie_ConcurrentSnapshotWrites(t *testing.T) { for _, operation := range operations { switch operation.op { case put: - trie.Put(operation.key, operation.key) + trie.Put(operation.key, operation.key, V0) case del: trie.Delete(operation.key) case clearPrefix: @@ -844,7 +844,7 @@ func TestTrie_ClearPrefixLimit(t *testing.T) { trieClearPrefix := NewEmptyTrie() for _, test := range testCase { - trieClearPrefix.Put(test.key, test.value) + trieClearPrefix.Put(test.key, test.value, V0) } num, allDeleted, err := trieClearPrefix.ClearPrefixLimit(prefix, uint32(lim)) @@ -950,7 +950,7 @@ func TestTrie_ClearPrefixLimitSnapshot(t *testing.T) { trieClearPrefix := NewEmptyTrie() for _, test := range testCase { - trieClearPrefix.Put(test.key, test.value) + trieClearPrefix.Put(test.key, test.value, V0) } dcTrie := trieClearPrefix.DeepCopy() @@ -1036,7 +1036,7 @@ func Test_encodeRoot_fuzz(t *testing.T) { kv := generateKeyValues(t, generator, kvSize) for keyString, value := range kv { key := []byte(keyString) - trie.Put(key, value) + trie.Put(key, value, V0) retrievedValue := trie.Get(key) assert.Equal(t, value, retrievedValue) @@ -1091,7 +1091,7 @@ func Test_Trie_Descendants_Fuzz(t *testing.T) { }) for _, key := range keys { - trie.Put(key, kv[string(key)]) + trie.Put(key, kv[string(key)], V0) } testDescendants(t, trie.root) diff --git a/lib/trie/trie_test.go b/lib/trie/trie_test.go index d484fe2be5..0ef35f9c6f 100644 --- a/lib/trie/trie_test.go +++ b/lib/trie/trie_test.go @@ -706,7 +706,7 @@ func Test_Trie_Entries(t *testing.T) { } for k, v := range kv { - trie.Put([]byte(k), v) + trie.Put([]byte(k), v, V0) } entries := trie.Entries() @@ -1100,7 +1100,7 @@ func Test_Trie_Put(t *testing.T) { t.Parallel() trie := testCase.trie - trie.Put(testCase.key, testCase.value) + trie.Put(testCase.key, testCase.value, V0) assert.Equal(t, testCase.expectedTrie, trie) }) diff --git a/lib/trie/version.go b/lib/trie/version.go index 23527890c8..16974e36fe 100644 --- a/lib/trie/version.go +++ b/lib/trie/version.go @@ -7,8 +7,12 @@ import ( "errors" "fmt" "strings" + + "github.com/ChainSafe/gossamer/lib/common" ) +const V1MaxValueSize = common.HashLength + // Version is the state trie version which dictates how a // Merkle root should be constructed. It is defined in // https://spec.polkadot.network/#defn-state-version @@ -18,13 +22,27 @@ const ( // V0 is the state trie version 0 where the values of the keys are // inserted into the trie directly. // TODO set to iota once CI passes - V0 Version = 1 + V0 Version = iota + V1 ) func (v Version) String() string { switch v { case V0: return "v0" + case V1: + return "v1" + default: + panic(fmt.Sprintf("unknown version %d", v)) + } +} + +func (v Version) ShouldHashValue(value []byte) bool { + switch v { + case V0: + return false + case V1: + return len(value) >= V1MaxValueSize default: panic(fmt.Sprintf("unknown version %d", v)) } @@ -37,8 +55,10 @@ func ParseVersion(s string) (version Version, err error) { switch { case strings.EqualFold(s, V0.String()): return V0, nil + case strings.EqualFold(s, V1.String()): + return V1, nil default: - return version, fmt.Errorf("%w: %q must be %s", - ErrParseVersion, s, V0) + return version, fmt.Errorf("%w: %q must be one of [%s, %s]", + ErrParseVersion, s, V0, V1) } } diff --git a/lib/trie/version_test.go b/lib/trie/version_test.go index ab2ac03ebe..0000a43099 100644 --- a/lib/trie/version_test.go +++ b/lib/trie/version_test.go @@ -65,7 +65,7 @@ func Test_ParseVersion(t *testing.T) { "invalid": { s: "xyz", errWrapped: ErrParseVersion, - errMessage: "parsing version failed: \"xyz\" must be v0", + errMessage: "parsing version failed: \"xyz\" must be one of [v0, v1]", }, } From b0a36cccab1ecc43f87eaba9c08655705c45231d Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 10 Aug 2023 13:11:40 -0300 Subject: [PATCH 003/128] Update tests --- dot/import.go | 3 +- lib/runtime/genesis.go | 3 +- lib/runtime/wasmer/exports_test.go | 5 ++-- lib/runtime/wazero/instance_test.go | 2 +- lib/trie/trie.go | 43 +++++++++++++++++++---------- lib/trie/trie_test.go | 17 +++++++----- 6 files changed, 47 insertions(+), 26 deletions(-) diff --git a/dot/import.go b/dot/import.go index c5d9c85f4e..5d0d9b313b 100644 --- a/dot/import.go +++ b/dot/import.go @@ -62,7 +62,8 @@ func newTrieFromPairs(filename string) (*trie.Trie, error) { entries[pairArr[0].(string)] = pairArr[1].(string) } - tr, err := trie.LoadFromMap(entries) + //TODO: revisit this to use the right trie version + tr, err := trie.LoadFromMap(entries, trie.V0) if err != nil { return nil, err } diff --git a/lib/runtime/genesis.go b/lib/runtime/genesis.go index 5912886bf1..b1547c3611 100644 --- a/lib/runtime/genesis.go +++ b/lib/runtime/genesis.go @@ -26,7 +26,8 @@ func NewTrieFromGenesis(gen genesis.Genesis) (tr trie.Trie, err error) { ErrGenesisTopNotFound, gen.Name) } - tr, err = trie.LoadFromMap(keyValues) + //TODO: check if we could get the trie version from genesis + tr, err = trie.LoadFromMap(keyValues, trie.V0) if err != nil { return tr, fmt.Errorf("loading genesis top key values into trie: %w", err) } diff --git a/lib/runtime/wasmer/exports_test.go b/lib/runtime/wasmer/exports_test.go index b02e3f1ece..367a03701a 100644 --- a/lib/runtime/wasmer/exports_test.go +++ b/lib/runtime/wasmer/exports_test.go @@ -998,7 +998,8 @@ func newTrieFromPairs(t *testing.T, filename string) *trie.Trie { entries[pairArr[0].(string)] = pairArr[1].(string) } - tr, err := trie.LoadFromMap(entries) + //TODO: revisit this to use the right trie version + tr, err := trie.LoadFromMap(entries, trie.V0) require.NoError(t, err) return &tr } @@ -1197,7 +1198,7 @@ func loadEntries(t *testing.T, filename string) *trie.Trie { err = json.Unmarshal(data, &entries) require.NoError(t, err) - tr, err := trie.LoadFromEntries(entries) + tr, err := trie.LoadFromEntries(entries, trie.V0) require.NoError(t, err) return tr } diff --git a/lib/runtime/wazero/instance_test.go b/lib/runtime/wazero/instance_test.go index b54ce2151c..1fc57eacad 100644 --- a/lib/runtime/wazero/instance_test.go +++ b/lib/runtime/wazero/instance_test.go @@ -962,7 +962,7 @@ func newTrieFromPairs(t *testing.T, filename string) *trie.Trie { entries[pairArr[0].(string)] = pairArr[1].(string) } - tr, err := trie.LoadFromMap(entries) + tr, err := trie.LoadFromMap(entries, trie.V0) require.NoError(t, err) return &tr } diff --git a/lib/trie/trie.go b/lib/trie/trie.go index 5f1067d5ba..025f6e619e 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -361,18 +361,25 @@ func (t *Trie) Put(keyLE, value []byte, version Version) (err error) { const success = true t.handleTrackedDeltas(success, pendingDeltas) }() - return t.insertKeyLE(keyLE, value, pendingDeltas) + return t.insertKeyLE(keyLE, value, pendingDeltas, version) } func (t *Trie) insertKeyLE(keyLE, value []byte, - pendingDeltas DeltaRecorder) (err error) { + pendingDeltas DeltaRecorder, version Version) (err error) { nibblesKey := codec.KeyLEToNibbles(keyLE) if value == nil { // Force nil value to be inserted to []byte{} since `nil` means there // is no value. value = []byte{} } - root, _, _, err := t.insert(t.root, nibblesKey, value, pendingDeltas) + + isValueHashed := version.ShouldHashValue(value) + if isValueHashed { + hashedValue := common.MustBlake2bHash(value) + copy(value, hashedValue[:]) + } + + root, _, _, err := t.insert(t.root, nibblesKey, value, isValueHashed, pendingDeltas) if err != nil { return err } @@ -383,7 +390,7 @@ func (t *Trie) insertKeyLE(keyLE, value []byte, // insert inserts a value in the trie at the key specified. // It may create one or more new nodes or update an existing node. func (t *Trie) insert(parent *Node, key, value []byte, - pendingDeltas DeltaRecorder) (newParent *Node, + isValueHashed bool, pendingDeltas DeltaRecorder) (newParent *Node, mutated bool, nodesCreated uint32, err error) { if parent == nil { mutated = true @@ -391,6 +398,7 @@ func (t *Trie) insert(parent *Node, key, value []byte, return &Node{ PartialKey: key, StorageValue: value, + HashedValue: isValueHashed, Generation: t.generation, Dirty: true, }, mutated, nodesCreated, nil @@ -398,7 +406,7 @@ func (t *Trie) insert(parent *Node, key, value []byte, if parent.Kind() == node.Branch { newParent, mutated, nodesCreated, err = t.insertInBranch( - parent, key, value, pendingDeltas) + parent, key, value, isValueHashed, pendingDeltas) if err != nil { // `insertInBranch` may call `insert` so do not wrap the // error since this may be a deep recursive call. @@ -408,7 +416,7 @@ func (t *Trie) insert(parent *Node, key, value []byte, } newParent, mutated, nodesCreated, err = t.insertInLeaf( - parent, key, value, pendingDeltas) + parent, key, value, isValueHashed, pendingDeltas) if err != nil { return nil, false, 0, fmt.Errorf("inserting in leaf: %w", err) } @@ -416,7 +424,7 @@ func (t *Trie) insert(parent *Node, key, value []byte, return newParent, mutated, nodesCreated, nil } -func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, +func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, isValueHashed bool, pendingDeltas DeltaRecorder) ( newParent *Node, mutated bool, nodesCreated uint32, err error) { if bytes.Equal(parentLeaf.PartialKey, key) { @@ -434,6 +442,7 @@ func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, } parentLeaf.StorageValue = value + parentLeaf.HashedValue = isValueHashed mutated = true return parentLeaf, mutated, nodesCreated, nil } @@ -453,6 +462,7 @@ func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, if len(key) == commonPrefixLength { // key is included in parent leaf key newBranchParent.StorageValue = value + newBranchParent.HashedValue = isValueHashed if len(key) < len(parentLeafKey) { // Move the current leaf parent as a child to the new branch. @@ -477,6 +487,7 @@ func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, if len(parentLeaf.PartialKey) == commonPrefixLength { // the key of the parent leaf is at this new branch newBranchParent.StorageValue = parentLeaf.StorageValue + newBranchParent.HashedValue = parentLeaf.HashedValue } else { // make the leaf a child of the new branch copySettings := node.DefaultCopySettings @@ -497,6 +508,7 @@ func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, newBranchParent.Children[childIndex] = &Node{ PartialKey: key[commonPrefixLength+1:], StorageValue: value, + HashedValue: isValueHashed, Generation: t.generation, Dirty: true, } @@ -506,7 +518,7 @@ func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, return newBranchParent, mutated, nodesCreated, nil } -func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, +func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, isValueHashed bool, pendingDeltas DeltaRecorder) ( newParent *Node, mutated bool, nodesCreated uint32, err error) { copySettings := node.DefaultCopySettings @@ -521,6 +533,7 @@ func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, return nil, false, 0, fmt.Errorf("preparing branch for mutation: %w", err) } parentBranch.StorageValue = value + parentBranch.HashedValue = isValueHashed mutated = true return parentBranch, mutated, 0, nil } @@ -536,6 +549,7 @@ func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, child = &Node{ PartialKey: remainingKey, StorageValue: value, + HashedValue: isValueHashed, Generation: t.generation, Dirty: true, } @@ -550,7 +564,7 @@ func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, return parentBranch, mutated, nodesCreated, nil } - child, mutated, nodesCreated, err = t.insert(child, remainingKey, value, pendingDeltas) + child, mutated, nodesCreated, err = t.insert(child, remainingKey, value, isValueHashed, pendingDeltas) if err != nil { // do not wrap error since `insert` may call `insertInBranch` recursively return nil, false, 0, err @@ -595,12 +609,13 @@ func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, if len(key) <= commonPrefixLength { newParentBranch.StorageValue = value + newParentBranch.HashedValue = isValueHashed } else { childIndex := key[commonPrefixLength] remainingKey := key[commonPrefixLength+1:] var additionalNodesCreated uint32 newParentBranch.Children[childIndex], _, additionalNodesCreated, err = t.insert( - nil, remainingKey, value, pendingDeltas) + nil, remainingKey, value, isValueHashed, pendingDeltas) if err != nil { // do not wrap error since `insert` may call `insertInBranch` recursively return nil, false, 0, err @@ -616,7 +631,7 @@ func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, // LoadFromMap loads the given data mapping of key to value into a new empty trie. // The keys are in hexadecimal little Endian encoding and the values // are hexadecimal encoded. -func LoadFromMap(data map[string]string) (trie Trie, err error) { +func LoadFromMap(data map[string]string, version Version) (trie Trie, err error) { trie = *NewEmptyTrie() pendingDeltas := tracking.New() @@ -635,7 +650,7 @@ func LoadFromMap(data map[string]string) (trie Trie, err error) { return Trie{}, fmt.Errorf("cannot convert value hex to bytes: %w", err) } - err = trie.insertKeyLE(keyLEBytes, valueBytes, pendingDeltas) + err = trie.insertKeyLE(keyLEBytes, valueBytes, pendingDeltas, version) if err != nil { return Trie{}, fmt.Errorf("inserting key value pair in trie: %w", err) } @@ -647,7 +662,7 @@ func LoadFromMap(data map[string]string) (trie Trie, err error) { // LoadFromEntries loads the given slice of key values into a new empty trie. // The keys are in hexadecimal little Endian encoding and the values // are hexadecimal encoded. -func LoadFromEntries(entries [][2][]byte) (trie *Trie, err error) { +func LoadFromEntries(entries [][2][]byte, version Version) (trie *Trie, err error) { trie = NewEmptyTrie() pendingDeltas := tracking.New() @@ -658,7 +673,7 @@ func LoadFromEntries(entries [][2][]byte) (trie *Trie, err error) { for _, keyValue := range entries { keyLE := keyValue[0] value := keyValue[1] - err := trie.insertKeyLE(keyLE, value, pendingDeltas) + err := trie.insertKeyLE(keyLE, value, pendingDeltas, version) if err != nil { return nil, err } diff --git a/lib/trie/trie_test.go b/lib/trie/trie_test.go index 0ef35f9c6f..2a36c65826 100644 --- a/lib/trie/trie_test.go +++ b/lib/trie/trie_test.go @@ -1115,6 +1115,7 @@ func Test_Trie_insert(t *testing.T) { parent *Node key []byte value []byte + isValueHashed bool pendingDeltas DeltaRecorder newNode *Node mutated bool @@ -1327,7 +1328,7 @@ func Test_Trie_insert(t *testing.T) { expectedTrie := *trie.DeepCopy() newNode, mutated, nodesCreated, err := trie.insert( - testCase.parent, testCase.key, testCase.value, + testCase.parent, testCase.key, testCase.value, testCase.isValueHashed, testCase.pendingDeltas) require.NoError(t, err) @@ -1347,6 +1348,7 @@ func Test_Trie_insertInBranch(t *testing.T) { parent *Node key []byte value []byte + isValueHashed bool pendingDeltas DeltaRecorder newNode *Node mutated bool @@ -1626,7 +1628,7 @@ func Test_Trie_insertInBranch(t *testing.T) { trie := new(Trie) newNode, mutated, nodesCreated, err := trie.insertInBranch( - testCase.parent, testCase.key, testCase.value, + testCase.parent, testCase.key, testCase.value, testCase.isValueHashed, testCase.pendingDeltas) assert.ErrorIs(t, err, testCase.errSentinel) @@ -1646,10 +1648,11 @@ func Test_LoadFromMap(t *testing.T) { t.Parallel() testCases := map[string]struct { - data map[string]string - expectedTrie Trie - errWrapped error - errMessage string + data map[string]string + stateTrieVersion Version + expectedTrie Trie + errWrapped error + errMessage string }{ "nil_data": { expectedTrie: Trie{ @@ -1734,7 +1737,7 @@ func Test_LoadFromMap(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - trie, err := LoadFromMap(testCase.data) + trie, err := LoadFromMap(testCase.data, testCase.stateTrieVersion) assert.ErrorIs(t, err, testCase.errWrapped) if testCase.errWrapped != nil { From 3293693d7724a61001078658650e505e3788596a Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 10 Aug 2023 14:29:00 -0300 Subject: [PATCH 004/128] Fix lint errors --- dot/core/service_integration_test.go | 4 ++-- dot/node_integration_test.go | 2 +- dot/rpc/modules/author_integration_test.go | 2 +- dot/rpc/modules/childstate_integration_test.go | 10 +++++----- dot/rpc/modules/dev_integration_test.go | 3 ++- dot/rpc/modules/state_integration_test.go | 5 +++-- dot/rpc/modules/system_integration_test.go | 2 +- dot/state/service_integration_test.go | 2 +- dot/utils_integration_test.go | 4 ++-- 9 files changed, 18 insertions(+), 16 deletions(-) diff --git a/dot/core/service_integration_test.go b/dot/core/service_integration_test.go index 1da5793dbf..e44dc01292 100644 --- a/dot/core/service_integration_test.go +++ b/dot/core/service_integration_test.go @@ -584,7 +584,7 @@ func createBlockUsingNewRuntime(t *testing.T, bestBlockHash common.Hash, newRunt testRuntime, err := os.ReadFile(newRuntimePath) require.NoError(t, err) - trieState.Put(common.CodeKey, testRuntime) + trieState.Put(common.CodeKey, testRuntime, trie.V0) primaryDigestData := types.NewBabePrimaryPreDigest(0, uint64(1), [32]byte{}, [64]byte{}) digest := types.NewDigest() @@ -673,7 +673,7 @@ func TestService_HandleRuntimeChangesAfterCodeSubstitutes(t *testing.T) { ts, err = s.storageState.TrieState(nil) require.NoError(t, err) - ts.Put(common.CodeKey, testRuntime) + ts.Put(common.CodeKey, testRuntime, trie.V0) rtUpdateBhash := newBlock.Header.Hash() // update runtime for new block diff --git a/dot/node_integration_test.go b/dot/node_integration_test.go index 283437a2af..64f3097b97 100644 --- a/dot/node_integration_test.go +++ b/dot/node_integration_test.go @@ -378,7 +378,7 @@ func TestInitNode_LoadStorageRoot(t *testing.T) { node, err := NewNode(config, ks) require.NoError(t, err) - expected, err := trie.LoadFromMap(gen.GenesisFields().Raw["top"]) + expected, err := trie.LoadFromMap(gen.GenesisFields().Raw["top"], trie.V0) require.NoError(t, err) expectedRoot, err := expected.Hash() diff --git a/dot/rpc/modules/author_integration_test.go b/dot/rpc/modules/author_integration_test.go index 9501e4288d..f1da5fe3fe 100644 --- a/dot/rpc/modules/author_integration_test.go +++ b/dot/rpc/modules/author_integration_test.go @@ -69,7 +69,7 @@ func useInstanceFromRuntimeV0929(t *testing.T, rtStorage *storage.TrieState) (in bytes, err := os.ReadFile(testRuntimeFilePath) require.NoError(t, err) - rtStorage.Put(common.CodeKey, bytes) + rtStorage.Put(common.CodeKey, bytes, trie.V0) cfg := wazero_runtime.Config{ Role: 0, diff --git a/dot/rpc/modules/childstate_integration_test.go b/dot/rpc/modules/childstate_integration_test.go index a46e8245b5..85b00f9ec6 100644 --- a/dot/rpc/modules/childstate_integration_test.go +++ b/dot/rpc/modules/childstate_integration_test.go @@ -244,13 +244,13 @@ func setupChildStateStorage(t *testing.T) (*ChildStateModule, common.Hash) { tr, err := st.Storage.TrieState(nil) require.NoError(t, err) - tr.Put([]byte(":first_key"), []byte(":value1")) - tr.Put([]byte(":second_key"), []byte(":second_value")) + tr.Put([]byte(":first_key"), []byte(":value1"), trie.V0) + tr.Put([]byte(":second_key"), []byte(":second_value"), trie.V0) childTr := trie.NewEmptyTrie() - childTr.Put([]byte(":child_first"), []byte(":child_first_value")) - childTr.Put([]byte(":child_second"), []byte(":child_second_value")) - childTr.Put([]byte(":another_child"), []byte("value")) + childTr.Put([]byte(":child_first"), []byte(":child_first_value"), trie.V0) + childTr.Put([]byte(":child_second"), []byte(":child_second_value"), trie.V0) + childTr.Put([]byte(":another_child"), []byte("value"), trie.V0) err = tr.SetChild([]byte(":child_storage_key"), childTr) require.NoError(t, err) diff --git a/dot/rpc/modules/dev_integration_test.go b/dot/rpc/modules/dev_integration_test.go index c779dbca5b..ebda0e5773 100644 --- a/dot/rpc/modules/dev_integration_test.go +++ b/dot/rpc/modules/dev_integration_test.go @@ -68,7 +68,8 @@ func newBABEService(t *testing.T) *babe.Service { "0d4a9e054df4e01000000000000001cbd2d43530a44705ad088af313e18f80b53ef16b36177cd4b77b846f2a5f07c010000"+ "00000000004603307f855321776922daeea21ee31720388d097cdaac66f05a6f8462b317570100000000000000be1d9d59d"+ "e1283380100550a7b024501cb62d6cc40e3db35fcc5cf341814986e01000000000000001206960f920a23f7f4c43cc9081"+ - "ec2ed0721f31a9bef2c10fd7602e16e08a32c0100000000000000")) + "ec2ed0721f31a9bef2c10fd7602e16e08a32c0100000000000000"), + trie.V0) cfg := &babe.ServiceConfig{ BlockState: bs, diff --git a/dot/rpc/modules/state_integration_test.go b/dot/rpc/modules/state_integration_test.go index 31ba029260..7fac6cccb8 100644 --- a/dot/rpc/modules/state_integration_test.go +++ b/dot/rpc/modules/state_integration_test.go @@ -17,6 +17,7 @@ import ( "github.com/ChainSafe/gossamer/dot/rpc/modules/mocks" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/trie" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -551,8 +552,8 @@ func setupStateModule(t *testing.T) (*StateModule, *common.Hash, *common.Hash) { ts, err := chain.Storage.TrieState(nil) require.NoError(t, err) - ts.Put([]byte(`:key2`), []byte(`value2`)) - ts.Put([]byte(`:key1`), []byte(`value1`)) + ts.Put([]byte(`:key2`), []byte(`value2`), trie.V0) + ts.Put([]byte(`:key1`), []byte(`value1`), trie.V0) ts.SetChildStorage([]byte(`:child1`), []byte(`:key1`), []byte(`:childValue1`)) sr1, err := ts.Root() diff --git a/dot/rpc/modules/system_integration_test.go b/dot/rpc/modules/system_integration_test.go index 453ac8bb72..b2a7dc745d 100644 --- a/dot/rpc/modules/system_integration_test.go +++ b/dot/rpc/modules/system_integration_test.go @@ -315,7 +315,7 @@ func setupSystemModule(t *testing.T) *SystemModule { aliceAcctEncoded, err := scale.Marshal(aliceAcctInfo) require.NoError(t, err) - ts.Put(aliceAcctStoKey, aliceAcctEncoded) + ts.Put(aliceAcctStoKey, aliceAcctEncoded, trie.V0) err = chain.Storage.StoreTrie(ts, nil) require.NoError(t, err) diff --git a/dot/state/service_integration_test.go b/dot/state/service_integration_test.go index 6ee2f9aa81..9dbc25c3d4 100644 --- a/dot/state/service_integration_test.go +++ b/dot/state/service_integration_test.go @@ -389,7 +389,7 @@ func TestService_Import(t *testing.T) { "bnm", } for _, tc := range testCases { - tr.Put([]byte(tc), []byte(tc)) + tr.Put([]byte(tc), []byte(tc), trie.V0) } digest := types.NewDigest() diff --git a/dot/utils_integration_test.go b/dot/utils_integration_test.go index 7f4e0a67a3..2b08c3dc03 100644 --- a/dot/utils_integration_test.go +++ b/dot/utils_integration_test.go @@ -27,7 +27,7 @@ func TestTrieSnapshot(t *testing.T) { for k, v := range genRaw.Genesis.Raw["top"] { val := []byte(v) - tri.Put([]byte(k), val) + tri.Put([]byte(k), val, trie.V0) } deepCopyTrie := tri.DeepCopy() @@ -51,7 +51,7 @@ func TestTrieSnapshot(t *testing.T) { // Modify the current trie. value[0] = 'w' - newTrie.Put(key, value) + newTrie.Put(key, value, trie.V0) // Get the updated root hash of all tries. tHash, err = tri.Hash() From 634f315140bcc0fd16217ba4e8fbd7c53f2a3734 Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 10 Aug 2023 18:14:39 -0300 Subject: [PATCH 005/128] Fix deepsource checks --- dot/state/storage_notify_test.go | 12 +++++------- dot/state/storage_test.go | 4 ++-- dot/state/test_helpers.go | 3 ++- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/dot/state/storage_notify_test.go b/dot/state/storage_notify_test.go index 1f2f0da551..50e9c36236 100644 --- a/dot/state/storage_notify_test.go +++ b/dot/state/storage_notify_test.go @@ -175,20 +175,18 @@ func Test_Example(t *testing.T) { } if err := db.Subscribe(ctx, cb, match); err != nil && err != context.Canceled { - log.Fatal(err) + t.Error(err) } log.Printf("subscription closed") }() // Write both keys, but only one should be printed in the Output. err := db.Put(aKey, aValue) - if err != nil { - log.Fatal(err) - } + require.NoError(t, err) + err = db.Put(bKey, bValue) - if err != nil { - log.Fatal(err) - } + require.NoError(t, err) + log.Printf("stopping subscription") cancel() log.Printf("waiting for subscription to close") diff --git a/dot/state/storage_test.go b/dot/state/storage_test.go index fd0e597f0a..7d6da57e65 100644 --- a/dot/state/storage_test.go +++ b/dot/state/storage_test.go @@ -45,8 +45,8 @@ func TestStorage_StoreAndLoadTrie(t *testing.T) { trie, err := storage.LoadFromDB(root) require.NoError(t, err) ts2 := runtime.NewTrieState(trie) - new := ts2.Snapshot() - require.Equal(t, ts.Trie(), new) + newTrie := ts2.Snapshot() + require.Equal(t, ts.Trie(), newTrie) } func TestStorage_GetStorageByBlockHash(t *testing.T) { diff --git a/dot/state/test_helpers.go b/dot/state/test_helpers.go index 56255b3198..11818a5924 100644 --- a/dot/state/test_helpers.go +++ b/dot/state/test_helpers.go @@ -153,7 +153,7 @@ func AddBlocksToState(t *testing.T, blockState *BlockState, depth uint, // branches are provided with a map of depth -> # of branches func AddBlocksToStateWithFixedBranches(t *testing.T, blockState *BlockState, depth uint, branches map[uint]int) { bestBlockHash := blockState.BestBlockHash() - tb := []testBranch{} + var tb []testBranch arrivalTime := time.Now() rt, err := blockState.GetRuntime(bestBlockHash) @@ -235,6 +235,7 @@ func AddBlocksToStateWithFixedBranches(t *testing.T, blockState *BlockState, dep } } +//lint:ignore U1000 this has a real reason func generateBlockWithRandomTrie(t *testing.T, serv *Service, parent *common.Hash, bNum uint) (*types.Block, *runtime.TrieState) { trieState, err := serv.Storage.TrieState(nil) From 1015ece4b9e49c5b9e7e282b55f596fbc08f6903 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 15 Aug 2023 11:38:20 -0300 Subject: [PATCH 006/128] Revisit TODOs --- cmd/gossamer/commands/import_state.go | 15 ++++++++++- dot/import.go | 8 +++--- lib/runtime/wasmer/exports_test.go | 17 ++++++------ lib/trie/proof/proof_test.go | 37 +++++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 14 deletions(-) diff --git a/cmd/gossamer/commands/import_state.go b/cmd/gossamer/commands/import_state.go index a12f137ba9..9f3f118b7b 100644 --- a/cmd/gossamer/commands/import_state.go +++ b/cmd/gossamer/commands/import_state.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/ChainSafe/gossamer/dot" + "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" "github.com/spf13/cobra" ) @@ -54,6 +55,18 @@ func execImportState(cmd *cobra.Command) error { return fmt.Errorf("state-file must be specified") } + stateVersionFlag, err := cmd.Flags().GetString("state-version") + if err != nil { + return fmt.Errorf("failed to get state-version: %s", err) + } + if stateVersionFlag == "" { + return fmt.Errorf("state-version must be specified") + } + stateVersion, err := trie.ParseVersion(stateVersionFlag) + if err != nil { + return fmt.Errorf("failed to parse state-version: %s", err) + } + headerFile, err := cmd.Flags().GetString("header-file") if err != nil { return fmt.Errorf("failed to get header-file: %s", err) @@ -64,5 +77,5 @@ func execImportState(cmd *cobra.Command) error { basePath = utils.ExpandDir(basePath) - return dot.ImportState(basePath, stateFile, headerFile, firstSlot) + return dot.ImportState(basePath, stateFile, headerFile, firstSlot, stateVersion) } diff --git a/dot/import.go b/dot/import.go index 5d0d9b313b..5e6687274e 100644 --- a/dot/import.go +++ b/dot/import.go @@ -20,8 +20,8 @@ import ( ) // ImportState imports the state in the given files to the database with the given path. -func ImportState(basepath, stateFP, headerFP string, firstSlot uint64) error { - tr, err := newTrieFromPairs(stateFP) +func ImportState(basepath, stateFP, headerFP string, firstSlot uint64, stateVersion trie.Version) error { + tr, err := newTrieFromPairs(stateFP, stateVersion) if err != nil { return err } @@ -41,7 +41,7 @@ func ImportState(basepath, stateFP, headerFP string, firstSlot uint64) error { return srv.Import(header, tr, firstSlot) } -func newTrieFromPairs(filename string) (*trie.Trie, error) { +func newTrieFromPairs(filename string, stateVersion trie.Version) (*trie.Trie, error) { data, err := os.ReadFile(filepath.Clean(filename)) if err != nil { return nil, err @@ -63,7 +63,7 @@ func newTrieFromPairs(filename string) (*trie.Trie, error) { } //TODO: revisit this to use the right trie version - tr, err := trie.LoadFromMap(entries, trie.V0) + tr, err := trie.LoadFromMap(entries, stateVersion) if err != nil { return nil, err } diff --git a/lib/runtime/wasmer/exports_test.go b/lib/runtime/wasmer/exports_test.go index 367a03701a..0be7186f70 100644 --- a/lib/runtime/wasmer/exports_test.go +++ b/lib/runtime/wasmer/exports_test.go @@ -627,7 +627,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1(t *testing.T) { } func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock3784(t *testing.T) { - gossTrie3783 := newTrieFromPairs(t, "../test_data/kusama/block3783.out") + gossTrie3783 := newTrieFromPairs(t, "../test_data/kusama/block3783.out", trie.V0) expectedRoot := common.MustHexToHash("0x948338bc0976aee78879d559a1f42385407e5a481b05a91d2a9386aa7507e7a0") require.Equal(t, expectedRoot, gossTrie3783.MustHash()) @@ -673,7 +673,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock3784(t *testing.T) { } func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock901442(t *testing.T) { - ksmTrie901441 := newTrieFromPairs(t, "../test_data/kusama/block901441.out") + ksmTrie901441 := newTrieFromPairs(t, "../test_data/kusama/block901441.out", trie.V0) expectedRoot := common.MustHexToHash("0x3a2ef7ee032f5810160bb8f3ffe3e3377bb6f2769ee9f79a5425973347acd504") require.Equal(t, expectedRoot, ksmTrie901441.MustHash()) @@ -719,7 +719,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock901442(t *testing.T) { } func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1377831(t *testing.T) { - ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block1377830.out") + ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block1377830.out", trie.V0) expectedRoot := common.MustHexToHash("0xe4de6fecda9e9e35f937d159665cf984bc1a68048b6c78912de0aeb6bd7f7e99") require.Equal(t, expectedRoot, ksmTrie.MustHash()) @@ -765,7 +765,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1377831(t *testing.T) { } func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1482003(t *testing.T) { - ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block1482002.out") + ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block1482002.out", trie.V0) expectedRoot := common.MustHexToHash("0x09f9ca28df0560c2291aa16b56e15e07d1e1927088f51356d522722aa90ca7cb") require.Equal(t, expectedRoot, ksmTrie.MustHash()) @@ -813,7 +813,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1482003(t *testing.T) { func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock4939774(t *testing.T) { t.Skip("skip for now as block4939773 is too large") - ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block4939773.out") + ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block4939773.out", trie.V0) expectedRoot := common.MustHexToHash("0xc45748e6e8632b44fc32b04cc4380098a9584cbd63ffbc59adce189574fc36fe") require.Equal(t, expectedRoot, ksmTrie.MustHash()) @@ -856,7 +856,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock4939774(t *testing.T) { } func TestInstance_ExecuteBlock_PolkadotBlock1089328(t *testing.T) { - dotTrie := newTrieFromPairs(t, "../test_data/polkadot/block1089327.json") + dotTrie := newTrieFromPairs(t, "../test_data/polkadot/block1089327.json", trie.V0) expectedRoot := common.MustHexToHash("0x87ed9ebe7fb645d3b5b0255cc16e78ed022d9fbb52486105436e15a74557535b") require.Equal(t, expectedRoot, dotTrie.MustHash()) @@ -983,7 +983,7 @@ func TestInstance_PaymentQueryInfo(t *testing.T) { } } -func newTrieFromPairs(t *testing.T, filename string) *trie.Trie { +func newTrieFromPairs(t *testing.T, filename string, version trie.Version) *trie.Trie { data, err := os.ReadFile(filename) require.NoError(t, err) @@ -998,8 +998,7 @@ func newTrieFromPairs(t *testing.T, filename string) *trie.Trie { entries[pairArr[0].(string)] = pairArr[1].(string) } - //TODO: revisit this to use the right trie version - tr, err := trie.LoadFromMap(entries, trie.V0) + tr, err := trie.LoadFromMap(entries, version) require.NoError(t, err) return &tr } diff --git a/lib/trie/proof/proof_test.go b/lib/trie/proof/proof_test.go index f808c999b4..49c1b3110f 100644 --- a/lib/trie/proof/proof_test.go +++ b/lib/trie/proof/proof_test.go @@ -99,3 +99,40 @@ func TestParachainHeaderStateProof(t *testing.T) { require.NoError(t, err) } + +func TestTrieProof(t *testing.T) { + key, err := hex.DecodeString("f0c365c3cf59d671eb72da0e7a4113c49f1f0515f462cdcf84e0f1d6045dfcbb") + if err != nil { + panic(err) + } + root, err := hex.DecodeString("dc4887669c2a6b3462e9557aa3105a66a02b6ec3b21784613de78c95dc3cbbe0") + if err != nil { + panic(err) + } + bytes1, err := hex.DecodeString("80fffd8028b54b9a0a90d41b7941c43e6a0597d5914e3b62bdcb244851b9fc806c28ea2480d5ba6d50586692888b0c2f5b3c3fc345eb3a2405996f025ed37982ca396f5ed580bd281c12f20f06077bffd56b2f8b6431ee6c9fd11fed9c22db86cea849aeff2280afa1e1b5ce72ea1675e5e69be85e98fbfb660691a76fee9229f758a75315f2bc80aafc60caa3519d4b861e6b8da226266a15060e2071bba4184e194da61dfb208e809d3f6ae8f655009551de95ae1ef863f6771522fd5c0475a50ff53c5c8169b5888024a760a8f6c27928ae9e2fed9968bc5f6e17c3ae647398d8a615e5b2bb4b425f8085a0da830399f25fca4b653de654ffd3c92be39f3ae4f54e7c504961b5bd00cf80c2d44d371e5fc1f50227d7491ad65ad049630361cefb4ab1844831237609f08380c644938921d14ae611f3a90991af8b7f5bdb8fa361ee2c646c849bca90f491e6806e729ad43a591cd1321762582782bbe4ed193c6f583ec76013126f7f786e376280509bb016f2887d12137e73d26d7ddcd7f9c8ff458147cb9d309494655fe68de180009f8697d760fbe020564b07f407e6aad58ba9451b3d2d88b3ee03e12db7c47480952dcc0804e1120508a1753f1de4aa5b7481026a3320df8b48e918f0cecbaed3803360bf948fddc403d345064082e8393d7a1aad7a19081f6d02d94358f242b86c") + if err != nil { + panic(err) + } + bytes2, err := hex.DecodeString("9ec365c3cf59d671eb72da0e7a4113c41002505f0e7b9012096b41c4eb3aaf947f6ea429080000685f0f1f0515f462cdcf84e0f1d6045dfcbb20865c4a2b7f010000") + if err != nil { + panic(err) + } + bytes3, err := hex.DecodeString("8005088076c66e2871b4fe037d112ebffb3bfc8bd83a4ec26047f58ee2df7be4e9ebe3d680c1638f702aaa71e4b78cc8538ecae03e827bb494cc54279606b201ec071a5e24806d2a1e6d5236e1e13c5a5c84831f5f5383f97eba32df6f9faf80e32cf2f129bc") + if err != nil { + panic(err) + } + var proof = [][]byte{ + bytes1, bytes2, bytes3, + } + proofDB, err := db.NewMemoryDBFromProof(proof) + require.NoError(t, err) + + trie, err := buildTrie(proof, root, proofDB) + require.NoError(t, err) + value := trie.Get(key) + + expectedValue := 0x865c4a2b7f100 + require.Equal(t, expectedValue, value) + + fmt.Printf("value: %s\n", value) +} From 9d828c7eb5a80a1d0a17f82d21b1abc52f075c7b Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 15 Aug 2023 12:22:20 -0300 Subject: [PATCH 007/128] Add more tests --- lib/trie/proof/proof_test.go | 37 ------------------- lib/trie/trie.go | 3 +- lib/trie/trie_test.go | 71 ++++++++++++++++++++++++++++++++++-- 3 files changed, 69 insertions(+), 42 deletions(-) diff --git a/lib/trie/proof/proof_test.go b/lib/trie/proof/proof_test.go index 49c1b3110f..f808c999b4 100644 --- a/lib/trie/proof/proof_test.go +++ b/lib/trie/proof/proof_test.go @@ -99,40 +99,3 @@ func TestParachainHeaderStateProof(t *testing.T) { require.NoError(t, err) } - -func TestTrieProof(t *testing.T) { - key, err := hex.DecodeString("f0c365c3cf59d671eb72da0e7a4113c49f1f0515f462cdcf84e0f1d6045dfcbb") - if err != nil { - panic(err) - } - root, err := hex.DecodeString("dc4887669c2a6b3462e9557aa3105a66a02b6ec3b21784613de78c95dc3cbbe0") - if err != nil { - panic(err) - } - bytes1, err := hex.DecodeString("80fffd8028b54b9a0a90d41b7941c43e6a0597d5914e3b62bdcb244851b9fc806c28ea2480d5ba6d50586692888b0c2f5b3c3fc345eb3a2405996f025ed37982ca396f5ed580bd281c12f20f06077bffd56b2f8b6431ee6c9fd11fed9c22db86cea849aeff2280afa1e1b5ce72ea1675e5e69be85e98fbfb660691a76fee9229f758a75315f2bc80aafc60caa3519d4b861e6b8da226266a15060e2071bba4184e194da61dfb208e809d3f6ae8f655009551de95ae1ef863f6771522fd5c0475a50ff53c5c8169b5888024a760a8f6c27928ae9e2fed9968bc5f6e17c3ae647398d8a615e5b2bb4b425f8085a0da830399f25fca4b653de654ffd3c92be39f3ae4f54e7c504961b5bd00cf80c2d44d371e5fc1f50227d7491ad65ad049630361cefb4ab1844831237609f08380c644938921d14ae611f3a90991af8b7f5bdb8fa361ee2c646c849bca90f491e6806e729ad43a591cd1321762582782bbe4ed193c6f583ec76013126f7f786e376280509bb016f2887d12137e73d26d7ddcd7f9c8ff458147cb9d309494655fe68de180009f8697d760fbe020564b07f407e6aad58ba9451b3d2d88b3ee03e12db7c47480952dcc0804e1120508a1753f1de4aa5b7481026a3320df8b48e918f0cecbaed3803360bf948fddc403d345064082e8393d7a1aad7a19081f6d02d94358f242b86c") - if err != nil { - panic(err) - } - bytes2, err := hex.DecodeString("9ec365c3cf59d671eb72da0e7a4113c41002505f0e7b9012096b41c4eb3aaf947f6ea429080000685f0f1f0515f462cdcf84e0f1d6045dfcbb20865c4a2b7f010000") - if err != nil { - panic(err) - } - bytes3, err := hex.DecodeString("8005088076c66e2871b4fe037d112ebffb3bfc8bd83a4ec26047f58ee2df7be4e9ebe3d680c1638f702aaa71e4b78cc8538ecae03e827bb494cc54279606b201ec071a5e24806d2a1e6d5236e1e13c5a5c84831f5f5383f97eba32df6f9faf80e32cf2f129bc") - if err != nil { - panic(err) - } - var proof = [][]byte{ - bytes1, bytes2, bytes3, - } - proofDB, err := db.NewMemoryDBFromProof(proof) - require.NoError(t, err) - - trie, err := buildTrie(proof, root, proofDB) - require.NoError(t, err) - value := trie.Get(key) - - expectedValue := 0x865c4a2b7f100 - require.Equal(t, expectedValue, value) - - fmt.Printf("value: %s\n", value) -} diff --git a/lib/trie/trie.go b/lib/trie/trie.go index 025f6e619e..6e8e50eb6a 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -375,8 +375,7 @@ func (t *Trie) insertKeyLE(keyLE, value []byte, isValueHashed := version.ShouldHashValue(value) if isValueHashed { - hashedValue := common.MustBlake2bHash(value) - copy(value, hashedValue[:]) + value = common.MustBlake2bHash(value).ToBytes() } root, _, _, err := t.insert(t.root, nibblesKey, value, isValueHashed, pendingDeltas) diff --git a/lib/trie/trie_test.go b/lib/trie/trie_test.go index 2a36c65826..2a58d7ebc3 100644 --- a/lib/trie/trie_test.go +++ b/lib/trie/trie_test.go @@ -690,7 +690,7 @@ func Test_Trie_Entries(t *testing.T) { entriesMatch(t, expectedEntries, entries) }) - t.Run("end_to_end", func(t *testing.T) { + t.Run("end_to_end_v0", func(t *testing.T) { t.Parallel() trie := Trie{ @@ -713,6 +713,30 @@ func Test_Trie_Entries(t *testing.T) { assert.Equal(t, kv, entries) }) + + t.Run("end_to_end_v1", func(t *testing.T) { + t.Parallel() + + trie := Trie{ + root: nil, + childTries: make(map[common.Hash]*Trie), + } + + kv := map[string][]byte{ + "ab": []byte("pen"), + "abc": []byte("penguin"), + "hy": []byte("feather"), + "long": []byte("newvaluewithmorethan32byteslength"), + } + + for k, v := range kv { + trie.Put([]byte(k), v, V1) + } + + entries := trie.Entries() + + assert.Equal(t, kv, entries) + }) } func Test_Trie_NextKey(t *testing.T) { @@ -1050,13 +1074,16 @@ func Test_nextKey(t *testing.T) { func Test_Trie_Put(t *testing.T) { t.Parallel() + longValue := []byte("newvaluewithmorethan32byteslength") + testCases := map[string]struct { trie Trie + stateVersion Version key []byte value []byte expectedTrie Trie }{ - "trie_with_key_and_value": { + "trie_v0_with_key_and_value": { trie: Trie{ generation: 1, deltas: newDeltas(), @@ -1092,6 +1119,44 @@ func Test_Trie_Put(t *testing.T) { }, }, }, + "trie_v1_with_key_and_value": { + trie: Trie{ + generation: 1, + deltas: newDeltas(), + root: &Node{ + PartialKey: []byte{1, 2, 0, 5}, + StorageValue: []byte{1}, + }, + }, + stateVersion: V1, + key: []byte{0x12, 0x16}, + value: longValue, + expectedTrie: Trie{ + generation: 1, + deltas: newDeltas("0xa195089c3e8f8b5b36978700ad954aed99e08413cfc1e2b4c00a5d064abe66a9"), + root: &Node{ + PartialKey: []byte{1, 2}, + Generation: 1, + Dirty: true, + Descendants: 2, + Children: padRightChildren([]*Node{ + { + PartialKey: []byte{5}, + StorageValue: []byte{1}, + Generation: 1, + Dirty: true, + }, + { + PartialKey: []byte{6}, + StorageValue: common.MustBlake2bHash(longValue).ToBytes(), + HashedValue: true, + Generation: 1, + Dirty: true, + }, + }), + }, + }, + }, } for name, testCase := range testCases { @@ -1100,7 +1165,7 @@ func Test_Trie_Put(t *testing.T) { t.Parallel() trie := testCase.trie - trie.Put(testCase.key, testCase.value, V0) + trie.Put(testCase.key, testCase.value, testCase.stateVersion) assert.Equal(t, testCase.expectedTrie, trie) }) From b09f7a5defac06d561f0138c71c246520cede6e9 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 15 Aug 2023 14:33:18 -0300 Subject: [PATCH 008/128] Fix linter --- dot/import_integration_test.go | 10 +++++++--- lib/runtime/wasmer/exports_test.go | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/dot/import_integration_test.go b/dot/import_integration_test.go index 534f50b4cf..8dcc0efc0c 100644 --- a/dot/import_integration_test.go +++ b/dot/import_integration_test.go @@ -13,6 +13,7 @@ import ( "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/internal/log" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/pkg/scale" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -24,6 +25,7 @@ func Test_newTrieFromPairs(t *testing.T) { tests := []struct { name string filename string + version trie.Version want common.Hash err error }{ @@ -35,6 +37,7 @@ func Test_newTrieFromPairs(t *testing.T) { { name: "working example", filename: setupStateFile(t), + version: trie.V0, want: common.MustHexToHash("0x09f9ca28df0560c2291aa16b56e15e07d1e1927088f51356d522722aa90ca7cb"), }, } @@ -43,7 +46,7 @@ func Test_newTrieFromPairs(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - got, err := newTrieFromPairs(tt.filename) + got, err := newTrieFromPairs(tt.filename, tt.version) if tt.err != nil { assert.EqualError(t, err, tt.err.Error()) } else { @@ -93,7 +96,7 @@ func TestImportState_Integration(t *testing.T) { headerFP := setupHeaderFile(t) const firstSlot = uint64(262493679) - err = ImportState(config.BasePath, stateFP, headerFP, firstSlot) + err = ImportState(config.BasePath, stateFP, headerFP, firstSlot, trie.V0) require.NoError(t, err) // confirm data is imported into db stateConfig := state.Config{ @@ -126,6 +129,7 @@ func TestImportState(t *testing.T) { type args struct { basepath string stateFP string + version trie.Version headerFP string firstSlot uint64 } @@ -153,7 +157,7 @@ func TestImportState(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - err := ImportState(tt.args.basepath, tt.args.stateFP, tt.args.headerFP, tt.args.firstSlot) + err := ImportState(tt.args.basepath, tt.args.stateFP, tt.args.headerFP, tt.args.firstSlot, tt.args.version) if tt.err != nil { assert.EqualError(t, err, tt.err.Error()) } else { diff --git a/lib/runtime/wasmer/exports_test.go b/lib/runtime/wasmer/exports_test.go index 0be7186f70..d20d6eaddc 100644 --- a/lib/runtime/wasmer/exports_test.go +++ b/lib/runtime/wasmer/exports_test.go @@ -627,7 +627,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1(t *testing.T) { } func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock3784(t *testing.T) { - gossTrie3783 := newTrieFromPairs(t, "../test_data/kusama/block3783.out", trie.V0) + gossTrie3783 := newTrieFromPairs(t, "../test_data/kusama/block3783.out", trie.V1) expectedRoot := common.MustHexToHash("0x948338bc0976aee78879d559a1f42385407e5a481b05a91d2a9386aa7507e7a0") require.Equal(t, expectedRoot, gossTrie3783.MustHash()) From 5d256447a892e76d1adcf6dcf83d51ea08b22839 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 15 Aug 2023 17:28:34 -0300 Subject: [PATCH 009/128] Fix linter --- dot/state/service_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dot/state/service_integration_test.go b/dot/state/service_integration_test.go index 3d696fd846..91572d482c 100644 --- a/dot/state/service_integration_test.go +++ b/dot/state/service_integration_test.go @@ -437,7 +437,7 @@ func generateBlockWithRandomTrie(t *testing.T, serv *Service, rand := time.Now().UnixNano() key := []byte("testKey" + fmt.Sprint(rand)) value := []byte("testValue" + fmt.Sprint(rand)) - err = trieState.Put(key, value) + err = trieState.Put(key, value, trie.V0) require.NoError(t, err) trieStateRoot, err := trieState.Root() From a625bc4914ccc396836be6a743f01b396519db87 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 15 Aug 2023 21:28:10 -0300 Subject: [PATCH 010/128] Get hashed values --- dot/state/db_getter_mocks_test.go | 49 --------------------- dot/state/db_mocks_test.go | 63 +++++++++++++++++++++++++++ dot/state/interfaces.go | 3 +- dot/state/mocks_generate_test.go | 2 +- dot/state/storage.go | 2 +- dot/state/storage_test.go | 6 +-- dot/state/tries_test.go | 14 +++--- lib/runtime/wasmer/exports_test.go | 2 +- lib/trie/db/db.go | 25 +++++++++-- lib/trie/proof/database_mocks_test.go | 14 ++++++ lib/trie/proof/generate.go | 3 +- lib/trie/trie.go | 43 +++++++++++------- lib/trie/trie_test.go | 23 ++++++++-- 13 files changed, 162 insertions(+), 87 deletions(-) delete mode 100644 dot/state/db_getter_mocks_test.go create mode 100644 dot/state/db_mocks_test.go diff --git a/dot/state/db_getter_mocks_test.go b/dot/state/db_getter_mocks_test.go deleted file mode 100644 index 9493704a06..0000000000 --- a/dot/state/db_getter_mocks_test.go +++ /dev/null @@ -1,49 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/ChainSafe/gossamer/lib/trie (interfaces: DBGetter) - -// Package state is a generated GoMock package. -package state - -import ( - reflect "reflect" - - gomock "github.com/golang/mock/gomock" -) - -// MockDBGetter is a mock of DBGetter interface. -type MockDBGetter struct { - ctrl *gomock.Controller - recorder *MockDBGetterMockRecorder -} - -// MockDBGetterMockRecorder is the mock recorder for MockDBGetter. -type MockDBGetterMockRecorder struct { - mock *MockDBGetter -} - -// NewMockDBGetter creates a new mock instance. -func NewMockDBGetter(ctrl *gomock.Controller) *MockDBGetter { - mock := &MockDBGetter{ctrl: ctrl} - mock.recorder = &MockDBGetterMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockDBGetter) EXPECT() *MockDBGetterMockRecorder { - return m.recorder -} - -// Get mocks base method. -func (m *MockDBGetter) Get(arg0 []byte) ([]byte, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Get", arg0) - ret0, _ := ret[0].([]byte) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Get indicates an expected call of Get. -func (mr *MockDBGetterMockRecorder) Get(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockDBGetter)(nil).Get), arg0) -} diff --git a/dot/state/db_mocks_test.go b/dot/state/db_mocks_test.go new file mode 100644 index 0000000000..6df73ac4d8 --- /dev/null +++ b/dot/state/db_mocks_test.go @@ -0,0 +1,63 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/ChainSafe/gossamer/lib/trie (interfaces: Database) + +// Package state is a generated GoMock package. +package state + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockDatabase is a mock of Database interface. +type MockDatabase struct { + ctrl *gomock.Controller + recorder *MockDatabaseMockRecorder +} + +// MockDatabaseMockRecorder is the mock recorder for MockDatabase. +type MockDatabaseMockRecorder struct { + mock *MockDatabase +} + +// NewMockDatabase creates a new mock instance. +func NewMockDatabase(ctrl *gomock.Controller) *MockDatabase { + mock := &MockDatabase{ctrl: ctrl} + mock.recorder = &MockDatabaseMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDatabase) EXPECT() *MockDatabaseMockRecorder { + return m.recorder +} + +// Get mocks base method. +func (m *MockDatabase) Get(arg0 []byte) ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", arg0) + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockDatabaseMockRecorder) Get(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockDatabase)(nil).Get), arg0) +} + +// Put mocks base method. +func (m *MockDatabase) Put(arg0, arg1 []byte) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Put", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Put indicates an expected call of Put. +func (mr *MockDatabaseMockRecorder) Put(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*MockDatabase)(nil).Put), arg0, arg1) +} diff --git a/dot/state/interfaces.go b/dot/state/interfaces.go index bae7631e51..7c193e3af6 100644 --- a/dot/state/interfaces.go +++ b/dot/state/interfaces.go @@ -31,8 +31,9 @@ type GetPutter interface { // GetNewBatcher has methods to get values and create a // new batch. -type GetNewBatcher interface { +type Batcher interface { Getter + Putter NewBatcher } diff --git a/dot/state/mocks_generate_test.go b/dot/state/mocks_generate_test.go index e52212d053..fdac9b006f 100644 --- a/dot/state/mocks_generate_test.go +++ b/dot/state/mocks_generate_test.go @@ -7,4 +7,4 @@ package state //go:generate mockgen -destination=mocks_runtime_test.go -package $GOPACKAGE github.com/ChainSafe/gossamer/lib/runtime Instance //go:generate mockgen -destination=mock_gauge_test.go -package $GOPACKAGE github.com/prometheus/client_golang/prometheus Gauge //go:generate mockgen -destination=mock_counter_test.go -package $GOPACKAGE github.com/prometheus/client_golang/prometheus Counter -//go:generate mockgen -destination=db_getter_mocks_test.go -package=$GOPACKAGE github.com/ChainSafe/gossamer/lib/trie DBGetter +//go:generate mockgen -destination=db_mocks_test.go -package=$GOPACKAGE github.com/ChainSafe/gossamer/lib/trie Database diff --git a/dot/state/storage.go b/dot/state/storage.go index efaba3acff..15d2810cb3 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -33,7 +33,7 @@ type StorageState struct { blockState *BlockState tries *Tries - db GetNewBatcher + db Batcher sync.RWMutex // change notifiers diff --git a/dot/state/storage_test.go b/dot/state/storage_test.go index 7d6da57e65..62de71be2c 100644 --- a/dot/state/storage_test.go +++ b/dot/state/storage_test.go @@ -178,15 +178,15 @@ func TestGetStorageChildAndGetStorageFromChild(t *testing.T) { genHeader.Hash(), "0", )) - dbGetter := NewMockDBGetter(ctrl) - dbGetter.EXPECT().Get(gomock.Any()).Times(0) + trieDB := NewMockDatabase(ctrl) + trieDB.EXPECT().Get(gomock.Any()).Times(0) trieRoot := &node.Node{ PartialKey: []byte{1, 2}, StorageValue: []byte{3, 4}, Dirty: true, } - testChildTrie := trie.NewTrie(trieRoot, dbGetter) + testChildTrie := trie.NewTrie(trieRoot, trieDB) testChildTrie.Put([]byte("keyInsidechild"), []byte("voila"), trie.V0) diff --git a/dot/state/tries_test.go b/dot/state/tries_test.go index f97c4934e9..4bdc84e1d2 100644 --- a/dot/state/tries_test.go +++ b/dot/state/tries_test.go @@ -49,10 +49,10 @@ func Test_Tries_SetEmptyTrie(t *testing.T) { func Test_Tries_SetTrie(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) - dbGetter := NewMockDBGetter(ctrl) - dbGetter.EXPECT().Get(gomock.Any()).Times(0) + db := NewMockDatabase(ctrl) + db.EXPECT().Get(gomock.Any()).Times(0) - tr := trie.NewTrie(&node.Node{PartialKey: []byte{1}}, dbGetter) + tr := trie.NewTrie(&node.Node{PartialKey: []byte{1}}, db) tries := NewTries() tries.SetTrie(tr) @@ -192,8 +192,8 @@ func Test_Tries_delete(t *testing.T) { func Test_Tries_get(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) - dbGetter := NewMockDBGetter(ctrl) - dbGetter.EXPECT().Get(gomock.Any()).Times(0) + db := NewMockDatabase(ctrl) + db.EXPECT().Get(gomock.Any()).Times(0) testCases := map[string]struct { tries *Tries @@ -206,14 +206,14 @@ func Test_Tries_get(t *testing.T) { {1, 2, 3}: trie.NewTrie(&node.Node{ PartialKey: []byte{1, 2, 3}, StorageValue: []byte{1}, - }, dbGetter), + }, db), }, }, root: common.Hash{1, 2, 3}, trie: trie.NewTrie(&node.Node{ PartialKey: []byte{1, 2, 3}, StorageValue: []byte{1}, - }, dbGetter), + }, db), }, "not_found_in_map": { // similar to not found in database diff --git a/lib/runtime/wasmer/exports_test.go b/lib/runtime/wasmer/exports_test.go index d20d6eaddc..0be7186f70 100644 --- a/lib/runtime/wasmer/exports_test.go +++ b/lib/runtime/wasmer/exports_test.go @@ -627,7 +627,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1(t *testing.T) { } func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock3784(t *testing.T) { - gossTrie3783 := newTrieFromPairs(t, "../test_data/kusama/block3783.out", trie.V1) + gossTrie3783 := newTrieFromPairs(t, "../test_data/kusama/block3783.out", trie.V0) expectedRoot := common.MustHexToHash("0x948338bc0976aee78879d559a1f42385407e5a481b05a91d2a9386aa7507e7a0") require.Equal(t, expectedRoot, gossTrie3783.MustHash()) diff --git a/lib/trie/db/db.go b/lib/trie/db/db.go index 2c04b28e05..cdbd3027c7 100644 --- a/lib/trie/db/db.go +++ b/lib/trie/db/db.go @@ -10,6 +10,13 @@ import ( type MemoryDB struct { data map[common.Hash][]byte + // TODO: add lock +} + +func NewEmptyMemoryDB() *MemoryDB { + return &MemoryDB{ + data: make(map[common.Hash][]byte), + } } func NewMemoryDBFromProof(encodedNodes [][]byte) (*MemoryDB, error) { @@ -30,9 +37,9 @@ func NewMemoryDBFromProof(encodedNodes [][]byte) (*MemoryDB, error) { } -func (mdb *MemoryDB) Get(key []byte) (value []byte, err error) { - if len(key) < common.HashLength { - return nil, fmt.Errorf("expected %d bytes length key, given %d (%x)", common.HashLength, len(key), value) +func (mdb *MemoryDB) Get(key []byte) ([]byte, error) { + if len(key) != common.HashLength { + return nil, fmt.Errorf("expected %d bytes length key, given %d (%x)", common.HashLength, len(key), key) } var hash common.Hash copy(hash[:], key) @@ -43,3 +50,15 @@ func (mdb *MemoryDB) Get(key []byte) (value []byte, err error) { return nil, nil } + +func (mdb *MemoryDB) Put(key []byte, value []byte) error { + if len(key) != common.HashLength { + return fmt.Errorf("expected %d bytes length key, given %d (%x)", common.HashLength, len(key), key) + } + + var hash common.Hash + copy(hash[:], key) + + mdb.data[hash] = value + return nil +} diff --git a/lib/trie/proof/database_mocks_test.go b/lib/trie/proof/database_mocks_test.go index 69262dc315..ec07c4c1ee 100644 --- a/lib/trie/proof/database_mocks_test.go +++ b/lib/trie/proof/database_mocks_test.go @@ -47,3 +47,17 @@ func (mr *MockDatabaseMockRecorder) Get(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockDatabase)(nil).Get), arg0) } + +// Put mocks base method. +func (m *MockDatabase) Put(arg0, arg1 []byte) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Put", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Put indicates an expected call of Put. +func (mr *MockDatabaseMockRecorder) Put(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*MockDatabase)(nil).Put), arg0, arg1) +} diff --git a/lib/trie/proof/generate.go b/lib/trie/proof/generate.go index 2fd30d9ddd..6155ebac1c 100644 --- a/lib/trie/proof/generate.go +++ b/lib/trie/proof/generate.go @@ -22,7 +22,8 @@ var ( // Database defines a key value Get method used // for proof generation. type Database interface { - Get(key []byte) (value []byte, err error) + trie.DBGetter + trie.DBPutter } // Generate generates and deduplicates the encoded proof nodes diff --git a/lib/trie/trie.go b/lib/trie/trie.go index 6e8e50eb6a..a5749e6123 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -11,17 +11,23 @@ import ( "github.com/ChainSafe/gossamer/internal/trie/node" "github.com/ChainSafe/gossamer/internal/trie/tracking" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/trie/db" ) // EmptyHash is the empty trie hash. var EmptyHash = common.MustBlake2bHash([]byte{0}) +type Database interface { + DBGetter + DBPutter +} + // Trie is a base 16 modified Merkle Patricia trie. type Trie struct { generation uint64 root *Node childTries map[common.Hash]*Trie - db DBGetter + db Database // deltas stores trie deltas since the last trie snapshot. // For example node hashes that were deleted since // the last snapshot. These are used by the online @@ -32,11 +38,11 @@ type Trie struct { // NewEmptyTrie creates a trie with a nil root func NewEmptyTrie() *Trie { - return NewTrie(nil, nil) + return NewTrie(nil, db.NewEmptyMemoryDB()) } // NewTrie creates a trie with an existing root node -func NewTrie(root *Node, db DBGetter) *Trie { +func NewTrie(root *Node, db Database) *Trie { return &Trie{ root: root, childTries: make(map[common.Hash]*Trie), @@ -232,33 +238,33 @@ func entriesList(parent *Node, prefix []byte, list *[][2][]byte) { // where the keys are encoded in Little Endian. func (t *Trie) Entries() (keyValueMap map[string][]byte) { keyValueMap = make(map[string][]byte) - entries(t.root, nil, keyValueMap) + t.buildEntriesMap(t.root, nil, keyValueMap) return keyValueMap } -func entries(parent *Node, prefix []byte, kv map[string][]byte) { - if parent == nil { +func (t *Trie) buildEntriesMap(currentNode *Node, prefix []byte, kv map[string][]byte) { + if currentNode == nil { return } - if parent.Kind() == node.Leaf { - parentKey := parent.PartialKey - fullKeyNibbles := concatenateSlices(prefix, parentKey) - keyLE := string(codec.NibblesToKeyLE(fullKeyNibbles)) - kv[keyLE] = parent.StorageValue + if currentNode.Kind() == node.Leaf { + key := currentNode.PartialKey + fullKeyNibbles := concatenateSlices(prefix, key) + keyLE := codec.NibblesToKeyLE(fullKeyNibbles) + kv[string(keyLE)] = t.Get(keyLE) return } - branch := parent + branch := currentNode if branch.StorageValue != nil { fullKeyNibbles := concatenateSlices(prefix, branch.PartialKey) - keyLE := string(codec.NibblesToKeyLE(fullKeyNibbles)) - kv[keyLE] = branch.StorageValue + keyLE := codec.NibblesToKeyLE(fullKeyNibbles) + kv[string(keyLE)] = t.Get(keyLE) } for i, child := range branch.Children { childPrefix := concatenateSlices(prefix, branch.PartialKey, intToByteSlice(i)) - entries(child, childPrefix, kv) + t.buildEntriesMap(child, childPrefix, kv) } } @@ -375,7 +381,12 @@ func (t *Trie) insertKeyLE(keyLE, value []byte, isValueHashed := version.ShouldHashValue(value) if isValueHashed { - value = common.MustBlake2bHash(value).ToBytes() + hashedValue, err := common.Blake2bHash(value) + if err != nil { + return err + } + t.db.Put(hashedValue.ToBytes(), value) + value = hashedValue.ToBytes() } root, _, _, err := t.insert(t.root, nibblesKey, value, isValueHashed, pendingDeltas) diff --git a/lib/trie/trie_test.go b/lib/trie/trie_test.go index 2a58d7ebc3..1adc8993e3 100644 --- a/lib/trie/trie_test.go +++ b/lib/trie/trie_test.go @@ -12,6 +12,7 @@ import ( "github.com/ChainSafe/gossamer/internal/trie/node" "github.com/ChainSafe/gossamer/internal/trie/tracking" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/trie/db" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -33,6 +34,7 @@ func Test_NewEmptyTrie(t *testing.T) { expectedTrie := &Trie{ childTries: make(map[common.Hash]*Trie), deltas: tracking.New(), + db: db.NewEmptyMemoryDB(), } trie := NewEmptyTrie() assert.Equal(t, expectedTrie, trie) @@ -604,12 +606,12 @@ func Test_Trie_Entries(t *testing.T) { t.Parallel() root := &Node{ - PartialKey: []byte{0xa}, + PartialKey: []byte{0x0, 0xa}, StorageValue: []byte("root"), Descendants: 2, Children: padRightChildren([]*Node{ { // index 0 - PartialKey: []byte{2, 0xb}, + PartialKey: []byte{0xb}, StorageValue: []byte("leaf"), }, nil, @@ -626,7 +628,7 @@ func Test_Trie_Entries(t *testing.T) { expectedEntries := map[string][]byte{ string([]byte{0x0a}): []byte("root"), - string([]byte{0xa0, 0x2b}): []byte("leaf"), + string([]byte{0x0a, 0x0b}): []byte("leaf"), string([]byte{0x0a, 0x2b}): []byte("leaf"), } @@ -696,6 +698,7 @@ func Test_Trie_Entries(t *testing.T) { trie := Trie{ root: nil, childTries: make(map[common.Hash]*Trie), + db: db.NewEmptyMemoryDB(), } kv := map[string][]byte{ @@ -720,6 +723,7 @@ func Test_Trie_Entries(t *testing.T) { trie := Trie{ root: nil, childTries: make(map[common.Hash]*Trie), + db: db.NewEmptyMemoryDB(), } kv := map[string][]byte{ @@ -1075,6 +1079,7 @@ func Test_Trie_Put(t *testing.T) { t.Parallel() longValue := []byte("newvaluewithmorethan32byteslength") + longValueHash := common.MustBlake2bHash(longValue).ToBytes() testCases := map[string]struct { trie Trie @@ -1127,6 +1132,7 @@ func Test_Trie_Put(t *testing.T) { PartialKey: []byte{1, 2, 0, 5}, StorageValue: []byte{1}, }, + db: db.NewEmptyMemoryDB(), }, stateVersion: V1, key: []byte{0x12, 0x16}, @@ -1148,13 +1154,18 @@ func Test_Trie_Put(t *testing.T) { }, { PartialKey: []byte{6}, - StorageValue: common.MustBlake2bHash(longValue).ToBytes(), + StorageValue: longValueHash, HashedValue: true, Generation: 1, Dirty: true, }, }), }, + db: func() Database { + db := db.NewEmptyMemoryDB() + db.Put(longValueHash, longValue) + return db + }(), }, }, } @@ -1723,6 +1734,7 @@ func Test_LoadFromMap(t *testing.T) { expectedTrie: Trie{ childTries: map[common.Hash]*Trie{}, deltas: newDeltas(), + db: db.NewEmptyMemoryDB(), }, }, "empty_data": { @@ -1730,6 +1742,7 @@ func Test_LoadFromMap(t *testing.T) { expectedTrie: Trie{ childTries: map[common.Hash]*Trie{}, deltas: newDeltas(), + db: db.NewEmptyMemoryDB(), }, }, "bad_key": { @@ -1763,6 +1776,7 @@ func Test_LoadFromMap(t *testing.T) { }, childTries: map[common.Hash]*Trie{}, deltas: newDeltas(), + db: db.NewEmptyMemoryDB(), }, }, "load_key_values": { @@ -1793,6 +1807,7 @@ func Test_LoadFromMap(t *testing.T) { }, childTries: map[common.Hash]*Trie{}, deltas: newDeltas(), + db: db.NewEmptyMemoryDB(), }, }, } From 40fccfff9240f0d7fa3f6d3a9e580fd85fe54cf4 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 15 Aug 2023 21:36:01 -0300 Subject: [PATCH 011/128] Check memory db put value error --- lib/trie/trie.go | 7 ++++++- lib/trie/trie_test.go | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/trie/trie.go b/lib/trie/trie.go index a5749e6123..c21f6d45da 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -385,7 +385,12 @@ func (t *Trie) insertKeyLE(keyLE, value []byte, if err != nil { return err } - t.db.Put(hashedValue.ToBytes(), value) + + // Add original value in memory db + err = t.db.Put(hashedValue.ToBytes(), value) + if err != nil { + return err + } value = hashedValue.ToBytes() } diff --git a/lib/trie/trie_test.go b/lib/trie/trie_test.go index 1adc8993e3..6ca47ce25a 100644 --- a/lib/trie/trie_test.go +++ b/lib/trie/trie_test.go @@ -628,7 +628,7 @@ func Test_Trie_Entries(t *testing.T) { expectedEntries := map[string][]byte{ string([]byte{0x0a}): []byte("root"), - string([]byte{0x0a, 0x0b}): []byte("leaf"), + string([]byte{0x0a, 0xb}): []byte("leaf"), string([]byte{0x0a, 0x2b}): []byte("leaf"), } From 59107ce45e0c4504f3a36408d73cbdac61f954e1 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 15 Aug 2023 21:39:31 -0300 Subject: [PATCH 012/128] Add memory db lock --- lib/trie/db/db.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/trie/db/db.go b/lib/trie/db/db.go index cdbd3027c7..e5d38ddc2c 100644 --- a/lib/trie/db/db.go +++ b/lib/trie/db/db.go @@ -4,13 +4,14 @@ package db import ( "fmt" + "sync" "github.com/ChainSafe/gossamer/lib/common" ) type MemoryDB struct { data map[common.Hash][]byte - // TODO: add lock + l sync.RWMutex } func NewEmptyMemoryDB() *MemoryDB { @@ -44,6 +45,9 @@ func (mdb *MemoryDB) Get(key []byte) ([]byte, error) { var hash common.Hash copy(hash[:], key) + mdb.l.RLock() + defer mdb.l.RUnlock() + if value, found := mdb.data[hash]; found { return value, nil } @@ -59,6 +63,9 @@ func (mdb *MemoryDB) Put(key []byte, value []byte) error { var hash common.Hash copy(hash[:], key) + mdb.l.Lock() + defer mdb.l.Unlock() + mdb.data[hash] = value return nil } From ce73de232915c360d2e952e5294a099fb9f4e743 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 15 Aug 2023 21:48:51 -0300 Subject: [PATCH 013/128] Fix linter --- lib/runtime/wasmer/exports_test.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/runtime/wasmer/exports_test.go b/lib/runtime/wasmer/exports_test.go index 0be7186f70..a72800bcf1 100644 --- a/lib/runtime/wasmer/exports_test.go +++ b/lib/runtime/wasmer/exports_test.go @@ -627,7 +627,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1(t *testing.T) { } func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock3784(t *testing.T) { - gossTrie3783 := newTrieFromPairs(t, "../test_data/kusama/block3783.out", trie.V0) + gossTrie3783 := newTrieFromPairs(t, "../test_data/kusama/block3783.out") expectedRoot := common.MustHexToHash("0x948338bc0976aee78879d559a1f42385407e5a481b05a91d2a9386aa7507e7a0") require.Equal(t, expectedRoot, gossTrie3783.MustHash()) @@ -673,7 +673,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock3784(t *testing.T) { } func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock901442(t *testing.T) { - ksmTrie901441 := newTrieFromPairs(t, "../test_data/kusama/block901441.out", trie.V0) + ksmTrie901441 := newTrieFromPairs(t, "../test_data/kusama/block901441.out") expectedRoot := common.MustHexToHash("0x3a2ef7ee032f5810160bb8f3ffe3e3377bb6f2769ee9f79a5425973347acd504") require.Equal(t, expectedRoot, ksmTrie901441.MustHash()) @@ -719,7 +719,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock901442(t *testing.T) { } func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1377831(t *testing.T) { - ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block1377830.out", trie.V0) + ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block1377830.out") expectedRoot := common.MustHexToHash("0xe4de6fecda9e9e35f937d159665cf984bc1a68048b6c78912de0aeb6bd7f7e99") require.Equal(t, expectedRoot, ksmTrie.MustHash()) @@ -765,7 +765,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1377831(t *testing.T) { } func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1482003(t *testing.T) { - ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block1482002.out", trie.V0) + ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block1482002.out") expectedRoot := common.MustHexToHash("0x09f9ca28df0560c2291aa16b56e15e07d1e1927088f51356d522722aa90ca7cb") require.Equal(t, expectedRoot, ksmTrie.MustHash()) @@ -813,7 +813,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1482003(t *testing.T) { func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock4939774(t *testing.T) { t.Skip("skip for now as block4939773 is too large") - ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block4939773.out", trie.V0) + ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block4939773.out") expectedRoot := common.MustHexToHash("0xc45748e6e8632b44fc32b04cc4380098a9584cbd63ffbc59adce189574fc36fe") require.Equal(t, expectedRoot, ksmTrie.MustHash()) @@ -856,7 +856,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock4939774(t *testing.T) { } func TestInstance_ExecuteBlock_PolkadotBlock1089328(t *testing.T) { - dotTrie := newTrieFromPairs(t, "../test_data/polkadot/block1089327.json", trie.V0) + dotTrie := newTrieFromPairs(t, "../test_data/polkadot/block1089327.json") expectedRoot := common.MustHexToHash("0x87ed9ebe7fb645d3b5b0255cc16e78ed022d9fbb52486105436e15a74557535b") require.Equal(t, expectedRoot, dotTrie.MustHash()) @@ -983,7 +983,7 @@ func TestInstance_PaymentQueryInfo(t *testing.T) { } } -func newTrieFromPairs(t *testing.T, filename string, version trie.Version) *trie.Trie { +func newTrieFromPairs(t *testing.T, filename string) *trie.Trie { data, err := os.ReadFile(filename) require.NoError(t, err) @@ -998,7 +998,8 @@ func newTrieFromPairs(t *testing.T, filename string, version trie.Version) *trie entries[pairArr[0].(string)] = pairArr[1].(string) } - tr, err := trie.LoadFromMap(entries, version) + // We need to use V0 for all tests using this function + tr, err := trie.LoadFromMap(entries, trie.V0) require.NoError(t, err) return &tr } From e0dd69154fa73be08a7a9b52b291bc6788fd40e5 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 15 Aug 2023 22:05:48 -0300 Subject: [PATCH 014/128] Deepsource --- dot/state/interfaces.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dot/state/interfaces.go b/dot/state/interfaces.go index 7c193e3af6..913a057504 100644 --- a/dot/state/interfaces.go +++ b/dot/state/interfaces.go @@ -29,7 +29,7 @@ type GetPutter interface { Putter } -// GetNewBatcher has methods to get values and create a +// Batcher has methods to get values and create a // new batch. type Batcher interface { Getter From 7e1606491fbf53ef610f1fdd13ba8c9d782c4aaa Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 15 Aug 2023 22:08:41 -0300 Subject: [PATCH 015/128] Add default version --- cmd/gossamer/commands/import_state.go | 4 +--- lib/trie/version.go | 3 +++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/gossamer/commands/import_state.go b/cmd/gossamer/commands/import_state.go index 9f3f118b7b..0141271af7 100644 --- a/cmd/gossamer/commands/import_state.go +++ b/cmd/gossamer/commands/import_state.go @@ -15,6 +15,7 @@ import ( func init() { ImportStateCmd.Flags().String("chain", "", "Chain id used to load default configuration for specified chain") ImportStateCmd.Flags().String("state-file", "", "Path to JSON file consisting of key-value pairs") + ImportStateCmd.Flags().String("state-version", trie.DefaultStateVersion.String(), "State version v0 or v1") ImportStateCmd.Flags().String("header-file", "", "Path to JSON file of block header corresponding to the given state") ImportStateCmd.Flags().Uint64("first-slot", 0, "The first BABE slot of the network") } @@ -59,9 +60,6 @@ func execImportState(cmd *cobra.Command) error { if err != nil { return fmt.Errorf("failed to get state-version: %s", err) } - if stateVersionFlag == "" { - return fmt.Errorf("state-version must be specified") - } stateVersion, err := trie.ParseVersion(stateVersionFlag) if err != nil { return fmt.Errorf("failed to parse state-version: %s", err) diff --git a/lib/trie/version.go b/lib/trie/version.go index 16974e36fe..c3b5aa78cb 100644 --- a/lib/trie/version.go +++ b/lib/trie/version.go @@ -26,6 +26,9 @@ const ( V1 ) +// See https://github.com/paritytech/substrate/blob/5e76587825b9a9d52d8cb02ba38828adf606157b/primitives/storage/src/lib.rs#L435-L439 +const DefaultStateVersion = V1 + func (v Version) String() string { switch v { case V0: From 61f0d8dacb28f53ae76ec66293c43b20e8ea0aa0 Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 16 Aug 2023 10:26:31 -0300 Subject: [PATCH 016/128] Fix doc --- lib/trie/version.go | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/trie/version.go b/lib/trie/version.go index c3b5aa78cb..ffc782b507 100644 --- a/lib/trie/version.go +++ b/lib/trie/version.go @@ -26,6 +26,7 @@ const ( V1 ) +// DefaultStateVersion sets the state version we should use as default // See https://github.com/paritytech/substrate/blob/5e76587825b9a9d52d8cb02ba38828adf606157b/primitives/storage/src/lib.rs#L435-L439 const DefaultStateVersion = V1 From 797401790a4edd2fe569d9d306f72280d54fe437 Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 16 Aug 2023 10:41:03 -0300 Subject: [PATCH 017/128] Add test to cover case for #2329 --- lib/trie/proof/proof_test.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/lib/trie/proof/proof_test.go b/lib/trie/proof/proof_test.go index f808c999b4..f47b797ed2 100644 --- a/lib/trie/proof/proof_test.go +++ b/lib/trie/proof/proof_test.go @@ -99,3 +99,37 @@ func TestParachainHeaderStateProof(t *testing.T) { require.NoError(t, err) } + +func TestTrieProof(t *testing.T) { + key, err := hex.DecodeString("f0c365c3cf59d671eb72da0e7a4113c49f1f0515f462cdcf84e0f1d6045dfcbb") + if err != nil { + panic(err) + } + root, err := hex.DecodeString("dc4887669c2a6b3462e9557aa3105a66a02b6ec3b21784613de78c95dc3cbbe0") + if err != nil { + panic(err) + } + proof1, err := hex.DecodeString("80fffd8028b54b9a0a90d41b7941c43e6a0597d5914e3b62bdcb244851b9fc806c28ea2480d5ba6d50586692888b0c2f5b3c3fc345eb3a2405996f025ed37982ca396f5ed580bd281c12f20f06077bffd56b2f8b6431ee6c9fd11fed9c22db86cea849aeff2280afa1e1b5ce72ea1675e5e69be85e98fbfb660691a76fee9229f758a75315f2bc80aafc60caa3519d4b861e6b8da226266a15060e2071bba4184e194da61dfb208e809d3f6ae8f655009551de95ae1ef863f6771522fd5c0475a50ff53c5c8169b5888024a760a8f6c27928ae9e2fed9968bc5f6e17c3ae647398d8a615e5b2bb4b425f8085a0da830399f25fca4b653de654ffd3c92be39f3ae4f54e7c504961b5bd00cf80c2d44d371e5fc1f50227d7491ad65ad049630361cefb4ab1844831237609f08380c644938921d14ae611f3a90991af8b7f5bdb8fa361ee2c646c849bca90f491e6806e729ad43a591cd1321762582782bbe4ed193c6f583ec76013126f7f786e376280509bb016f2887d12137e73d26d7ddcd7f9c8ff458147cb9d309494655fe68de180009f8697d760fbe020564b07f407e6aad58ba9451b3d2d88b3ee03e12db7c47480952dcc0804e1120508a1753f1de4aa5b7481026a3320df8b48e918f0cecbaed3803360bf948fddc403d345064082e8393d7a1aad7a19081f6d02d94358f242b86c") + if err != nil { + panic(err) + } + proof2, err := hex.DecodeString("9ec365c3cf59d671eb72da0e7a4113c41002505f0e7b9012096b41c4eb3aaf947f6ea429080000685f0f1f0515f462cdcf84e0f1d6045dfcbb20865c4a2b7f010000") + if err != nil { + panic(err) + } + proof3, err := hex.DecodeString("8005088076c66e2871b4fe037d112ebffb3bfc8bd83a4ec26047f58ee2df7be4e9ebe3d680c1638f702aaa71e4b78cc8538ecae03e827bb494cc54279606b201ec071a5e24806d2a1e6d5236e1e13c5a5c84831f5f5383f97eba32df6f9faf80e32cf2f129bc") + if err != nil { + panic(err) + } + + proof := [][]byte{proof1, proof2, proof3} + proofDB, err := db.NewMemoryDBFromProof(proof) + + require.NoError(t, err) + + trie, err := buildTrie(proof, root, proofDB) + require.NoError(t, err) + value := trie.Get(key) + + require.Equal(t, []byte{0x86, 0x5c, 0x4a, 0x2b, 0x7f, 0x1, 0x0, 0x0}, value) +} From 494a85b75f3d9bd07b7bb451af8fb65f9dda347a Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 16 Aug 2023 10:47:56 -0300 Subject: [PATCH 018/128] Fix linter --- lib/trie/proof/proof_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/trie/proof/proof_test.go b/lib/trie/proof/proof_test.go index f47b797ed2..be8517b71e 100644 --- a/lib/trie/proof/proof_test.go +++ b/lib/trie/proof/proof_test.go @@ -109,15 +109,15 @@ func TestTrieProof(t *testing.T) { if err != nil { panic(err) } - proof1, err := hex.DecodeString("80fffd8028b54b9a0a90d41b7941c43e6a0597d5914e3b62bdcb244851b9fc806c28ea2480d5ba6d50586692888b0c2f5b3c3fc345eb3a2405996f025ed37982ca396f5ed580bd281c12f20f06077bffd56b2f8b6431ee6c9fd11fed9c22db86cea849aeff2280afa1e1b5ce72ea1675e5e69be85e98fbfb660691a76fee9229f758a75315f2bc80aafc60caa3519d4b861e6b8da226266a15060e2071bba4184e194da61dfb208e809d3f6ae8f655009551de95ae1ef863f6771522fd5c0475a50ff53c5c8169b5888024a760a8f6c27928ae9e2fed9968bc5f6e17c3ae647398d8a615e5b2bb4b425f8085a0da830399f25fca4b653de654ffd3c92be39f3ae4f54e7c504961b5bd00cf80c2d44d371e5fc1f50227d7491ad65ad049630361cefb4ab1844831237609f08380c644938921d14ae611f3a90991af8b7f5bdb8fa361ee2c646c849bca90f491e6806e729ad43a591cd1321762582782bbe4ed193c6f583ec76013126f7f786e376280509bb016f2887d12137e73d26d7ddcd7f9c8ff458147cb9d309494655fe68de180009f8697d760fbe020564b07f407e6aad58ba9451b3d2d88b3ee03e12db7c47480952dcc0804e1120508a1753f1de4aa5b7481026a3320df8b48e918f0cecbaed3803360bf948fddc403d345064082e8393d7a1aad7a19081f6d02d94358f242b86c") + proof1, err := hex.DecodeString("80fffd8028b54b9a0a90d41b7941c43e6a0597d5914e3b62bdcb244851b9fc806c28ea2480d5ba6d50586692888b0c2f5b3c3fc345eb3a2405996f025ed37982ca396f5ed580bd281c12f20f06077bffd56b2f8b6431ee6c9fd11fed9c22db86cea849aeff2280afa1e1b5ce72ea1675e5e69be85e98fbfb660691a76fee9229f758a75315f2bc80aafc60caa3519d4b861e6b8da226266a15060e2071bba4184e194da61dfb208e809d3f6ae8f655009551de95ae1ef863f6771522fd5c0475a50ff53c5c8169b5888024a760a8f6c27928ae9e2fed9968bc5f6e17c3ae647398d8a615e5b2bb4b425f8085a0da830399f25fca4b653de654ffd3c92be39f3ae4f54e7c504961b5bd00cf80c2d44d371e5fc1f50227d7491ad65ad049630361cefb4ab1844831237609f08380c644938921d14ae611f3a90991af8b7f5bdb8fa361ee2c646c849bca90f491e6806e729ad43a591cd1321762582782bbe4ed193c6f583ec76013126f7f786e376280509bb016f2887d12137e73d26d7ddcd7f9c8ff458147cb9d309494655fe68de180009f8697d760fbe020564b07f407e6aad58ba9451b3d2d88b3ee03e12db7c47480952dcc0804e1120508a1753f1de4aa5b7481026a3320df8b48e918f0cecbaed3803360bf948fddc403d345064082e8393d7a1aad7a19081f6d02d94358f242b86c") //nolint:lll if err != nil { panic(err) } - proof2, err := hex.DecodeString("9ec365c3cf59d671eb72da0e7a4113c41002505f0e7b9012096b41c4eb3aaf947f6ea429080000685f0f1f0515f462cdcf84e0f1d6045dfcbb20865c4a2b7f010000") + proof2, err := hex.DecodeString("9ec365c3cf59d671eb72da0e7a4113c41002505f0e7b9012096b41c4eb3aaf947f6ea429080000685f0f1f0515f462cdcf84e0f1d6045dfcbb20865c4a2b7f010000") //nolint:lll if err != nil { panic(err) } - proof3, err := hex.DecodeString("8005088076c66e2871b4fe037d112ebffb3bfc8bd83a4ec26047f58ee2df7be4e9ebe3d680c1638f702aaa71e4b78cc8538ecae03e827bb494cc54279606b201ec071a5e24806d2a1e6d5236e1e13c5a5c84831f5f5383f97eba32df6f9faf80e32cf2f129bc") + proof3, err := hex.DecodeString("8005088076c66e2871b4fe037d112ebffb3bfc8bd83a4ec26047f58ee2df7be4e9ebe3d680c1638f702aaa71e4b78cc8538ecae03e827bb494cc54279606b201ec071a5e24806d2a1e6d5236e1e13c5a5c84831f5f5383f97eba32df6f9faf80e32cf2f129bc") //nolint:lll if err != nil { panic(err) } From 2703b93cea4ed4b7eeff047ebf90ef6aacee6fd4 Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 10 Aug 2023 12:03:16 -0300 Subject: [PATCH 019/128] chore(trie): version trie functions --- dot/core/helpers_test.go | 4 +- dot/rpc/modules/childstate_test.go | 10 ++--- dot/state/base_test.go | 2 +- dot/state/storage_notify_test.go | 7 +-- dot/state/storage_test.go | 10 ++--- dot/state/test_helpers.go | 2 +- lib/runtime/interfaces.go | 2 +- lib/runtime/storage/trie.go | 4 +- lib/runtime/storage/trie_test.go | 22 +++++----- lib/runtime/wasmer/exports_test.go | 16 +++---- lib/runtime/wasmer/helpers.go | 5 ++- lib/runtime/wasmer/imports.go | 8 ++-- lib/runtime/wasmer/imports_test.go | 68 ++++++++++++++--------------- lib/runtime/wazero/imports.go | 12 ++--- lib/runtime/wazero/imports_test.go | 68 ++++++++++++++--------------- lib/runtime/wazero/instance_test.go | 16 +++---- lib/trie/child_storage.go | 4 +- lib/trie/database_test.go | 4 +- lib/trie/helpers_test.go | 2 +- lib/trie/mem_test.go | 4 +- lib/trie/proof/generate_test.go | 3 +- lib/trie/proof/proof_test.go | 3 +- lib/trie/trie.go | 2 +- lib/trie/trie_endtoend_test.go | 44 +++++++++---------- lib/trie/trie_test.go | 4 +- lib/trie/version.go | 26 +++++++++-- lib/trie/version_test.go | 2 +- 27 files changed, 189 insertions(+), 165 deletions(-) diff --git a/dot/core/helpers_test.go b/dot/core/helpers_test.go index f3802c850a..737ab367fc 100644 --- a/dot/core/helpers_test.go +++ b/dot/core/helpers_test.go @@ -108,9 +108,9 @@ func createTestService(t *testing.T, genesisFilePath string, cfgRuntime, err := wazero_runtime.NewRuntimeFromGenesis(rtCfg) require.NoError(t, err) - cfgRuntime.Context.Storage.Put(aliceBalanceKey, encodedAccountInfo) + cfgRuntime.Context.Storage.Put(aliceBalanceKey, encodedAccountInfo, trie.V0) // this key is System.UpgradedToDualRefCount -> set to true since all accounts have been upgraded to v0.9 format - cfgRuntime.Context.Storage.Put(common.UpgradedToDualRefKey, []byte{1}) + cfgRuntime.Context.Storage.Put(common.UpgradedToDualRefKey, []byte{1}, trie.V0) cfgBlockState.StoreRuntime(cfgBlockState.BestBlockHash(), cfgRuntime) diff --git a/dot/rpc/modules/childstate_test.go b/dot/rpc/modules/childstate_test.go index c84df630ce..533e6cb57e 100644 --- a/dot/rpc/modules/childstate_test.go +++ b/dot/rpc/modules/childstate_test.go @@ -24,13 +24,13 @@ func createTestTrieState(t *testing.T) (*trie.Trie, common.Hash) { _, genesisTrie, _ := newWestendLocalGenesisWithTrieAndHeader(t) tr := rtstorage.NewTrieState(&genesisTrie) - tr.Put([]byte(":first_key"), []byte(":value1")) - tr.Put([]byte(":second_key"), []byte(":second_value")) + tr.Put([]byte(":first_key"), []byte(":value1"), trie.V0) + tr.Put([]byte(":second_key"), []byte(":second_value"), trie.V0) childTr := trie.NewEmptyTrie() - childTr.Put([]byte(":child_first"), []byte(":child_first_value")) - childTr.Put([]byte(":child_second"), []byte(":child_second_value")) - childTr.Put([]byte(":another_child"), []byte("value")) + childTr.Put([]byte(":child_first"), []byte(":child_first_value"), trie.V0) + childTr.Put([]byte(":child_second"), []byte(":child_second_value"), trie.V0) + childTr.Put([]byte(":another_child"), []byte("value"), trie.V0) err := tr.SetChild([]byte(":child_storage_key"), childTr) require.NoError(t, err) diff --git a/dot/state/base_test.go b/dot/state/base_test.go index a6bedcde74..1286373e33 100644 --- a/dot/state/base_test.go +++ b/dot/state/base_test.go @@ -23,7 +23,7 @@ func TestTrie_StoreAndLoadFromDB(t *testing.T) { for keyString, value := range kv { key := []byte(keyString) - tt.Put(key, value) + tt.Put(key, value, trie.V0) } err := tt.WriteDirty(db) diff --git a/dot/state/storage_notify_test.go b/dot/state/storage_notify_test.go index 4751e6137f..1f2f0da551 100644 --- a/dot/state/storage_notify_test.go +++ b/dot/state/storage_notify_test.go @@ -13,6 +13,7 @@ import ( "github.com/ChainSafe/chaindb" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/trie" "github.com/dgraph-io/badger/v4/pb" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" @@ -49,7 +50,7 @@ func TestStorageState_RegisterStorageObserver(t *testing.T) { ss.RegisterStorageObserver(mockobs) defer ss.UnregisterStorageObserver(mockobs) - ts.Put([]byte("mackcom"), []byte("wuz here")) + ts.Put([]byte("mackcom"), []byte("wuz here"), trie.V0) err = ss.StoreTrie(ts, nil) require.NoError(t, err) @@ -86,7 +87,7 @@ func TestStorageState_RegisterStorageObserver_Multi(t *testing.T) { key1 := []byte("key1") value1 := []byte("value1") - ts.Put(key1, value1) + ts.Put(key1, value1, trie.V0) err = ss.StoreTrie(ts, nil) require.NoError(t, err) @@ -125,7 +126,7 @@ func TestStorageState_RegisterStorageObserver_Multi_Filter(t *testing.T) { ss.RegisterStorageObserver(mockobs) } - ts.Put(key1, value1) + ts.Put(key1, value1, trie.V0) err = ss.StoreTrie(ts, nil) require.NoError(t, err) diff --git a/dot/state/storage_test.go b/dot/state/storage_test.go index 1af0d3c768..fd0e597f0a 100644 --- a/dot/state/storage_test.go +++ b/dot/state/storage_test.go @@ -56,7 +56,7 @@ func TestStorage_GetStorageByBlockHash(t *testing.T) { key := []byte("testkey") value := []byte("testvalue") - ts.Put(key, value) + ts.Put(key, value, trie.V0) root, err := ts.Root() require.NoError(t, err) @@ -88,7 +88,7 @@ func TestStorage_TrieState(t *testing.T) { storage := newTestStorageState(t) ts, err := storage.TrieState(&trie.EmptyHash) require.NoError(t, err) - ts.Put([]byte("noot"), []byte("washere")) + ts.Put([]byte("noot"), []byte("washere"), trie.V0) root, err := ts.Root() require.NoError(t, err) @@ -120,7 +120,7 @@ func TestStorage_LoadFromDB(t *testing.T) { } for _, kv := range trieKV { - ts.Put(kv.key, kv.value) + ts.Put(kv.key, kv.value, trie.V0) } root, err := ts.Root() @@ -157,7 +157,7 @@ func TestStorage_StoreTrie_NotSyncing(t *testing.T) { key := []byte("testkey") value := []byte("testvalue") - ts.Put(key, value) + ts.Put(key, value, trie.V0) err = storage.StoreTrie(ts, nil) require.NoError(t, err) @@ -188,7 +188,7 @@ func TestGetStorageChildAndGetStorageFromChild(t *testing.T) { } testChildTrie := trie.NewTrie(trieRoot, dbGetter) - testChildTrie.Put([]byte("keyInsidechild"), []byte("voila")) + testChildTrie.Put([]byte("keyInsidechild"), []byte("voila"), trie.V0) err = genTrie.SetChild([]byte("keyToChild"), testChildTrie) require.NoError(t, err) diff --git a/dot/state/test_helpers.go b/dot/state/test_helpers.go index 82c2d60858..56255b3198 100644 --- a/dot/state/test_helpers.go +++ b/dot/state/test_helpers.go @@ -244,7 +244,7 @@ func generateBlockWithRandomTrie(t *testing.T, serv *Service, rand := time.Now().UnixNano() key := []byte("testKey" + fmt.Sprint(rand)) value := []byte("testValue" + fmt.Sprint(rand)) - err = trieState.Put(key, value) + err = trieState.Put(key, value, trie.V0) require.NoError(t, err) trieStateRoot, err := trieState.Root() diff --git a/lib/runtime/interfaces.go b/lib/runtime/interfaces.go index 2f813a21f6..9a46e5a650 100644 --- a/lib/runtime/interfaces.go +++ b/lib/runtime/interfaces.go @@ -11,7 +11,7 @@ import ( // Storage runtime interface. type Storage interface { - Put(key []byte, value []byte) (err error) + Put(key []byte, value []byte, version trie.Version) (err error) Get(key []byte) []byte Root() (common.Hash, error) SetChild(keyToChild []byte, child *trie.Trie) error diff --git a/lib/runtime/storage/trie.go b/lib/runtime/storage/trie.go index b88e8fb56a..0c6f1a9619 100644 --- a/lib/runtime/storage/trie.go +++ b/lib/runtime/storage/trie.go @@ -70,10 +70,10 @@ func (s *TrieState) RollbackStorageTransaction() { } // Put puts a key-value pair in the trie -func (s *TrieState) Put(key, value []byte) (err error) { +func (s *TrieState) Put(key, value []byte, version trie.Version) (err error) { s.lock.Lock() defer s.lock.Unlock() - return s.t.Put(key, value) + return s.t.Put(key, value, version) } // Get gets a value from the trie diff --git a/lib/runtime/storage/trie_test.go b/lib/runtime/storage/trie_test.go index 01e0e2f732..73e528db89 100644 --- a/lib/runtime/storage/trie_test.go +++ b/lib/runtime/storage/trie_test.go @@ -27,7 +27,7 @@ var testCases = []string{ func TestTrieState_SetGet(t *testing.T) { testFunc := func(ts *TrieState) { for _, tc := range testCases { - ts.Put([]byte(tc), []byte(tc)) + ts.Put([]byte(tc), []byte(tc), trie.V0) } for _, tc := range testCases { @@ -43,7 +43,7 @@ func TestTrieState_SetGet(t *testing.T) { func TestTrieState_Delete(t *testing.T) { testFunc := func(ts *TrieState) { for _, tc := range testCases { - ts.Put([]byte(tc), []byte(tc)) + ts.Put([]byte(tc), []byte(tc), trie.V0) } ts.Delete([]byte(testCases[0])) @@ -58,7 +58,7 @@ func TestTrieState_Delete(t *testing.T) { func TestTrieState_Root(t *testing.T) { testFunc := func(ts *TrieState) { for _, tc := range testCases { - ts.Put([]byte(tc), []byte(tc)) + ts.Put([]byte(tc), []byte(tc), trie.V0) } expected := ts.MustRoot() @@ -79,7 +79,7 @@ func TestTrieState_ClearPrefix(t *testing.T) { } for i, key := range keys { - ts.Put([]byte(key), []byte{byte(i)}) + ts.Put([]byte(key), []byte{byte(i)}, trie.V0) } ts.ClearPrefix([]byte("noo")) @@ -105,7 +105,7 @@ func TestTrieState_ClearPrefixInChild(t *testing.T) { } for i, key := range keys { - child.Put([]byte(key), []byte{byte(i)}) + child.Put([]byte(key), []byte{byte(i)}, trie.V0) } keyToChild := []byte("keytochild") @@ -131,7 +131,7 @@ func TestTrieState_NextKey(t *testing.T) { ts := &TrieState{t: trie.NewEmptyTrie()} for _, tc := range testCases { - ts.Put([]byte(tc), []byte(tc)) + ts.Put([]byte(tc), []byte(tc), trie.V0) } sort.Slice(testCases, func(i, j int) bool { @@ -152,12 +152,12 @@ func TestTrieState_CommitStorageTransaction(t *testing.T) { ts := &TrieState{t: trie.NewEmptyTrie()} for _, tc := range testCases { - ts.Put([]byte(tc), []byte(tc)) + ts.Put([]byte(tc), []byte(tc), trie.V0) } ts.BeginStorageTransaction() testValue := []byte("noot") - ts.Put([]byte(testCases[0]), testValue) + ts.Put([]byte(testCases[0]), testValue, trie.V0) ts.CommitStorageTransaction() val := ts.Get([]byte(testCases[0])) @@ -168,12 +168,12 @@ func TestTrieState_RollbackStorageTransaction(t *testing.T) { ts := &TrieState{t: trie.NewEmptyTrie()} for _, tc := range testCases { - ts.Put([]byte(tc), []byte(tc)) + ts.Put([]byte(tc), []byte(tc), trie.V0) } ts.BeginStorageTransaction() testValue := []byte("noot") - ts.Put([]byte(testCases[0]), testValue) + ts.Put([]byte(testCases[0]), testValue, trie.V0) ts.RollbackStorageTransaction() val := ts.Get([]byte(testCases[0])) @@ -191,7 +191,7 @@ func TestTrieState_DeleteChildLimit(t *testing.T) { } for i, key := range keys { - child.Put([]byte(key), []byte{byte(i)}) + child.Put([]byte(key), []byte{byte(i)}, trie.V0) } keyToChild := []byte("keytochild") diff --git a/lib/runtime/wasmer/exports_test.go b/lib/runtime/wasmer/exports_test.go index 7a9f937a6e..b02e3f1ece 100644 --- a/lib/runtime/wasmer/exports_test.go +++ b/lib/runtime/wasmer/exports_test.go @@ -204,9 +204,9 @@ func TestWestendRuntime_ValidateTransaction(t *testing.T) { encBal, err := scale.Marshal(accInfo) require.NoError(t, err) - rt.ctx.Storage.Put(aliceBalanceKey, encBal) + rt.ctx.Storage.Put(aliceBalanceKey, encBal, trie.V0) // this key is System.UpgradedToDualRefCount -> set to true since all accounts have been upgraded to v0.9 format - rt.ctx.Storage.Put(common.UpgradedToDualRefKey, []byte{1}) + rt.ctx.Storage.Put(common.UpgradedToDualRefKey, []byte{1}, trie.V0) genesisHeader := &types.Header{ Number: 0, @@ -237,7 +237,7 @@ func TestInstance_GrandpaAuthorities_NodeRuntime(t *testing.T) { require.NoError(t, err) key := common.MustHexToBytes(genesis.GrandpaAuthoritiesKeyHex) - tt.Put(key, value) + tt.Put(key, value, trie.V0) rt := NewTestInstanceWithTrie(t, runtime.WESTEND_RUNTIME_v0929, tt) @@ -265,7 +265,7 @@ func TestInstance_GrandpaAuthorities_PolkadotRuntime(t *testing.T) { require.NoError(t, err) key := common.MustHexToBytes(genesis.GrandpaAuthoritiesKeyHex) - tt.Put(key, value) + tt.Put(key, value, trie.V0) rt := NewTestInstanceWithTrie(t, runtime.POLKADOT_RUNTIME_v0929, tt) @@ -312,13 +312,13 @@ func TestInstance_BabeGenerateKeyOwnershipProof(t *testing.T) { randomnessValue, err := common.HexToHash("0x01") require.NoError(t, err) key := common.MustHexToBytes(genesis.BABERandomnessKeyHex) - tt.Put(key, randomnessValue[:]) + tt.Put(key, randomnessValue[:], trie.V0) authorityValue, err := common.HexToBytes("0x08eea1eabcac7d2c8a6459b7322cf997874482bfc3d2ec7a80888a3a7d714103640100000000000000b64994460e59b30364cad3c92e3df6052f9b0ebbb8f88460c194dc5794d6d7170100000000000000") //nolint:lll require.NoError(t, err) key = common.MustHexToBytes(genesis.BABEAuthoritiesKeyHex) - tt.Put(key, authorityValue) + tt.Put(key, authorityValue, trie.V0) rt := NewTestInstanceWithTrie(t, testCase.targetRuntime, tt) @@ -399,13 +399,13 @@ func TestInstance_BabeConfiguration_WestendRuntime_WithAuthorities(t *testing.T) randomnessValue, err := common.HexToHash("0x01") require.NoError(t, err) key := common.MustHexToBytes(genesis.BABERandomnessKeyHex) - tt.Put(key, randomnessValue[:]) + tt.Put(key, randomnessValue[:], trie.V0) authorityValue, err := common.HexToBytes("0x08eea1eabcac7d2c8a6459b7322cf997874482bfc3d2ec7a80888a3a7d714103640100000000000000b64994460e59b30364cad3c92e3df6052f9b0ebbb8f88460c194dc5794d6d7170100000000000000") //nolint:lll require.NoError(t, err) key = common.MustHexToBytes(genesis.BABEAuthoritiesKeyHex) - tt.Put(key, authorityValue) + tt.Put(key, authorityValue, trie.V0) rt := NewTestInstanceWithTrie(t, runtime.WESTEND_RUNTIME_v0929, tt) diff --git a/lib/runtime/wasmer/helpers.go b/lib/runtime/wasmer/helpers.go index 1cfbf372de..9c95f220e2 100644 --- a/lib/runtime/wasmer/helpers.go +++ b/lib/runtime/wasmer/helpers.go @@ -12,6 +12,7 @@ import ( "github.com/ChainSafe/gossamer/lib/common/types" "github.com/ChainSafe/gossamer/lib/runtime" + "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/pkg/scale" "github.com/wasmerio/go-ext-wasm/wasmer" ) @@ -208,7 +209,7 @@ func toWasmMemoryFixedSizeOptional(context wasmer.InstanceContext, data []byte) return toWasmMemory(context, encodedOptionalFixedSize) } -func storageAppend(storage runtime.Storage, key, valueToAppend []byte) (err error) { +func storageAppend(storage runtime.Storage, key, valueToAppend []byte, version trie.Version) (err error) { // this function assumes the item in storage is a SCALE encoded array of items // the valueToAppend is a new item, so it appends the item and increases the length prefix by 1 currentValue := storage.Get(key) @@ -259,7 +260,7 @@ func storageAppend(storage runtime.Storage, key, valueToAppend []byte) (err erro } } - err = storage.Put(key, value) + err = storage.Put(key, value, version) if err != nil { return fmt.Errorf("putting key and value in storage: %w", err) } diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index 02ad2826a2..a0a0568a57 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -820,7 +820,7 @@ func ext_trie_blake2_256_root_version_1(context unsafe.Pointer, dataSpan C.int64 } for _, kv := range kvs { - err := t.Put(kv.Key, kv.Value) + err := t.Put(kv.Key, kv.Value, trie.V0) if err != nil { logger.Errorf("failed putting key 0x%x and value 0x%x into trie: %s", kv.Key, kv.Value, err) @@ -873,7 +873,7 @@ func ext_trie_blake2_256_ordered_root_version_1(context unsafe.Pointer, dataSpan "put key=0x%x and value=0x%x", key, value) - err = t.Put(key, value) + err = t.Put(key, value, trie.V0) if err != nil { logger.Errorf("failed putting key 0x%x and value 0x%x into trie: %s", key, value, err) @@ -1916,7 +1916,7 @@ func ext_storage_append_version_1(context unsafe.Pointer, keySpan, valueSpan C.i cp := make([]byte, len(valueAppend)) copy(cp, valueAppend) - err := storageAppend(storage, key, cp) + err := storageAppend(storage, key, cp, trie.V0) if err != nil { logger.Errorf("failed appending to storage: %s", err) } @@ -2156,7 +2156,7 @@ func ext_storage_set_version_1(context unsafe.Pointer, keySpan, valueSpan C.int6 logger.Debugf( "key 0x%x has value 0x%x", key, value) - err := storage.Put(key, cp) + err := storage.Put(key, cp, trie.V0) panicOnError(err) } diff --git a/lib/runtime/wasmer/imports_test.go b/lib/runtime/wasmer/imports_test.go index 7f060d496c..a936da44af 100644 --- a/lib/runtime/wasmer/imports_test.go +++ b/lib/runtime/wasmer/imports_test.go @@ -207,7 +207,7 @@ func Test_ext_storage_clear_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") - inst.ctx.Storage.Put(testkey, []byte{1}) + inst.ctx.Storage.Put(testkey, []byte{1}, trie.V0) enc, err := scale.Marshal(testkey) require.NoError(t, err) @@ -393,10 +393,10 @@ func Test_ext_storage_clear_prefix_version_1_hostAPI(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("static") - inst.ctx.Storage.Put(testkey, []byte("Inverse")) + inst.ctx.Storage.Put(testkey, []byte("Inverse"), trie.V0) testkey2 := []byte("even-keeled") - inst.ctx.Storage.Put(testkey2, []byte("Future-proofed")) + inst.ctx.Storage.Put(testkey2, []byte("Future-proofed"), trie.V0) enc, err := scale.Marshal(testkey[:3]) require.NoError(t, err) @@ -416,10 +416,10 @@ func Test_ext_storage_clear_prefix_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") - inst.ctx.Storage.Put(testkey, []byte{1}) + inst.ctx.Storage.Put(testkey, []byte{1}, trie.V0) testkey2 := []byte("spaghet") - inst.ctx.Storage.Put(testkey2, []byte{2}) + inst.ctx.Storage.Put(testkey2, []byte{2}, trie.V0) enc, err := scale.Marshal(testkey[:3]) require.NoError(t, err) @@ -439,20 +439,20 @@ func Test_ext_storage_clear_prefix_version_2(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") - inst.ctx.Storage.Put(testkey, []byte{1}) + inst.ctx.Storage.Put(testkey, []byte{1}, trie.V0) testkey2 := []byte("noot1") - inst.ctx.Storage.Put(testkey2, []byte{1}) + inst.ctx.Storage.Put(testkey2, []byte{1}, trie.V0) testkey3 := []byte("noot2") - inst.ctx.Storage.Put(testkey3, []byte{1}) + inst.ctx.Storage.Put(testkey3, []byte{1}, trie.V0) testkey4 := []byte("noot3") - inst.ctx.Storage.Put(testkey4, []byte{1}) + inst.ctx.Storage.Put(testkey4, []byte{1}, trie.V0) testkey5 := []byte("spaghet") testValue5 := []byte{2} - inst.ctx.Storage.Put(testkey5, testValue5) + inst.ctx.Storage.Put(testkey5, testValue5, trie.V0) enc, err := scale.Marshal(testkey[:3]) require.NoError(t, err) @@ -513,7 +513,7 @@ func Test_ext_storage_get_version_1(t *testing.T) { testkey := []byte("noot") testvalue := []byte{1, 2} - inst.ctx.Storage.Put(testkey, testvalue) + inst.ctx.Storage.Put(testkey, testvalue, trie.V0) enc, err := scale.Marshal(testkey) require.NoError(t, err) @@ -559,7 +559,7 @@ func Test_ext_storage_exists_version_1(t *testing.T) { instance := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) if testCase.value != nil { - instance.ctx.Storage.Put(testCase.key, testCase.value) + instance.ctx.Storage.Put(testCase.key, testCase.value, trie.V0) } encodedKey, err := scale.Marshal(testCase.key) @@ -582,10 +582,10 @@ func Test_ext_storage_next_key_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") - inst.ctx.Storage.Put(testkey, []byte{1}) + inst.ctx.Storage.Put(testkey, []byte{1}, trie.V0) nextkey := []byte("oot") - inst.ctx.Storage.Put(nextkey, []byte{1}) + inst.ctx.Storage.Put(nextkey, []byte{1}, trie.V0) enc, err := scale.Marshal(testkey) require.NoError(t, err) @@ -606,7 +606,7 @@ func Test_ext_storage_read_version_1(t *testing.T) { testkey := []byte("noot") testvalue := []byte("washere") - inst.ctx.Storage.Put(testkey, testvalue) + inst.ctx.Storage.Put(testkey, testvalue, trie.V0) testoffset := uint32(2) testBufferSize := uint32(100) @@ -635,7 +635,7 @@ func Test_ext_storage_read_version_1_again(t *testing.T) { testkey := []byte("noot") testvalue := []byte("_was_here_") - inst.ctx.Storage.Put(testkey, testvalue) + inst.ctx.Storage.Put(testkey, testvalue, trie.V0) testoffset := uint32(8) testBufferSize := uint32(5) @@ -665,7 +665,7 @@ func Test_ext_storage_read_version_1_OffsetLargerThanValue(t *testing.T) { testkey := []byte("noot") testvalue := []byte("washere") - inst.ctx.Storage.Put(testkey, testvalue) + inst.ctx.Storage.Put(testkey, testvalue, trie.V0) testoffset := uint32(len(testvalue)) testBufferSize := uint32(8) @@ -1574,8 +1574,8 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_all(t *testing. inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`)) - tr.Put([]byte(`key1`), []byte(`value1`)) + tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) + tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) err := inst.ctx.Storage.SetChild(testChildKey, tr) require.NoError(t, err) @@ -1608,8 +1608,8 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_1(t *testing.T) inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`)) - tr.Put([]byte(`key1`), []byte(`value1`)) + tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) + tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) err := inst.ctx.Storage.SetChild(testChildKey, tr) require.NoError(t, err) @@ -1642,8 +1642,8 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_none(t *testing inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`)) - tr.Put([]byte(`key1`), []byte(`value1`)) + tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) + tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) err := inst.ctx.Storage.SetChild(testChildKey, tr) require.NoError(t, err) @@ -1673,9 +1673,9 @@ func Test_ext_default_child_storage_storage_kill_version_3(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`)) - tr.Put([]byte(`key1`), []byte(`value1`)) - tr.Put([]byte(`key3`), []byte(`value3`)) + tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) + tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) + tr.Put([]byte(`key3`), []byte(`value3`), trie.V0) err := inst.ctx.Storage.SetChild(testChildKey, tr) require.NoError(t, err) @@ -1856,8 +1856,8 @@ func Test_ext_trie_blake2_256_root_version_1(t *testing.T) { require.NoError(t, err) tt := trie.NewEmptyTrie() - tt.Put([]byte("noot"), []byte("was")) - tt.Put([]byte("here"), []byte("??")) + tt.Put([]byte("noot"), []byte("was"), trie.V0) + tt.Put([]byte("here"), []byte("??"), trie.V0) expected := tt.MustHash() require.Equal(t, expected[:], hash) @@ -1875,17 +1875,17 @@ func Test_ext_trie_blake2_256_verify_proof_version_1(t *testing.T) { require.NoError(t, err) otherTrie := trie.NewEmptyTrie() - otherTrie.Put([]byte("simple"), []byte("cat")) + otherTrie.Put([]byte("simple"), []byte("cat"), trie.V0) otherHash, err := otherTrie.Hash() require.NoError(t, err) tr := trie.NewEmptyTrie() - tr.Put([]byte("do"), []byte("verb")) - tr.Put([]byte("domain"), []byte("website")) - tr.Put([]byte("other"), []byte("random")) - tr.Put([]byte("otherwise"), []byte("randomstuff")) - tr.Put([]byte("cat"), []byte("another animal")) + tr.Put([]byte("do"), []byte("verb"), trie.V0) + tr.Put([]byte("domain"), []byte("website"), trie.V0) + tr.Put([]byte("other"), []byte("random"), trie.V0) + tr.Put([]byte("otherwise"), []byte("randomstuff"), trie.V0) + tr.Put([]byte("cat"), []byte("another animal"), trie.V0) err = tr.WriteDirty(memdb) require.NoError(t, err) diff --git a/lib/runtime/wazero/imports.go b/lib/runtime/wazero/imports.go index 25db775147..32d2cbda8a 100644 --- a/lib/runtime/wazero/imports.go +++ b/lib/runtime/wazero/imports.go @@ -833,7 +833,7 @@ func ext_trie_blake2_256_root_version_1(ctx context.Context, m api.Module, dataS } for _, kv := range kvs { - err := t.Put(kv.Key, kv.Value) + err := t.Put(kv.Key, kv.Value, trie.V0) if err != nil { logger.Errorf("failed putting key 0x%x and value 0x%x into trie: %s", kv.Key, kv.Value, err) @@ -935,7 +935,7 @@ func ext_trie_blake2_256_ordered_root_version_1(ctx context.Context, m api.Modul "put key=0x%x and value=0x%x", key, value) - err = t.Put(key, value) + err = t.Put(key, value, trie.V0) if err != nil { logger.Errorf("failed putting key 0x%x and value 0x%x into trie: %s", key, value, err) @@ -2071,7 +2071,7 @@ func ext_offchain_http_request_add_header_version_1( return ptr } -func storageAppend(storage runtime.Storage, key, valueToAppend []byte) (err error) { +func storageAppend(storage runtime.Storage, key, valueToAppend []byte, version trie.Version) (err error) { // this function assumes the item in storage is a SCALE encoded array of items // the valueToAppend is a new item, so it appends the item and increases the length prefix by 1 currentValue := storage.Get(key) @@ -2122,7 +2122,7 @@ func storageAppend(storage runtime.Storage, key, valueToAppend []byte) (err erro } } - err = storage.Put(key, value) + err = storage.Put(key, value, version) if err != nil { return fmt.Errorf("putting key and value in storage: %w", err) } @@ -2146,7 +2146,7 @@ func ext_storage_append_version_1(ctx context.Context, m api.Module, keySpan, va cp := make([]byte, len(valueAppend)) copy(cp, valueAppend) - err := storageAppend(storage, key, cp) + err := storageAppend(storage, key, cp, trie.V0) if err != nil { logger.Errorf("failed appending to storage: %s", err) } @@ -2460,7 +2460,7 @@ func ext_storage_set_version_1(ctx context.Context, m api.Module, keySpan, value logger.Debugf( "key 0x%x has value 0x%x", key, value) - err := storage.Put(key, cp) + err := storage.Put(key, cp, trie.V0) if err != nil { panic(err) } diff --git a/lib/runtime/wazero/imports_test.go b/lib/runtime/wazero/imports_test.go index c9d95f0a89..815970dae5 100644 --- a/lib/runtime/wazero/imports_test.go +++ b/lib/runtime/wazero/imports_test.go @@ -562,8 +562,8 @@ func Test_ext_trie_blake2_256_root_version_1(t *testing.T) { require.NoError(t, err) tt := trie.NewEmptyTrie() - tt.Put([]byte("noot"), []byte("was")) - tt.Put([]byte("here"), []byte("??")) + tt.Put([]byte("noot"), []byte("was"), trie.V0) + tt.Put([]byte("here"), []byte("??"), trie.V0) expected := tt.MustHash() require.Equal(t, expected[:], hash) @@ -597,17 +597,17 @@ func Test_ext_trie_blake2_256_verify_proof_version_1(t *testing.T) { require.NoError(t, err) otherTrie := trie.NewEmptyTrie() - otherTrie.Put([]byte("simple"), []byte("cat")) + otherTrie.Put([]byte("simple"), []byte("cat"), trie.V0) otherHash, err := otherTrie.Hash() require.NoError(t, err) tr := trie.NewEmptyTrie() - tr.Put([]byte("do"), []byte("verb")) - tr.Put([]byte("domain"), []byte("website")) - tr.Put([]byte("other"), []byte("random")) - tr.Put([]byte("otherwise"), []byte("randomstuff")) - tr.Put([]byte("cat"), []byte("another animal")) + tr.Put([]byte("do"), []byte("verb"), trie.V0) + tr.Put([]byte("domain"), []byte("website"), trie.V0) + tr.Put([]byte("other"), []byte("random"), trie.V0) + tr.Put([]byte("otherwise"), []byte("randomstuff"), trie.V0) + tr.Put([]byte("cat"), []byte("another animal"), trie.V0) err = tr.WriteDirty(memdb) require.NoError(t, err) @@ -1023,8 +1023,8 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_all(t *testing. inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`)) - tr.Put([]byte(`key1`), []byte(`value1`)) + tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) + tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) err := inst.Context.Storage.SetChild(testChildKey, tr) require.NoError(t, err) @@ -1056,8 +1056,8 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_1(t *testing.T) inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`)) - tr.Put([]byte(`key1`), []byte(`value1`)) + tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) + tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) err := inst.Context.Storage.SetChild(testChildKey, tr) require.NoError(t, err) @@ -1089,8 +1089,8 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_none(t *testing inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`)) - tr.Put([]byte(`key1`), []byte(`value1`)) + tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) + tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) err := inst.Context.Storage.SetChild(testChildKey, tr) require.NoError(t, err) @@ -1119,9 +1119,9 @@ func Test_ext_default_child_storage_storage_kill_version_3(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`)) - tr.Put([]byte(`key1`), []byte(`value1`)) - tr.Put([]byte(`key3`), []byte(`value3`)) + tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) + tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) + tr.Put([]byte(`key3`), []byte(`value3`), trie.V0) err := inst.Context.Storage.SetChild(testChildKey, tr) require.NoError(t, err) @@ -1524,7 +1524,7 @@ func Test_ext_storage_clear_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") - inst.Context.Storage.Put(testkey, []byte{1}) + inst.Context.Storage.Put(testkey, []byte{1}, trie.V0) enc, err := scale.Marshal(testkey) require.NoError(t, err) @@ -1540,10 +1540,10 @@ func Test_ext_storage_clear_prefix_version_1_hostAPI(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("static") - inst.Context.Storage.Put(testkey, []byte("Inverse")) + inst.Context.Storage.Put(testkey, []byte("Inverse"), trie.V0) testkey2 := []byte("even-keeled") - inst.Context.Storage.Put(testkey2, []byte("Future-proofed")) + inst.Context.Storage.Put(testkey2, []byte("Future-proofed"), trie.V0) enc, err := scale.Marshal(testkey[:3]) require.NoError(t, err) @@ -1562,10 +1562,10 @@ func Test_ext_storage_clear_prefix_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") - inst.Context.Storage.Put(testkey, []byte{1}) + inst.Context.Storage.Put(testkey, []byte{1}, trie.V0) testkey2 := []byte("spaghet") - inst.Context.Storage.Put(testkey2, []byte{2}) + inst.Context.Storage.Put(testkey2, []byte{2}, trie.V0) enc, err := scale.Marshal(testkey[:3]) require.NoError(t, err) @@ -1584,20 +1584,20 @@ func Test_ext_storage_clear_prefix_version_2(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") - inst.Context.Storage.Put(testkey, []byte{1}) + inst.Context.Storage.Put(testkey, []byte{1}, trie.V0) testkey2 := []byte("noot1") - inst.Context.Storage.Put(testkey2, []byte{1}) + inst.Context.Storage.Put(testkey2, []byte{1}, trie.V0) testkey3 := []byte("noot2") - inst.Context.Storage.Put(testkey3, []byte{1}) + inst.Context.Storage.Put(testkey3, []byte{1}, trie.V0) testkey4 := []byte("noot3") - inst.Context.Storage.Put(testkey4, []byte{1}) + inst.Context.Storage.Put(testkey4, []byte{1}, trie.V0) testkey5 := []byte("spaghet") testValue5 := []byte{2} - inst.Context.Storage.Put(testkey5, testValue5) + inst.Context.Storage.Put(testkey5, testValue5, trie.V0) enc, err := scale.Marshal(testkey[:3]) require.NoError(t, err) @@ -1657,7 +1657,7 @@ func Test_ext_storage_get_version_1(t *testing.T) { testkey := []byte("noot") testvalue := []byte{1, 2} - inst.Context.Storage.Put(testkey, testvalue) + inst.Context.Storage.Put(testkey, testvalue, trie.V0) enc, err := scale.Marshal(testkey) require.NoError(t, err) @@ -1700,7 +1700,7 @@ func Test_ext_storage_exists_version_1(t *testing.T) { instance := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) if testCase.value != nil { - instance.Context.Storage.Put(testCase.key, testCase.value) + instance.Context.Storage.Put(testCase.key, testCase.value, trie.V0) } encodedKey, err := scale.Marshal(testCase.key) @@ -1722,10 +1722,10 @@ func Test_ext_storage_next_key_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") - inst.Context.Storage.Put(testkey, []byte{1}) + inst.Context.Storage.Put(testkey, []byte{1}, trie.V0) nextkey := []byte("oot") - inst.Context.Storage.Put(nextkey, []byte{1}) + inst.Context.Storage.Put(nextkey, []byte{1}, trie.V0) enc, err := scale.Marshal(testkey) require.NoError(t, err) @@ -1745,7 +1745,7 @@ func Test_ext_storage_read_version_1(t *testing.T) { testkey := []byte("noot") testvalue := []byte("washere") - inst.Context.Storage.Put(testkey, testvalue) + inst.Context.Storage.Put(testkey, testvalue, trie.V0) testoffset := uint32(2) testBufferSize := uint32(100) @@ -1773,7 +1773,7 @@ func Test_ext_storage_read_version_1_again(t *testing.T) { testkey := []byte("noot") testvalue := []byte("_was_here_") - inst.Context.Storage.Put(testkey, testvalue) + inst.Context.Storage.Put(testkey, testvalue, trie.V0) testoffset := uint32(8) testBufferSize := uint32(5) @@ -1802,7 +1802,7 @@ func Test_ext_storage_read_version_1_OffsetLargerThanValue(t *testing.T) { testkey := []byte("noot") testvalue := []byte("washere") - inst.Context.Storage.Put(testkey, testvalue) + inst.Context.Storage.Put(testkey, testvalue, trie.V0) testoffset := uint32(len(testvalue)) testBufferSize := uint32(8) diff --git a/lib/runtime/wazero/instance_test.go b/lib/runtime/wazero/instance_test.go index 204266e1c8..b54ce2151c 100644 --- a/lib/runtime/wazero/instance_test.go +++ b/lib/runtime/wazero/instance_test.go @@ -222,9 +222,9 @@ func TestWestendRuntime_ValidateTransaction(t *testing.T) { encBal, err := scale.Marshal(accInfo) require.NoError(t, err) - rt.Context.Storage.Put(aliceBalanceKey, encBal) + rt.Context.Storage.Put(aliceBalanceKey, encBal, trie.V0) // this key is System.UpgradedToDualRefCount -> set to true since all accounts have been upgraded to v0.9 format - rt.Context.Storage.Put(common.UpgradedToDualRefKey, []byte{1}) + rt.Context.Storage.Put(common.UpgradedToDualRefKey, []byte{1}, trie.V0) genesisHeader := &types.Header{ Number: 0, @@ -255,7 +255,7 @@ func TestInstance_GrandpaAuthorities_NodeRuntime(t *testing.T) { require.NoError(t, err) key := common.MustHexToBytes(genesis.GrandpaAuthoritiesKeyHex) - tt.Put(key, value) + tt.Put(key, value, trie.V0) rt := NewTestInstanceWithTrie(t, runtime.WESTEND_RUNTIME_v0929, tt) @@ -283,7 +283,7 @@ func TestInstance_GrandpaAuthorities_PolkadotRuntime(t *testing.T) { require.NoError(t, err) key := common.MustHexToBytes(genesis.GrandpaAuthoritiesKeyHex) - tt.Put(key, value) + tt.Put(key, value, trie.V0) rt := NewTestInstanceWithTrie(t, runtime.POLKADOT_RUNTIME_v0929, tt) @@ -326,13 +326,13 @@ func TestInstance_BabeGenerateKeyOwnershipProof(t *testing.T) { randomnessValue, err := common.HexToHash("0x01") require.NoError(t, err) key := common.MustHexToBytes(genesis.BABERandomnessKeyHex) - tt.Put(key, randomnessValue[:]) + tt.Put(key, randomnessValue[:], trie.V0) authorityValue, err := common.HexToBytes("0x08eea1eabcac7d2c8a6459b7322cf997874482bfc3d2ec7a80888a3a7d714103640100000000000000b64994460e59b30364cad3c92e3df6052f9b0ebbb8f88460c194dc5794d6d7170100000000000000") //nolint:lll require.NoError(t, err) key = common.MustHexToBytes(genesis.BABEAuthoritiesKeyHex) - tt.Put(key, authorityValue) + tt.Put(key, authorityValue, trie.V0) rt := NewTestInstanceWithTrie(t, testCase.targetRuntime, tt) @@ -409,13 +409,13 @@ func TestInstance_BabeConfiguration_WestendRuntime_WithAuthorities(t *testing.T) randomnessValue, err := common.HexToHash("0x01") require.NoError(t, err) key := common.MustHexToBytes(genesis.BABERandomnessKeyHex) - tt.Put(key, randomnessValue[:]) + tt.Put(key, randomnessValue[:], trie.V0) authorityValue, err := common.HexToBytes("0x08eea1eabcac7d2c8a6459b7322cf997874482bfc3d2ec7a80888a3a7d714103640100000000000000b64994460e59b30364cad3c92e3df6052f9b0ebbb8f88460c194dc5794d6d7170100000000000000") //nolint:lll require.NoError(t, err) key = common.MustHexToBytes(genesis.BABEAuthoritiesKeyHex) - tt.Put(key, authorityValue) + tt.Put(key, authorityValue, trie.V0) rt := NewTestInstanceWithTrie(t, runtime.WESTEND_RUNTIME_v0929, tt) diff --git a/lib/trie/child_storage.go b/lib/trie/child_storage.go index 78506eaa8c..444c713da0 100644 --- a/lib/trie/child_storage.go +++ b/lib/trie/child_storage.go @@ -28,7 +28,7 @@ func (t *Trie) SetChild(keyToChild []byte, child *Trie) error { copy(key, ChildStorageKeyPrefix) copy(key[len(ChildStorageKeyPrefix):], keyToChild) - err = t.Put(key, childHash.ToBytes()) + err = t.Put(key, childHash.ToBytes(), V0) if err != nil { return fmt.Errorf("putting child trie root hash %s in trie: %w", childHash, err) } @@ -63,7 +63,7 @@ func (t *Trie) PutIntoChild(keyToChild, key, value []byte) error { return err } - err = child.Put(key, value) + err = child.Put(key, value, V0) if err != nil { return fmt.Errorf("putting into child trie located at key 0x%x: %w", keyToChild, err) } diff --git a/lib/trie/database_test.go b/lib/trie/database_test.go index 2300adb415..0737e3703d 100644 --- a/lib/trie/database_test.go +++ b/lib/trie/database_test.go @@ -54,7 +54,7 @@ func Test_Trie_WriteDirty_Put(t *testing.T) { for keyString, value := range keyValues { key := []byte(keyString) - trie.Put(key, value) + trie.Put(key, value, V0) err := trie.WriteDirty(db) require.NoError(t, err) @@ -75,7 +75,7 @@ func Test_Trie_WriteDirty_Put(t *testing.T) { newValue := make([]byte, len(existingValue)) copy(newValue, existingValue) newValue = append(newValue, 99) - trie.Put(existingKey, newValue) + trie.Put(existingKey, newValue, V0) err = trie.WriteDirty(db) require.NoError(t, err) diff --git a/lib/trie/helpers_test.go b/lib/trie/helpers_test.go index cbae542394..b36385e098 100644 --- a/lib/trie/helpers_test.go +++ b/lib/trie/helpers_test.go @@ -93,7 +93,7 @@ func makeSeededTrie(t *testing.T, size int) ( for keyString, value := range keyValues { key := []byte(keyString) - trie.Put(key, value) + trie.Put(key, value, V0) } return trie, keyValues diff --git a/lib/trie/mem_test.go b/lib/trie/mem_test.go index 56096571bd..757da3185f 100644 --- a/lib/trie/mem_test.go +++ b/lib/trie/mem_test.go @@ -93,7 +93,7 @@ func populateTrieAtPrefix(trie *Trie, for keyString, value := range kv { key := append(prefix, []byte(keyString)...) //skipcq: CRT-D0001 - trie.Put(key, value) + trie.Put(key, value, V0) } } @@ -113,6 +113,6 @@ func mutateTrieLeavesAtPrefix(trie *Trie, } } - trie.Put(key, newValue) + trie.Put(key, newValue, V0) } } diff --git a/lib/trie/proof/generate_test.go b/lib/trie/proof/generate_test.go index 49f519b3c0..6517ed8bd0 100644 --- a/lib/trie/proof/generate_test.go +++ b/lib/trie/proof/generate_test.go @@ -828,6 +828,7 @@ func Test_lenCommonPrefix(t *testing.T) { // so the code is kept to this inefficient-looking append, // which is in the end quite performant still. func Benchmark_walkRoot(b *testing.B) { + stateVersion := trie.V0 trie := trie.NewEmptyTrie() // Build a deep trie. @@ -838,7 +839,7 @@ func Benchmark_walkRoot(b *testing.B) { const trieValueSize = 10 value := make([]byte, trieValueSize) - trie.Put(key, value) + trie.Put(key, value, stateVersion) } longestKeyLE := make([]byte, trieDepth) diff --git a/lib/trie/proof/proof_test.go b/lib/trie/proof/proof_test.go index 8f813b4f61..f808c999b4 100644 --- a/lib/trie/proof/proof_test.go +++ b/lib/trie/proof/proof_test.go @@ -16,6 +16,7 @@ import ( func Test_Generate_Verify(t *testing.T) { t.Parallel() + stateVersion := trie.V0 keys := []string{ "cat", @@ -29,7 +30,7 @@ func Test_Generate_Verify(t *testing.T) { for i, key := range keys { value := fmt.Sprintf("%x-%d", key, i) - trie.Put([]byte(key), []byte(value)) + trie.Put([]byte(key), []byte(value), stateVersion) } rootHash, err := trie.Hash() diff --git a/lib/trie/trie.go b/lib/trie/trie.go index ea2d1dd4ae..5f1067d5ba 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -355,7 +355,7 @@ func findNextKeyChild(children []*Node, startIndex byte, // Put inserts a value into the trie at the // key specified in little Endian format. -func (t *Trie) Put(keyLE, value []byte) (err error) { +func (t *Trie) Put(keyLE, value []byte, version Version) (err error) { pendingDeltas := tracking.New() defer func() { const success = true diff --git a/lib/trie/trie_endtoend_test.go b/lib/trie/trie_endtoend_test.go index 3ca189ebe5..a6d87e9360 100644 --- a/lib/trie/trie_endtoend_test.go +++ b/lib/trie/trie_endtoend_test.go @@ -40,7 +40,7 @@ func buildSmallTrie() *Trie { } for _, test := range tests { - trie.Put(test.key, test.value) + trie.Put(test.key, test.value, V0) } return trie @@ -50,7 +50,7 @@ func runTests(t *testing.T, trie *Trie, tests []keyValues) { for _, test := range tests { switch test.op { case put: - trie.Put(test.key, test.value) + trie.Put(test.key, test.value, V0) case get: val := trie.Get(test.key) assert.Equal(t, test.value, val) @@ -104,7 +104,7 @@ func TestPutAndGetOddKeyLengths(t *testing.T) { func Fuzz_Trie_PutAndGet_Single(f *testing.F) { f.Fuzz(func(t *testing.T, key, value []byte) { trie := NewEmptyTrie() - trie.Put(key, value) + trie.Put(key, value, V0) retrievedValue := trie.Get(key) assert.Equal(t, value, retrievedValue) }) @@ -119,7 +119,7 @@ func Test_Trie_PutAndGet_Multiple(t *testing.T) { keyValues := generateKeyValues(t, generator, numberOfKeyValuePairs) for keyString, value := range keyValues { key := []byte(keyString) - trie.Put(key, value) + trie.Put(key, value, V0) // Check value is inserted correctly. retrievedValue := trie.Get(key) @@ -296,7 +296,7 @@ func TestTrieDiff(t *testing.T) { } for _, test := range tests { - trie.Put(test.key, test.value) + trie.Put(test.key, test.value, V0) } newTrie := trie.Snapshot() @@ -312,7 +312,7 @@ func TestTrieDiff(t *testing.T) { } for _, test := range tests { - newTrie.Put(test.key, test.value) + newTrie.Put(test.key, test.value, V0) } deletedNodeHashes := newTrie.deltas.Deleted() @@ -349,7 +349,7 @@ func TestDelete(t *testing.T) { for keyString, value := range kv { key := []byte(keyString) - trie.Put(key, value) + trie.Put(key, value, V0) } dcTrie := trie.DeepCopy() @@ -432,7 +432,7 @@ func TestClearPrefix(t *testing.T) { trie := NewEmptyTrie() for _, test := range tests { - trie.Put(test.key, test.value) + trie.Put(test.key, test.value, V0) } dcTrie := trie.DeepCopy() @@ -517,7 +517,7 @@ func TestClearPrefix_Small(t *testing.T) { "other", } for _, key := range keys { - ssTrie.Put([]byte(key), []byte(key)) + ssTrie.Put([]byte(key), []byte(key), V0) } ssTrie.ClearPrefix([]byte("noo")) @@ -598,8 +598,8 @@ func TestTrie_ClearPrefixVsDelete(t *testing.T) { trieClearPrefix := NewEmptyTrie() for _, test := range testCase { - trieDelete.Put(test.key, test.value) - trieClearPrefix.Put(test.key, test.value) + trieDelete.Put(test.key, test.value, V0) + trieClearPrefix.Put(test.key, test.value, V0) } prefixedKeys := trieDelete.GetKeysWithPrefix(prefix) @@ -627,7 +627,7 @@ func TestSnapshot(t *testing.T) { expectedTrie := NewEmptyTrie() for _, test := range tests { - expectedTrie.Put(test.key, test.value) + expectedTrie.Put(test.key, test.value, V0) } // put all keys except first @@ -636,11 +636,11 @@ func TestSnapshot(t *testing.T) { if i == 0 { continue } - parentTrie.Put(test.key, test.value) + parentTrie.Put(test.key, test.value, V0) } newTrie := parentTrie.Snapshot() - newTrie.Put(tests[0].key, tests[0].value) + newTrie.Put(tests[0].key, tests[0].value, V0) require.Equal(t, expectedTrie.MustHash(), newTrie.MustHash()) require.NotEqual(t, parentTrie.MustHash(), newTrie.MustHash()) @@ -667,7 +667,7 @@ func Test_Trie_NextKey_Random(t *testing.T) { for _, key := range sortedKeys { value := []byte{1} - trie.Put(key, value) + trie.Put(key, value, V0) } for i, key := range sortedKeys { @@ -691,7 +691,7 @@ func Benchmark_Trie_Hash(b *testing.B) { trie := NewEmptyTrie() for keyString, value := range kv { key := []byte(keyString) - trie.Put(key, value) + trie.Put(key, value, V0) } b.StartTimer() @@ -732,7 +732,7 @@ func TestTrie_ConcurrentSnapshotWrites(t *testing.T) { switch op { case put: - expectedTries[i].Put(k, k) + expectedTries[i].Put(k, k, V0) case del: expectedTries[i].Delete(k) case clearPrefix: @@ -763,7 +763,7 @@ func TestTrie_ConcurrentSnapshotWrites(t *testing.T) { for _, operation := range operations { switch operation.op { case put: - trie.Put(operation.key, operation.key) + trie.Put(operation.key, operation.key, V0) case del: trie.Delete(operation.key) case clearPrefix: @@ -844,7 +844,7 @@ func TestTrie_ClearPrefixLimit(t *testing.T) { trieClearPrefix := NewEmptyTrie() for _, test := range testCase { - trieClearPrefix.Put(test.key, test.value) + trieClearPrefix.Put(test.key, test.value, V0) } num, allDeleted, err := trieClearPrefix.ClearPrefixLimit(prefix, uint32(lim)) @@ -950,7 +950,7 @@ func TestTrie_ClearPrefixLimitSnapshot(t *testing.T) { trieClearPrefix := NewEmptyTrie() for _, test := range testCase { - trieClearPrefix.Put(test.key, test.value) + trieClearPrefix.Put(test.key, test.value, V0) } dcTrie := trieClearPrefix.DeepCopy() @@ -1036,7 +1036,7 @@ func Test_encodeRoot_fuzz(t *testing.T) { kv := generateKeyValues(t, generator, kvSize) for keyString, value := range kv { key := []byte(keyString) - trie.Put(key, value) + trie.Put(key, value, V0) retrievedValue := trie.Get(key) assert.Equal(t, value, retrievedValue) @@ -1091,7 +1091,7 @@ func Test_Trie_Descendants_Fuzz(t *testing.T) { }) for _, key := range keys { - trie.Put(key, kv[string(key)]) + trie.Put(key, kv[string(key)], V0) } testDescendants(t, trie.root) diff --git a/lib/trie/trie_test.go b/lib/trie/trie_test.go index d484fe2be5..0ef35f9c6f 100644 --- a/lib/trie/trie_test.go +++ b/lib/trie/trie_test.go @@ -706,7 +706,7 @@ func Test_Trie_Entries(t *testing.T) { } for k, v := range kv { - trie.Put([]byte(k), v) + trie.Put([]byte(k), v, V0) } entries := trie.Entries() @@ -1100,7 +1100,7 @@ func Test_Trie_Put(t *testing.T) { t.Parallel() trie := testCase.trie - trie.Put(testCase.key, testCase.value) + trie.Put(testCase.key, testCase.value, V0) assert.Equal(t, testCase.expectedTrie, trie) }) diff --git a/lib/trie/version.go b/lib/trie/version.go index 23527890c8..16974e36fe 100644 --- a/lib/trie/version.go +++ b/lib/trie/version.go @@ -7,8 +7,12 @@ import ( "errors" "fmt" "strings" + + "github.com/ChainSafe/gossamer/lib/common" ) +const V1MaxValueSize = common.HashLength + // Version is the state trie version which dictates how a // Merkle root should be constructed. It is defined in // https://spec.polkadot.network/#defn-state-version @@ -18,13 +22,27 @@ const ( // V0 is the state trie version 0 where the values of the keys are // inserted into the trie directly. // TODO set to iota once CI passes - V0 Version = 1 + V0 Version = iota + V1 ) func (v Version) String() string { switch v { case V0: return "v0" + case V1: + return "v1" + default: + panic(fmt.Sprintf("unknown version %d", v)) + } +} + +func (v Version) ShouldHashValue(value []byte) bool { + switch v { + case V0: + return false + case V1: + return len(value) >= V1MaxValueSize default: panic(fmt.Sprintf("unknown version %d", v)) } @@ -37,8 +55,10 @@ func ParseVersion(s string) (version Version, err error) { switch { case strings.EqualFold(s, V0.String()): return V0, nil + case strings.EqualFold(s, V1.String()): + return V1, nil default: - return version, fmt.Errorf("%w: %q must be %s", - ErrParseVersion, s, V0) + return version, fmt.Errorf("%w: %q must be one of [%s, %s]", + ErrParseVersion, s, V0, V1) } } diff --git a/lib/trie/version_test.go b/lib/trie/version_test.go index ab2ac03ebe..0000a43099 100644 --- a/lib/trie/version_test.go +++ b/lib/trie/version_test.go @@ -65,7 +65,7 @@ func Test_ParseVersion(t *testing.T) { "invalid": { s: "xyz", errWrapped: ErrParseVersion, - errMessage: "parsing version failed: \"xyz\" must be v0", + errMessage: "parsing version failed: \"xyz\" must be one of [v0, v1]", }, } From 00d4231e35f8cb038181ffe0be725fbaadad119c Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 10 Aug 2023 13:11:40 -0300 Subject: [PATCH 020/128] Update tests --- dot/import.go | 3 +- lib/runtime/genesis.go | 3 +- lib/runtime/wasmer/exports_test.go | 5 ++-- lib/runtime/wazero/instance_test.go | 2 +- lib/trie/trie.go | 43 +++++++++++++++++++---------- lib/trie/trie_test.go | 17 +++++++----- 6 files changed, 47 insertions(+), 26 deletions(-) diff --git a/dot/import.go b/dot/import.go index c5d9c85f4e..5d0d9b313b 100644 --- a/dot/import.go +++ b/dot/import.go @@ -62,7 +62,8 @@ func newTrieFromPairs(filename string) (*trie.Trie, error) { entries[pairArr[0].(string)] = pairArr[1].(string) } - tr, err := trie.LoadFromMap(entries) + //TODO: revisit this to use the right trie version + tr, err := trie.LoadFromMap(entries, trie.V0) if err != nil { return nil, err } diff --git a/lib/runtime/genesis.go b/lib/runtime/genesis.go index 5912886bf1..b1547c3611 100644 --- a/lib/runtime/genesis.go +++ b/lib/runtime/genesis.go @@ -26,7 +26,8 @@ func NewTrieFromGenesis(gen genesis.Genesis) (tr trie.Trie, err error) { ErrGenesisTopNotFound, gen.Name) } - tr, err = trie.LoadFromMap(keyValues) + //TODO: check if we could get the trie version from genesis + tr, err = trie.LoadFromMap(keyValues, trie.V0) if err != nil { return tr, fmt.Errorf("loading genesis top key values into trie: %w", err) } diff --git a/lib/runtime/wasmer/exports_test.go b/lib/runtime/wasmer/exports_test.go index b02e3f1ece..367a03701a 100644 --- a/lib/runtime/wasmer/exports_test.go +++ b/lib/runtime/wasmer/exports_test.go @@ -998,7 +998,8 @@ func newTrieFromPairs(t *testing.T, filename string) *trie.Trie { entries[pairArr[0].(string)] = pairArr[1].(string) } - tr, err := trie.LoadFromMap(entries) + //TODO: revisit this to use the right trie version + tr, err := trie.LoadFromMap(entries, trie.V0) require.NoError(t, err) return &tr } @@ -1197,7 +1198,7 @@ func loadEntries(t *testing.T, filename string) *trie.Trie { err = json.Unmarshal(data, &entries) require.NoError(t, err) - tr, err := trie.LoadFromEntries(entries) + tr, err := trie.LoadFromEntries(entries, trie.V0) require.NoError(t, err) return tr } diff --git a/lib/runtime/wazero/instance_test.go b/lib/runtime/wazero/instance_test.go index b54ce2151c..1fc57eacad 100644 --- a/lib/runtime/wazero/instance_test.go +++ b/lib/runtime/wazero/instance_test.go @@ -962,7 +962,7 @@ func newTrieFromPairs(t *testing.T, filename string) *trie.Trie { entries[pairArr[0].(string)] = pairArr[1].(string) } - tr, err := trie.LoadFromMap(entries) + tr, err := trie.LoadFromMap(entries, trie.V0) require.NoError(t, err) return &tr } diff --git a/lib/trie/trie.go b/lib/trie/trie.go index 5f1067d5ba..025f6e619e 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -361,18 +361,25 @@ func (t *Trie) Put(keyLE, value []byte, version Version) (err error) { const success = true t.handleTrackedDeltas(success, pendingDeltas) }() - return t.insertKeyLE(keyLE, value, pendingDeltas) + return t.insertKeyLE(keyLE, value, pendingDeltas, version) } func (t *Trie) insertKeyLE(keyLE, value []byte, - pendingDeltas DeltaRecorder) (err error) { + pendingDeltas DeltaRecorder, version Version) (err error) { nibblesKey := codec.KeyLEToNibbles(keyLE) if value == nil { // Force nil value to be inserted to []byte{} since `nil` means there // is no value. value = []byte{} } - root, _, _, err := t.insert(t.root, nibblesKey, value, pendingDeltas) + + isValueHashed := version.ShouldHashValue(value) + if isValueHashed { + hashedValue := common.MustBlake2bHash(value) + copy(value, hashedValue[:]) + } + + root, _, _, err := t.insert(t.root, nibblesKey, value, isValueHashed, pendingDeltas) if err != nil { return err } @@ -383,7 +390,7 @@ func (t *Trie) insertKeyLE(keyLE, value []byte, // insert inserts a value in the trie at the key specified. // It may create one or more new nodes or update an existing node. func (t *Trie) insert(parent *Node, key, value []byte, - pendingDeltas DeltaRecorder) (newParent *Node, + isValueHashed bool, pendingDeltas DeltaRecorder) (newParent *Node, mutated bool, nodesCreated uint32, err error) { if parent == nil { mutated = true @@ -391,6 +398,7 @@ func (t *Trie) insert(parent *Node, key, value []byte, return &Node{ PartialKey: key, StorageValue: value, + HashedValue: isValueHashed, Generation: t.generation, Dirty: true, }, mutated, nodesCreated, nil @@ -398,7 +406,7 @@ func (t *Trie) insert(parent *Node, key, value []byte, if parent.Kind() == node.Branch { newParent, mutated, nodesCreated, err = t.insertInBranch( - parent, key, value, pendingDeltas) + parent, key, value, isValueHashed, pendingDeltas) if err != nil { // `insertInBranch` may call `insert` so do not wrap the // error since this may be a deep recursive call. @@ -408,7 +416,7 @@ func (t *Trie) insert(parent *Node, key, value []byte, } newParent, mutated, nodesCreated, err = t.insertInLeaf( - parent, key, value, pendingDeltas) + parent, key, value, isValueHashed, pendingDeltas) if err != nil { return nil, false, 0, fmt.Errorf("inserting in leaf: %w", err) } @@ -416,7 +424,7 @@ func (t *Trie) insert(parent *Node, key, value []byte, return newParent, mutated, nodesCreated, nil } -func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, +func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, isValueHashed bool, pendingDeltas DeltaRecorder) ( newParent *Node, mutated bool, nodesCreated uint32, err error) { if bytes.Equal(parentLeaf.PartialKey, key) { @@ -434,6 +442,7 @@ func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, } parentLeaf.StorageValue = value + parentLeaf.HashedValue = isValueHashed mutated = true return parentLeaf, mutated, nodesCreated, nil } @@ -453,6 +462,7 @@ func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, if len(key) == commonPrefixLength { // key is included in parent leaf key newBranchParent.StorageValue = value + newBranchParent.HashedValue = isValueHashed if len(key) < len(parentLeafKey) { // Move the current leaf parent as a child to the new branch. @@ -477,6 +487,7 @@ func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, if len(parentLeaf.PartialKey) == commonPrefixLength { // the key of the parent leaf is at this new branch newBranchParent.StorageValue = parentLeaf.StorageValue + newBranchParent.HashedValue = parentLeaf.HashedValue } else { // make the leaf a child of the new branch copySettings := node.DefaultCopySettings @@ -497,6 +508,7 @@ func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, newBranchParent.Children[childIndex] = &Node{ PartialKey: key[commonPrefixLength+1:], StorageValue: value, + HashedValue: isValueHashed, Generation: t.generation, Dirty: true, } @@ -506,7 +518,7 @@ func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, return newBranchParent, mutated, nodesCreated, nil } -func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, +func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, isValueHashed bool, pendingDeltas DeltaRecorder) ( newParent *Node, mutated bool, nodesCreated uint32, err error) { copySettings := node.DefaultCopySettings @@ -521,6 +533,7 @@ func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, return nil, false, 0, fmt.Errorf("preparing branch for mutation: %w", err) } parentBranch.StorageValue = value + parentBranch.HashedValue = isValueHashed mutated = true return parentBranch, mutated, 0, nil } @@ -536,6 +549,7 @@ func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, child = &Node{ PartialKey: remainingKey, StorageValue: value, + HashedValue: isValueHashed, Generation: t.generation, Dirty: true, } @@ -550,7 +564,7 @@ func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, return parentBranch, mutated, nodesCreated, nil } - child, mutated, nodesCreated, err = t.insert(child, remainingKey, value, pendingDeltas) + child, mutated, nodesCreated, err = t.insert(child, remainingKey, value, isValueHashed, pendingDeltas) if err != nil { // do not wrap error since `insert` may call `insertInBranch` recursively return nil, false, 0, err @@ -595,12 +609,13 @@ func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, if len(key) <= commonPrefixLength { newParentBranch.StorageValue = value + newParentBranch.HashedValue = isValueHashed } else { childIndex := key[commonPrefixLength] remainingKey := key[commonPrefixLength+1:] var additionalNodesCreated uint32 newParentBranch.Children[childIndex], _, additionalNodesCreated, err = t.insert( - nil, remainingKey, value, pendingDeltas) + nil, remainingKey, value, isValueHashed, pendingDeltas) if err != nil { // do not wrap error since `insert` may call `insertInBranch` recursively return nil, false, 0, err @@ -616,7 +631,7 @@ func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, // LoadFromMap loads the given data mapping of key to value into a new empty trie. // The keys are in hexadecimal little Endian encoding and the values // are hexadecimal encoded. -func LoadFromMap(data map[string]string) (trie Trie, err error) { +func LoadFromMap(data map[string]string, version Version) (trie Trie, err error) { trie = *NewEmptyTrie() pendingDeltas := tracking.New() @@ -635,7 +650,7 @@ func LoadFromMap(data map[string]string) (trie Trie, err error) { return Trie{}, fmt.Errorf("cannot convert value hex to bytes: %w", err) } - err = trie.insertKeyLE(keyLEBytes, valueBytes, pendingDeltas) + err = trie.insertKeyLE(keyLEBytes, valueBytes, pendingDeltas, version) if err != nil { return Trie{}, fmt.Errorf("inserting key value pair in trie: %w", err) } @@ -647,7 +662,7 @@ func LoadFromMap(data map[string]string) (trie Trie, err error) { // LoadFromEntries loads the given slice of key values into a new empty trie. // The keys are in hexadecimal little Endian encoding and the values // are hexadecimal encoded. -func LoadFromEntries(entries [][2][]byte) (trie *Trie, err error) { +func LoadFromEntries(entries [][2][]byte, version Version) (trie *Trie, err error) { trie = NewEmptyTrie() pendingDeltas := tracking.New() @@ -658,7 +673,7 @@ func LoadFromEntries(entries [][2][]byte) (trie *Trie, err error) { for _, keyValue := range entries { keyLE := keyValue[0] value := keyValue[1] - err := trie.insertKeyLE(keyLE, value, pendingDeltas) + err := trie.insertKeyLE(keyLE, value, pendingDeltas, version) if err != nil { return nil, err } diff --git a/lib/trie/trie_test.go b/lib/trie/trie_test.go index 0ef35f9c6f..2a36c65826 100644 --- a/lib/trie/trie_test.go +++ b/lib/trie/trie_test.go @@ -1115,6 +1115,7 @@ func Test_Trie_insert(t *testing.T) { parent *Node key []byte value []byte + isValueHashed bool pendingDeltas DeltaRecorder newNode *Node mutated bool @@ -1327,7 +1328,7 @@ func Test_Trie_insert(t *testing.T) { expectedTrie := *trie.DeepCopy() newNode, mutated, nodesCreated, err := trie.insert( - testCase.parent, testCase.key, testCase.value, + testCase.parent, testCase.key, testCase.value, testCase.isValueHashed, testCase.pendingDeltas) require.NoError(t, err) @@ -1347,6 +1348,7 @@ func Test_Trie_insertInBranch(t *testing.T) { parent *Node key []byte value []byte + isValueHashed bool pendingDeltas DeltaRecorder newNode *Node mutated bool @@ -1626,7 +1628,7 @@ func Test_Trie_insertInBranch(t *testing.T) { trie := new(Trie) newNode, mutated, nodesCreated, err := trie.insertInBranch( - testCase.parent, testCase.key, testCase.value, + testCase.parent, testCase.key, testCase.value, testCase.isValueHashed, testCase.pendingDeltas) assert.ErrorIs(t, err, testCase.errSentinel) @@ -1646,10 +1648,11 @@ func Test_LoadFromMap(t *testing.T) { t.Parallel() testCases := map[string]struct { - data map[string]string - expectedTrie Trie - errWrapped error - errMessage string + data map[string]string + stateTrieVersion Version + expectedTrie Trie + errWrapped error + errMessage string }{ "nil_data": { expectedTrie: Trie{ @@ -1734,7 +1737,7 @@ func Test_LoadFromMap(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - trie, err := LoadFromMap(testCase.data) + trie, err := LoadFromMap(testCase.data, testCase.stateTrieVersion) assert.ErrorIs(t, err, testCase.errWrapped) if testCase.errWrapped != nil { From 447ccf58c695af508467d2c67184608ec191c1d4 Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 10 Aug 2023 14:29:00 -0300 Subject: [PATCH 021/128] Fix lint errors --- dot/core/service_integration_test.go | 4 ++-- dot/node_integration_test.go | 2 +- dot/rpc/modules/author_integration_test.go | 2 +- dot/rpc/modules/childstate_integration_test.go | 10 +++++----- dot/rpc/modules/dev_integration_test.go | 3 ++- dot/rpc/modules/state_integration_test.go | 5 +++-- dot/rpc/modules/system_integration_test.go | 2 +- dot/state/service_integration_test.go | 2 +- dot/utils_integration_test.go | 4 ++-- 9 files changed, 18 insertions(+), 16 deletions(-) diff --git a/dot/core/service_integration_test.go b/dot/core/service_integration_test.go index 1da5793dbf..e44dc01292 100644 --- a/dot/core/service_integration_test.go +++ b/dot/core/service_integration_test.go @@ -584,7 +584,7 @@ func createBlockUsingNewRuntime(t *testing.T, bestBlockHash common.Hash, newRunt testRuntime, err := os.ReadFile(newRuntimePath) require.NoError(t, err) - trieState.Put(common.CodeKey, testRuntime) + trieState.Put(common.CodeKey, testRuntime, trie.V0) primaryDigestData := types.NewBabePrimaryPreDigest(0, uint64(1), [32]byte{}, [64]byte{}) digest := types.NewDigest() @@ -673,7 +673,7 @@ func TestService_HandleRuntimeChangesAfterCodeSubstitutes(t *testing.T) { ts, err = s.storageState.TrieState(nil) require.NoError(t, err) - ts.Put(common.CodeKey, testRuntime) + ts.Put(common.CodeKey, testRuntime, trie.V0) rtUpdateBhash := newBlock.Header.Hash() // update runtime for new block diff --git a/dot/node_integration_test.go b/dot/node_integration_test.go index 283437a2af..64f3097b97 100644 --- a/dot/node_integration_test.go +++ b/dot/node_integration_test.go @@ -378,7 +378,7 @@ func TestInitNode_LoadStorageRoot(t *testing.T) { node, err := NewNode(config, ks) require.NoError(t, err) - expected, err := trie.LoadFromMap(gen.GenesisFields().Raw["top"]) + expected, err := trie.LoadFromMap(gen.GenesisFields().Raw["top"], trie.V0) require.NoError(t, err) expectedRoot, err := expected.Hash() diff --git a/dot/rpc/modules/author_integration_test.go b/dot/rpc/modules/author_integration_test.go index 9501e4288d..f1da5fe3fe 100644 --- a/dot/rpc/modules/author_integration_test.go +++ b/dot/rpc/modules/author_integration_test.go @@ -69,7 +69,7 @@ func useInstanceFromRuntimeV0929(t *testing.T, rtStorage *storage.TrieState) (in bytes, err := os.ReadFile(testRuntimeFilePath) require.NoError(t, err) - rtStorage.Put(common.CodeKey, bytes) + rtStorage.Put(common.CodeKey, bytes, trie.V0) cfg := wazero_runtime.Config{ Role: 0, diff --git a/dot/rpc/modules/childstate_integration_test.go b/dot/rpc/modules/childstate_integration_test.go index a46e8245b5..85b00f9ec6 100644 --- a/dot/rpc/modules/childstate_integration_test.go +++ b/dot/rpc/modules/childstate_integration_test.go @@ -244,13 +244,13 @@ func setupChildStateStorage(t *testing.T) (*ChildStateModule, common.Hash) { tr, err := st.Storage.TrieState(nil) require.NoError(t, err) - tr.Put([]byte(":first_key"), []byte(":value1")) - tr.Put([]byte(":second_key"), []byte(":second_value")) + tr.Put([]byte(":first_key"), []byte(":value1"), trie.V0) + tr.Put([]byte(":second_key"), []byte(":second_value"), trie.V0) childTr := trie.NewEmptyTrie() - childTr.Put([]byte(":child_first"), []byte(":child_first_value")) - childTr.Put([]byte(":child_second"), []byte(":child_second_value")) - childTr.Put([]byte(":another_child"), []byte("value")) + childTr.Put([]byte(":child_first"), []byte(":child_first_value"), trie.V0) + childTr.Put([]byte(":child_second"), []byte(":child_second_value"), trie.V0) + childTr.Put([]byte(":another_child"), []byte("value"), trie.V0) err = tr.SetChild([]byte(":child_storage_key"), childTr) require.NoError(t, err) diff --git a/dot/rpc/modules/dev_integration_test.go b/dot/rpc/modules/dev_integration_test.go index c779dbca5b..ebda0e5773 100644 --- a/dot/rpc/modules/dev_integration_test.go +++ b/dot/rpc/modules/dev_integration_test.go @@ -68,7 +68,8 @@ func newBABEService(t *testing.T) *babe.Service { "0d4a9e054df4e01000000000000001cbd2d43530a44705ad088af313e18f80b53ef16b36177cd4b77b846f2a5f07c010000"+ "00000000004603307f855321776922daeea21ee31720388d097cdaac66f05a6f8462b317570100000000000000be1d9d59d"+ "e1283380100550a7b024501cb62d6cc40e3db35fcc5cf341814986e01000000000000001206960f920a23f7f4c43cc9081"+ - "ec2ed0721f31a9bef2c10fd7602e16e08a32c0100000000000000")) + "ec2ed0721f31a9bef2c10fd7602e16e08a32c0100000000000000"), + trie.V0) cfg := &babe.ServiceConfig{ BlockState: bs, diff --git a/dot/rpc/modules/state_integration_test.go b/dot/rpc/modules/state_integration_test.go index 31ba029260..7fac6cccb8 100644 --- a/dot/rpc/modules/state_integration_test.go +++ b/dot/rpc/modules/state_integration_test.go @@ -17,6 +17,7 @@ import ( "github.com/ChainSafe/gossamer/dot/rpc/modules/mocks" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/trie" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -551,8 +552,8 @@ func setupStateModule(t *testing.T) (*StateModule, *common.Hash, *common.Hash) { ts, err := chain.Storage.TrieState(nil) require.NoError(t, err) - ts.Put([]byte(`:key2`), []byte(`value2`)) - ts.Put([]byte(`:key1`), []byte(`value1`)) + ts.Put([]byte(`:key2`), []byte(`value2`), trie.V0) + ts.Put([]byte(`:key1`), []byte(`value1`), trie.V0) ts.SetChildStorage([]byte(`:child1`), []byte(`:key1`), []byte(`:childValue1`)) sr1, err := ts.Root() diff --git a/dot/rpc/modules/system_integration_test.go b/dot/rpc/modules/system_integration_test.go index 453ac8bb72..b2a7dc745d 100644 --- a/dot/rpc/modules/system_integration_test.go +++ b/dot/rpc/modules/system_integration_test.go @@ -315,7 +315,7 @@ func setupSystemModule(t *testing.T) *SystemModule { aliceAcctEncoded, err := scale.Marshal(aliceAcctInfo) require.NoError(t, err) - ts.Put(aliceAcctStoKey, aliceAcctEncoded) + ts.Put(aliceAcctStoKey, aliceAcctEncoded, trie.V0) err = chain.Storage.StoreTrie(ts, nil) require.NoError(t, err) diff --git a/dot/state/service_integration_test.go b/dot/state/service_integration_test.go index 6ee2f9aa81..9dbc25c3d4 100644 --- a/dot/state/service_integration_test.go +++ b/dot/state/service_integration_test.go @@ -389,7 +389,7 @@ func TestService_Import(t *testing.T) { "bnm", } for _, tc := range testCases { - tr.Put([]byte(tc), []byte(tc)) + tr.Put([]byte(tc), []byte(tc), trie.V0) } digest := types.NewDigest() diff --git a/dot/utils_integration_test.go b/dot/utils_integration_test.go index 7f4e0a67a3..2b08c3dc03 100644 --- a/dot/utils_integration_test.go +++ b/dot/utils_integration_test.go @@ -27,7 +27,7 @@ func TestTrieSnapshot(t *testing.T) { for k, v := range genRaw.Genesis.Raw["top"] { val := []byte(v) - tri.Put([]byte(k), val) + tri.Put([]byte(k), val, trie.V0) } deepCopyTrie := tri.DeepCopy() @@ -51,7 +51,7 @@ func TestTrieSnapshot(t *testing.T) { // Modify the current trie. value[0] = 'w' - newTrie.Put(key, value) + newTrie.Put(key, value, trie.V0) // Get the updated root hash of all tries. tHash, err = tri.Hash() From 752ce6dff78fd6cc2079e6cd5581833bf8bd67df Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 10 Aug 2023 18:14:39 -0300 Subject: [PATCH 022/128] Fix deepsource checks --- dot/state/storage_notify_test.go | 12 +++++------- dot/state/storage_test.go | 4 ++-- dot/state/test_helpers.go | 3 ++- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/dot/state/storage_notify_test.go b/dot/state/storage_notify_test.go index 1f2f0da551..50e9c36236 100644 --- a/dot/state/storage_notify_test.go +++ b/dot/state/storage_notify_test.go @@ -175,20 +175,18 @@ func Test_Example(t *testing.T) { } if err := db.Subscribe(ctx, cb, match); err != nil && err != context.Canceled { - log.Fatal(err) + t.Error(err) } log.Printf("subscription closed") }() // Write both keys, but only one should be printed in the Output. err := db.Put(aKey, aValue) - if err != nil { - log.Fatal(err) - } + require.NoError(t, err) + err = db.Put(bKey, bValue) - if err != nil { - log.Fatal(err) - } + require.NoError(t, err) + log.Printf("stopping subscription") cancel() log.Printf("waiting for subscription to close") diff --git a/dot/state/storage_test.go b/dot/state/storage_test.go index fd0e597f0a..7d6da57e65 100644 --- a/dot/state/storage_test.go +++ b/dot/state/storage_test.go @@ -45,8 +45,8 @@ func TestStorage_StoreAndLoadTrie(t *testing.T) { trie, err := storage.LoadFromDB(root) require.NoError(t, err) ts2 := runtime.NewTrieState(trie) - new := ts2.Snapshot() - require.Equal(t, ts.Trie(), new) + newTrie := ts2.Snapshot() + require.Equal(t, ts.Trie(), newTrie) } func TestStorage_GetStorageByBlockHash(t *testing.T) { diff --git a/dot/state/test_helpers.go b/dot/state/test_helpers.go index 56255b3198..11818a5924 100644 --- a/dot/state/test_helpers.go +++ b/dot/state/test_helpers.go @@ -153,7 +153,7 @@ func AddBlocksToState(t *testing.T, blockState *BlockState, depth uint, // branches are provided with a map of depth -> # of branches func AddBlocksToStateWithFixedBranches(t *testing.T, blockState *BlockState, depth uint, branches map[uint]int) { bestBlockHash := blockState.BestBlockHash() - tb := []testBranch{} + var tb []testBranch arrivalTime := time.Now() rt, err := blockState.GetRuntime(bestBlockHash) @@ -235,6 +235,7 @@ func AddBlocksToStateWithFixedBranches(t *testing.T, blockState *BlockState, dep } } +//lint:ignore U1000 this has a real reason func generateBlockWithRandomTrie(t *testing.T, serv *Service, parent *common.Hash, bNum uint) (*types.Block, *runtime.TrieState) { trieState, err := serv.Storage.TrieState(nil) From 05145016348b0dc89d884d7b7caf8c437a7c4e8c Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 15 Aug 2023 11:38:20 -0300 Subject: [PATCH 023/128] Revisit TODOs --- cmd/gossamer/commands/import_state.go | 15 ++++++++++- dot/import.go | 8 +++--- lib/runtime/wasmer/exports_test.go | 17 ++++++------ lib/trie/proof/proof_test.go | 37 +++++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 14 deletions(-) diff --git a/cmd/gossamer/commands/import_state.go b/cmd/gossamer/commands/import_state.go index a12f137ba9..9f3f118b7b 100644 --- a/cmd/gossamer/commands/import_state.go +++ b/cmd/gossamer/commands/import_state.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/ChainSafe/gossamer/dot" + "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" "github.com/spf13/cobra" ) @@ -54,6 +55,18 @@ func execImportState(cmd *cobra.Command) error { return fmt.Errorf("state-file must be specified") } + stateVersionFlag, err := cmd.Flags().GetString("state-version") + if err != nil { + return fmt.Errorf("failed to get state-version: %s", err) + } + if stateVersionFlag == "" { + return fmt.Errorf("state-version must be specified") + } + stateVersion, err := trie.ParseVersion(stateVersionFlag) + if err != nil { + return fmt.Errorf("failed to parse state-version: %s", err) + } + headerFile, err := cmd.Flags().GetString("header-file") if err != nil { return fmt.Errorf("failed to get header-file: %s", err) @@ -64,5 +77,5 @@ func execImportState(cmd *cobra.Command) error { basePath = utils.ExpandDir(basePath) - return dot.ImportState(basePath, stateFile, headerFile, firstSlot) + return dot.ImportState(basePath, stateFile, headerFile, firstSlot, stateVersion) } diff --git a/dot/import.go b/dot/import.go index 5d0d9b313b..5e6687274e 100644 --- a/dot/import.go +++ b/dot/import.go @@ -20,8 +20,8 @@ import ( ) // ImportState imports the state in the given files to the database with the given path. -func ImportState(basepath, stateFP, headerFP string, firstSlot uint64) error { - tr, err := newTrieFromPairs(stateFP) +func ImportState(basepath, stateFP, headerFP string, firstSlot uint64, stateVersion trie.Version) error { + tr, err := newTrieFromPairs(stateFP, stateVersion) if err != nil { return err } @@ -41,7 +41,7 @@ func ImportState(basepath, stateFP, headerFP string, firstSlot uint64) error { return srv.Import(header, tr, firstSlot) } -func newTrieFromPairs(filename string) (*trie.Trie, error) { +func newTrieFromPairs(filename string, stateVersion trie.Version) (*trie.Trie, error) { data, err := os.ReadFile(filepath.Clean(filename)) if err != nil { return nil, err @@ -63,7 +63,7 @@ func newTrieFromPairs(filename string) (*trie.Trie, error) { } //TODO: revisit this to use the right trie version - tr, err := trie.LoadFromMap(entries, trie.V0) + tr, err := trie.LoadFromMap(entries, stateVersion) if err != nil { return nil, err } diff --git a/lib/runtime/wasmer/exports_test.go b/lib/runtime/wasmer/exports_test.go index 367a03701a..0be7186f70 100644 --- a/lib/runtime/wasmer/exports_test.go +++ b/lib/runtime/wasmer/exports_test.go @@ -627,7 +627,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1(t *testing.T) { } func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock3784(t *testing.T) { - gossTrie3783 := newTrieFromPairs(t, "../test_data/kusama/block3783.out") + gossTrie3783 := newTrieFromPairs(t, "../test_data/kusama/block3783.out", trie.V0) expectedRoot := common.MustHexToHash("0x948338bc0976aee78879d559a1f42385407e5a481b05a91d2a9386aa7507e7a0") require.Equal(t, expectedRoot, gossTrie3783.MustHash()) @@ -673,7 +673,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock3784(t *testing.T) { } func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock901442(t *testing.T) { - ksmTrie901441 := newTrieFromPairs(t, "../test_data/kusama/block901441.out") + ksmTrie901441 := newTrieFromPairs(t, "../test_data/kusama/block901441.out", trie.V0) expectedRoot := common.MustHexToHash("0x3a2ef7ee032f5810160bb8f3ffe3e3377bb6f2769ee9f79a5425973347acd504") require.Equal(t, expectedRoot, ksmTrie901441.MustHash()) @@ -719,7 +719,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock901442(t *testing.T) { } func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1377831(t *testing.T) { - ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block1377830.out") + ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block1377830.out", trie.V0) expectedRoot := common.MustHexToHash("0xe4de6fecda9e9e35f937d159665cf984bc1a68048b6c78912de0aeb6bd7f7e99") require.Equal(t, expectedRoot, ksmTrie.MustHash()) @@ -765,7 +765,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1377831(t *testing.T) { } func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1482003(t *testing.T) { - ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block1482002.out") + ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block1482002.out", trie.V0) expectedRoot := common.MustHexToHash("0x09f9ca28df0560c2291aa16b56e15e07d1e1927088f51356d522722aa90ca7cb") require.Equal(t, expectedRoot, ksmTrie.MustHash()) @@ -813,7 +813,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1482003(t *testing.T) { func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock4939774(t *testing.T) { t.Skip("skip for now as block4939773 is too large") - ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block4939773.out") + ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block4939773.out", trie.V0) expectedRoot := common.MustHexToHash("0xc45748e6e8632b44fc32b04cc4380098a9584cbd63ffbc59adce189574fc36fe") require.Equal(t, expectedRoot, ksmTrie.MustHash()) @@ -856,7 +856,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock4939774(t *testing.T) { } func TestInstance_ExecuteBlock_PolkadotBlock1089328(t *testing.T) { - dotTrie := newTrieFromPairs(t, "../test_data/polkadot/block1089327.json") + dotTrie := newTrieFromPairs(t, "../test_data/polkadot/block1089327.json", trie.V0) expectedRoot := common.MustHexToHash("0x87ed9ebe7fb645d3b5b0255cc16e78ed022d9fbb52486105436e15a74557535b") require.Equal(t, expectedRoot, dotTrie.MustHash()) @@ -983,7 +983,7 @@ func TestInstance_PaymentQueryInfo(t *testing.T) { } } -func newTrieFromPairs(t *testing.T, filename string) *trie.Trie { +func newTrieFromPairs(t *testing.T, filename string, version trie.Version) *trie.Trie { data, err := os.ReadFile(filename) require.NoError(t, err) @@ -998,8 +998,7 @@ func newTrieFromPairs(t *testing.T, filename string) *trie.Trie { entries[pairArr[0].(string)] = pairArr[1].(string) } - //TODO: revisit this to use the right trie version - tr, err := trie.LoadFromMap(entries, trie.V0) + tr, err := trie.LoadFromMap(entries, version) require.NoError(t, err) return &tr } diff --git a/lib/trie/proof/proof_test.go b/lib/trie/proof/proof_test.go index f808c999b4..49c1b3110f 100644 --- a/lib/trie/proof/proof_test.go +++ b/lib/trie/proof/proof_test.go @@ -99,3 +99,40 @@ func TestParachainHeaderStateProof(t *testing.T) { require.NoError(t, err) } + +func TestTrieProof(t *testing.T) { + key, err := hex.DecodeString("f0c365c3cf59d671eb72da0e7a4113c49f1f0515f462cdcf84e0f1d6045dfcbb") + if err != nil { + panic(err) + } + root, err := hex.DecodeString("dc4887669c2a6b3462e9557aa3105a66a02b6ec3b21784613de78c95dc3cbbe0") + if err != nil { + panic(err) + } + bytes1, err := hex.DecodeString("80fffd8028b54b9a0a90d41b7941c43e6a0597d5914e3b62bdcb244851b9fc806c28ea2480d5ba6d50586692888b0c2f5b3c3fc345eb3a2405996f025ed37982ca396f5ed580bd281c12f20f06077bffd56b2f8b6431ee6c9fd11fed9c22db86cea849aeff2280afa1e1b5ce72ea1675e5e69be85e98fbfb660691a76fee9229f758a75315f2bc80aafc60caa3519d4b861e6b8da226266a15060e2071bba4184e194da61dfb208e809d3f6ae8f655009551de95ae1ef863f6771522fd5c0475a50ff53c5c8169b5888024a760a8f6c27928ae9e2fed9968bc5f6e17c3ae647398d8a615e5b2bb4b425f8085a0da830399f25fca4b653de654ffd3c92be39f3ae4f54e7c504961b5bd00cf80c2d44d371e5fc1f50227d7491ad65ad049630361cefb4ab1844831237609f08380c644938921d14ae611f3a90991af8b7f5bdb8fa361ee2c646c849bca90f491e6806e729ad43a591cd1321762582782bbe4ed193c6f583ec76013126f7f786e376280509bb016f2887d12137e73d26d7ddcd7f9c8ff458147cb9d309494655fe68de180009f8697d760fbe020564b07f407e6aad58ba9451b3d2d88b3ee03e12db7c47480952dcc0804e1120508a1753f1de4aa5b7481026a3320df8b48e918f0cecbaed3803360bf948fddc403d345064082e8393d7a1aad7a19081f6d02d94358f242b86c") + if err != nil { + panic(err) + } + bytes2, err := hex.DecodeString("9ec365c3cf59d671eb72da0e7a4113c41002505f0e7b9012096b41c4eb3aaf947f6ea429080000685f0f1f0515f462cdcf84e0f1d6045dfcbb20865c4a2b7f010000") + if err != nil { + panic(err) + } + bytes3, err := hex.DecodeString("8005088076c66e2871b4fe037d112ebffb3bfc8bd83a4ec26047f58ee2df7be4e9ebe3d680c1638f702aaa71e4b78cc8538ecae03e827bb494cc54279606b201ec071a5e24806d2a1e6d5236e1e13c5a5c84831f5f5383f97eba32df6f9faf80e32cf2f129bc") + if err != nil { + panic(err) + } + var proof = [][]byte{ + bytes1, bytes2, bytes3, + } + proofDB, err := db.NewMemoryDBFromProof(proof) + require.NoError(t, err) + + trie, err := buildTrie(proof, root, proofDB) + require.NoError(t, err) + value := trie.Get(key) + + expectedValue := 0x865c4a2b7f100 + require.Equal(t, expectedValue, value) + + fmt.Printf("value: %s\n", value) +} From a0b8dc11be2fc9114705db1373d5afb30cbdc1e7 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 15 Aug 2023 12:22:20 -0300 Subject: [PATCH 024/128] Add more tests --- lib/trie/proof/proof_test.go | 37 ------------------- lib/trie/trie.go | 3 +- lib/trie/trie_test.go | 71 ++++++++++++++++++++++++++++++++++-- 3 files changed, 69 insertions(+), 42 deletions(-) diff --git a/lib/trie/proof/proof_test.go b/lib/trie/proof/proof_test.go index 49c1b3110f..f808c999b4 100644 --- a/lib/trie/proof/proof_test.go +++ b/lib/trie/proof/proof_test.go @@ -99,40 +99,3 @@ func TestParachainHeaderStateProof(t *testing.T) { require.NoError(t, err) } - -func TestTrieProof(t *testing.T) { - key, err := hex.DecodeString("f0c365c3cf59d671eb72da0e7a4113c49f1f0515f462cdcf84e0f1d6045dfcbb") - if err != nil { - panic(err) - } - root, err := hex.DecodeString("dc4887669c2a6b3462e9557aa3105a66a02b6ec3b21784613de78c95dc3cbbe0") - if err != nil { - panic(err) - } - bytes1, err := hex.DecodeString("80fffd8028b54b9a0a90d41b7941c43e6a0597d5914e3b62bdcb244851b9fc806c28ea2480d5ba6d50586692888b0c2f5b3c3fc345eb3a2405996f025ed37982ca396f5ed580bd281c12f20f06077bffd56b2f8b6431ee6c9fd11fed9c22db86cea849aeff2280afa1e1b5ce72ea1675e5e69be85e98fbfb660691a76fee9229f758a75315f2bc80aafc60caa3519d4b861e6b8da226266a15060e2071bba4184e194da61dfb208e809d3f6ae8f655009551de95ae1ef863f6771522fd5c0475a50ff53c5c8169b5888024a760a8f6c27928ae9e2fed9968bc5f6e17c3ae647398d8a615e5b2bb4b425f8085a0da830399f25fca4b653de654ffd3c92be39f3ae4f54e7c504961b5bd00cf80c2d44d371e5fc1f50227d7491ad65ad049630361cefb4ab1844831237609f08380c644938921d14ae611f3a90991af8b7f5bdb8fa361ee2c646c849bca90f491e6806e729ad43a591cd1321762582782bbe4ed193c6f583ec76013126f7f786e376280509bb016f2887d12137e73d26d7ddcd7f9c8ff458147cb9d309494655fe68de180009f8697d760fbe020564b07f407e6aad58ba9451b3d2d88b3ee03e12db7c47480952dcc0804e1120508a1753f1de4aa5b7481026a3320df8b48e918f0cecbaed3803360bf948fddc403d345064082e8393d7a1aad7a19081f6d02d94358f242b86c") - if err != nil { - panic(err) - } - bytes2, err := hex.DecodeString("9ec365c3cf59d671eb72da0e7a4113c41002505f0e7b9012096b41c4eb3aaf947f6ea429080000685f0f1f0515f462cdcf84e0f1d6045dfcbb20865c4a2b7f010000") - if err != nil { - panic(err) - } - bytes3, err := hex.DecodeString("8005088076c66e2871b4fe037d112ebffb3bfc8bd83a4ec26047f58ee2df7be4e9ebe3d680c1638f702aaa71e4b78cc8538ecae03e827bb494cc54279606b201ec071a5e24806d2a1e6d5236e1e13c5a5c84831f5f5383f97eba32df6f9faf80e32cf2f129bc") - if err != nil { - panic(err) - } - var proof = [][]byte{ - bytes1, bytes2, bytes3, - } - proofDB, err := db.NewMemoryDBFromProof(proof) - require.NoError(t, err) - - trie, err := buildTrie(proof, root, proofDB) - require.NoError(t, err) - value := trie.Get(key) - - expectedValue := 0x865c4a2b7f100 - require.Equal(t, expectedValue, value) - - fmt.Printf("value: %s\n", value) -} diff --git a/lib/trie/trie.go b/lib/trie/trie.go index 025f6e619e..6e8e50eb6a 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -375,8 +375,7 @@ func (t *Trie) insertKeyLE(keyLE, value []byte, isValueHashed := version.ShouldHashValue(value) if isValueHashed { - hashedValue := common.MustBlake2bHash(value) - copy(value, hashedValue[:]) + value = common.MustBlake2bHash(value).ToBytes() } root, _, _, err := t.insert(t.root, nibblesKey, value, isValueHashed, pendingDeltas) diff --git a/lib/trie/trie_test.go b/lib/trie/trie_test.go index 2a36c65826..2a58d7ebc3 100644 --- a/lib/trie/trie_test.go +++ b/lib/trie/trie_test.go @@ -690,7 +690,7 @@ func Test_Trie_Entries(t *testing.T) { entriesMatch(t, expectedEntries, entries) }) - t.Run("end_to_end", func(t *testing.T) { + t.Run("end_to_end_v0", func(t *testing.T) { t.Parallel() trie := Trie{ @@ -713,6 +713,30 @@ func Test_Trie_Entries(t *testing.T) { assert.Equal(t, kv, entries) }) + + t.Run("end_to_end_v1", func(t *testing.T) { + t.Parallel() + + trie := Trie{ + root: nil, + childTries: make(map[common.Hash]*Trie), + } + + kv := map[string][]byte{ + "ab": []byte("pen"), + "abc": []byte("penguin"), + "hy": []byte("feather"), + "long": []byte("newvaluewithmorethan32byteslength"), + } + + for k, v := range kv { + trie.Put([]byte(k), v, V1) + } + + entries := trie.Entries() + + assert.Equal(t, kv, entries) + }) } func Test_Trie_NextKey(t *testing.T) { @@ -1050,13 +1074,16 @@ func Test_nextKey(t *testing.T) { func Test_Trie_Put(t *testing.T) { t.Parallel() + longValue := []byte("newvaluewithmorethan32byteslength") + testCases := map[string]struct { trie Trie + stateVersion Version key []byte value []byte expectedTrie Trie }{ - "trie_with_key_and_value": { + "trie_v0_with_key_and_value": { trie: Trie{ generation: 1, deltas: newDeltas(), @@ -1092,6 +1119,44 @@ func Test_Trie_Put(t *testing.T) { }, }, }, + "trie_v1_with_key_and_value": { + trie: Trie{ + generation: 1, + deltas: newDeltas(), + root: &Node{ + PartialKey: []byte{1, 2, 0, 5}, + StorageValue: []byte{1}, + }, + }, + stateVersion: V1, + key: []byte{0x12, 0x16}, + value: longValue, + expectedTrie: Trie{ + generation: 1, + deltas: newDeltas("0xa195089c3e8f8b5b36978700ad954aed99e08413cfc1e2b4c00a5d064abe66a9"), + root: &Node{ + PartialKey: []byte{1, 2}, + Generation: 1, + Dirty: true, + Descendants: 2, + Children: padRightChildren([]*Node{ + { + PartialKey: []byte{5}, + StorageValue: []byte{1}, + Generation: 1, + Dirty: true, + }, + { + PartialKey: []byte{6}, + StorageValue: common.MustBlake2bHash(longValue).ToBytes(), + HashedValue: true, + Generation: 1, + Dirty: true, + }, + }), + }, + }, + }, } for name, testCase := range testCases { @@ -1100,7 +1165,7 @@ func Test_Trie_Put(t *testing.T) { t.Parallel() trie := testCase.trie - trie.Put(testCase.key, testCase.value, V0) + trie.Put(testCase.key, testCase.value, testCase.stateVersion) assert.Equal(t, testCase.expectedTrie, trie) }) From db45755011ba818f8c7fb609d85ad5fba10659bf Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 15 Aug 2023 14:33:18 -0300 Subject: [PATCH 025/128] Fix linter --- dot/import_integration_test.go | 10 +++++++--- lib/runtime/wasmer/exports_test.go | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/dot/import_integration_test.go b/dot/import_integration_test.go index 534f50b4cf..8dcc0efc0c 100644 --- a/dot/import_integration_test.go +++ b/dot/import_integration_test.go @@ -13,6 +13,7 @@ import ( "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/internal/log" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/pkg/scale" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -24,6 +25,7 @@ func Test_newTrieFromPairs(t *testing.T) { tests := []struct { name string filename string + version trie.Version want common.Hash err error }{ @@ -35,6 +37,7 @@ func Test_newTrieFromPairs(t *testing.T) { { name: "working example", filename: setupStateFile(t), + version: trie.V0, want: common.MustHexToHash("0x09f9ca28df0560c2291aa16b56e15e07d1e1927088f51356d522722aa90ca7cb"), }, } @@ -43,7 +46,7 @@ func Test_newTrieFromPairs(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - got, err := newTrieFromPairs(tt.filename) + got, err := newTrieFromPairs(tt.filename, tt.version) if tt.err != nil { assert.EqualError(t, err, tt.err.Error()) } else { @@ -93,7 +96,7 @@ func TestImportState_Integration(t *testing.T) { headerFP := setupHeaderFile(t) const firstSlot = uint64(262493679) - err = ImportState(config.BasePath, stateFP, headerFP, firstSlot) + err = ImportState(config.BasePath, stateFP, headerFP, firstSlot, trie.V0) require.NoError(t, err) // confirm data is imported into db stateConfig := state.Config{ @@ -126,6 +129,7 @@ func TestImportState(t *testing.T) { type args struct { basepath string stateFP string + version trie.Version headerFP string firstSlot uint64 } @@ -153,7 +157,7 @@ func TestImportState(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - err := ImportState(tt.args.basepath, tt.args.stateFP, tt.args.headerFP, tt.args.firstSlot) + err := ImportState(tt.args.basepath, tt.args.stateFP, tt.args.headerFP, tt.args.firstSlot, tt.args.version) if tt.err != nil { assert.EqualError(t, err, tt.err.Error()) } else { diff --git a/lib/runtime/wasmer/exports_test.go b/lib/runtime/wasmer/exports_test.go index 0be7186f70..d20d6eaddc 100644 --- a/lib/runtime/wasmer/exports_test.go +++ b/lib/runtime/wasmer/exports_test.go @@ -627,7 +627,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1(t *testing.T) { } func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock3784(t *testing.T) { - gossTrie3783 := newTrieFromPairs(t, "../test_data/kusama/block3783.out", trie.V0) + gossTrie3783 := newTrieFromPairs(t, "../test_data/kusama/block3783.out", trie.V1) expectedRoot := common.MustHexToHash("0x948338bc0976aee78879d559a1f42385407e5a481b05a91d2a9386aa7507e7a0") require.Equal(t, expectedRoot, gossTrie3783.MustHash()) From 5719b28785b307e8e59058ea3e996d7c97a83cef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20Junior?= Date: Fri, 11 Aug 2023 11:50:30 -0400 Subject: [PATCH 026/128] fix(dot/sync): rework on bootstrap/tip sync (#3227) --- .github/workflows/zombienet.yml | 5 +- chain/westend/genesis.json | 18 +- dot/network/discovery.go | 4 +- dot/network/errors.go | 2 + dot/network/message.go | 61 +- dot/network/message_test.go | 131 + dot/network/request_response.go | 2 +- dot/network/service.go | 5 + dot/network/sync.go | 6 - dot/node_integration_test.go | 1 - dot/services.go | 14 +- dot/sync/benchmark.go | 67 - dot/sync/benchmark_test.go | 239 -- dot/sync/block_queue.go | 57 - dot/sync/block_queue_test.go | 252 -- dot/sync/bootstrap_syncer.go | 95 - dot/sync/bootstrap_syncer_integration_test.go | 135 - dot/sync/bootstrap_syncer_test.go | 113 - dot/sync/chain_processor.go | 300 -- dot/sync/chain_processor_integration_test.go | 348 -- dot/sync/chain_processor_test.go | 1181 ------- dot/sync/chain_sync.go | 1500 ++++----- dot/sync/chain_sync_integration_test.go | 88 - dot/sync/chain_sync_test.go | 2793 +++++++++-------- dot/sync/disjoint_block_set.go | 51 +- .../disjoint_block_set_integration_test.go | 96 - dot/sync/errors.go | 25 +- dot/sync/interfaces.go | 2 + dot/sync/message.go | 13 +- dot/sync/message_integration_test.go | 24 +- dot/sync/mock_chain_processor_test.go | 58 - dot/sync/mock_chain_sync_test.go | 141 +- dot/sync/mock_disjoint_block_set_test.go | 23 +- dot/sync/{mock_req_res.go => mock_request.go} | 0 dot/sync/mocks_generate_test.go | 5 +- dot/sync/mocks_test.go | 14 + dot/sync/syncer.go | 120 +- dot/sync/syncer_integration_test.go | 4 +- dot/sync/syncer_test.go | 331 +- dot/sync/tip_syncer.go | 220 -- dot/sync/tip_syncer_integration_test.go | 372 --- dot/sync/tip_syncer_test.go | 401 --- dot/sync/worker.go | 109 +- dot/sync/worker_pool.go | 227 ++ dot/sync/worker_pool_test.go | 247 ++ dot/sync/worker_test.go | 61 + 46 files changed, 3373 insertions(+), 6588 deletions(-) delete mode 100644 dot/sync/benchmark.go delete mode 100644 dot/sync/benchmark_test.go delete mode 100644 dot/sync/block_queue.go delete mode 100644 dot/sync/block_queue_test.go delete mode 100644 dot/sync/bootstrap_syncer.go delete mode 100644 dot/sync/bootstrap_syncer_integration_test.go delete mode 100644 dot/sync/bootstrap_syncer_test.go delete mode 100644 dot/sync/chain_processor.go delete mode 100644 dot/sync/chain_processor_integration_test.go delete mode 100644 dot/sync/chain_processor_test.go delete mode 100644 dot/sync/chain_sync_integration_test.go delete mode 100644 dot/sync/mock_chain_processor_test.go rename dot/sync/{mock_req_res.go => mock_request.go} (100%) delete mode 100644 dot/sync/tip_syncer.go delete mode 100644 dot/sync/tip_syncer_integration_test.go delete mode 100644 dot/sync/tip_syncer_test.go create mode 100644 dot/sync/worker_pool.go create mode 100644 dot/sync/worker_pool_test.go create mode 100644 dot/sync/worker_test.go diff --git a/.github/workflows/zombienet.yml b/.github/workflows/zombienet.yml index 586391c646..559ca8f1d2 100644 --- a/.github/workflows/zombienet.yml +++ b/.github/workflows/zombienet.yml @@ -9,7 +9,7 @@ jobs: steps: - uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: "1.20" stable: true check-latest: true @@ -19,7 +19,6 @@ jobs: echo "::set-output name=go-build::$(go env GOCACHE)" echo "::set-output name=go-mod::$(go env GOMODCACHE)" - uses: actions/checkout@v3 - - name: Go build cache uses: actions/cache@v3 with: @@ -51,4 +50,4 @@ jobs: chmod +x /usr/local/bin/zombienet - name: Zombienet test run: | - zombienet test -p native zombienet_tests/functional/0001-basic-network.zndsl \ No newline at end of file + zombienet test -p native zombienet_tests/functional/0001-basic-network.zndsl diff --git a/chain/westend/genesis.json b/chain/westend/genesis.json index 6527d81001..a6045e6975 100644 --- a/chain/westend/genesis.json +++ b/chain/westend/genesis.json @@ -17,7 +17,23 @@ "/dns/boot-node.helikon.io/tcp/7080/p2p/12D3KooWRFDPyT8vA8mLzh6dJoyujn4QNjeqi6Ch79eSMz9beKXC", "/dns/boot-node.helikon.io/tcp/7082/wss/p2p/12D3KooWRFDPyT8vA8mLzh6dJoyujn4QNjeqi6Ch79eSMz9beKXC", "/dns/westend.bootnode.amforc.com/tcp/30333/p2p/12D3KooWJ5y9ZgVepBQNW4aabrxgmnrApdVnscqgKWiUu4BNJbC8", - "/dns/westend.bootnode.amforc.com/tcp/30334/wss/p2p/12D3KooWJ5y9ZgVepBQNW4aabrxgmnrApdVnscqgKWiUu4BNJbC8" + "/dns/westend.bootnode.amforc.com/tcp/30334/wss/p2p/12D3KooWJ5y9ZgVepBQNW4aabrxgmnrApdVnscqgKWiUu4BNJbC8", + "/dns/westend-bootnode.polkadotters.com/tcp/30333/p2p/12D3KooWHPHb64jXMtSRJDrYFATWeLnvChL8NtWVttY67DCH1eC5", + "/dns/westend-bootnode.polkadotters.com/tcp/30334/wss/p2p/12D3KooWHPHb64jXMtSRJDrYFATWeLnvChL8NtWVttY67DCH1eC5", + "/dns/boot-cr.gatotech.network/tcp/33300/p2p/12D3KooWQGR1vUhoy6mvQorFp3bZFn6NNezhQZ6NWnVV7tpFgoPd", + "/dns/boot-cr.gatotech.network/tcp/35300/wss/p2p/12D3KooWQGR1vUhoy6mvQorFp3bZFn6NNezhQZ6NWnVV7tpFgoPd", + "/dns/boot-westend.metaspan.io/tcp/33012/p2p/12D3KooWNTau7iG4G9cUJSwwt2QJP1W88pUf2SgqsHjRU2RL8pfa", + "/dns/boot-westend.metaspan.io/tcp/33015/ws/p2p/12D3KooWNTau7iG4G9cUJSwwt2QJP1W88pUf2SgqsHjRU2RL8pfa", + "/dns/boot-westend.metaspan.io/tcp/33016/wss/p2p/12D3KooWNTau7iG4G9cUJSwwt2QJP1W88pUf2SgqsHjRU2RL8pfa", + "/dns/westend-bootnode.turboflakes.io/tcp/30310/p2p/12D3KooWJvPDCZmReU46ghpCMJCPVUvUCav4WQdKtXQhZgJdH6tZ", + "/dns/westend-bootnode.turboflakes.io/tcp/30410/wss/p2p/12D3KooWJvPDCZmReU46ghpCMJCPVUvUCav4WQdKtXQhZgJdH6tZ", + "/dns/westend-boot-ng.dwellir.com/tcp/443/wss/p2p/12D3KooWJifoDhCL3swAKt7MWhFb7wLRFD9oG33AL3nAathmU24x", + "/dns/westend-boot-ng.dwellir.com/tcp/30335/p2p/12D3KooWJifoDhCL3swAKt7MWhFb7wLRFD9oG33AL3nAathmU24x", + "/dns/westend-bootnode.radiumblock.com/tcp/30335/wss/p2p/12D3KooWJBowJuX1TaWNWHt8Dz8z44BoCZunLCfFqxA2rLTn6TBD", + "/dns/westend-bootnode.radiumblock.com/tcp/30333/p2p/12D3KooWJBowJuX1TaWNWHt8Dz8z44BoCZunLCfFqxA2rLTn6TBD", + "/dns/wnd-bootnode.stakeworld.io/tcp/30320/p2p/12D3KooWBYdKipcNbrV5rCbgT5hco8HMLME7cE9hHC3ckqCKDuzP", + "/dns/wnd-bootnode.stakeworld.io/tcp/30321/ws/p2p/12D3KooWBYdKipcNbrV5rCbgT5hco8HMLME7cE9hHC3ckqCKDuzP", + "/dns/wnd-bootnode.stakeworld.io/tcp/30322/wss/p2p/12D3KooWBYdKipcNbrV5rCbgT5hco8HMLME7cE9hHC3ckqCKDuzP" ], "telemetryEndpoints": [ [ diff --git a/dot/network/discovery.go b/dot/network/discovery.go index e8cb8212c1..ca9d47adb2 100644 --- a/dot/network/discovery.go +++ b/dot/network/discovery.go @@ -28,7 +28,7 @@ var ( startDHTTimeout = time.Second * 10 initialAdvertisementTimeout = time.Millisecond tryAdvertiseTimeout = time.Second * 30 - connectToPeersTimeout = time.Minute * 5 + connectToPeersTimeout = time.Minute findPeersTimeout = time.Minute ) @@ -183,7 +183,7 @@ func (d *discovery) checkPeerCount() { case <-d.ctx.Done(): return case <-ticker.C: - if len(d.h.Network().Peers()) > d.minPeers { + if len(d.h.Network().Peers()) >= d.maxPeers { continue } diff --git a/dot/network/errors.go b/dot/network/errors.go index ef895a6735..640f15c4ba 100644 --- a/dot/network/errors.go +++ b/dot/network/errors.go @@ -8,6 +8,8 @@ import ( ) var ( + ErrReceivedEmptyMessage = errors.New("received empty message") + errCannotValidateHandshake = errors.New("failed to validate handshake") errMessageTypeNotValid = errors.New("message type is not valid") errInvalidHandshakeForPeer = errors.New("peer previously sent invalid handshake") diff --git a/dot/network/message.go b/dot/network/message.go index 144abe78cf..14768f6514 100644 --- a/dot/network/message.go +++ b/dot/network/message.go @@ -17,6 +17,9 @@ import ( "github.com/ChainSafe/gossamer/pkg/scale" ) +// MaxBlocksInResponse is maximum number of block data a BlockResponse message can contain +const MaxBlocksInResponse = 128 + type MessageType byte // Message types for notifications protocol messages. Used internally to map message to protocol. @@ -44,6 +47,9 @@ const ( RequestedDataReceipt = byte(4) RequestedDataMessageQueue = byte(8) RequestedDataJustification = byte(16) + BootstrapRequestData = RequestedDataHeader + + RequestedDataBody + + RequestedDataJustification ) var _ Message = (*BlockRequestMessage)(nil) @@ -325,7 +331,7 @@ type ConsensusMessage struct { } // Type returns ConsensusMsgType -func (cm *ConsensusMessage) Type() MessageType { +func (*ConsensusMessage) Type() MessageType { return ConsensusMsgType } @@ -354,3 +360,56 @@ func (cm *ConsensusMessage) Hash() (common.Hash, error) { } return common.Blake2bHash(encMsg) } + +func NewBlockRequest(startingBlock variadic.Uint32OrHash, amount uint32, + requestedData byte, direction SyncDirection) *BlockRequestMessage { + return &BlockRequestMessage{ + RequestedData: requestedData, + StartingBlock: startingBlock, + Direction: direction, + Max: &amount, + } +} + +func NewAscendingBlockRequests(startNumber, targetNumber uint, requestedData byte) []*BlockRequestMessage { + if startNumber > targetNumber { + return []*BlockRequestMessage{} + } + + diff := targetNumber - (startNumber - 1) + + // start and end block are the same, just request 1 block + if diff == 0 { + return []*BlockRequestMessage{ + NewBlockRequest(*variadic.MustNewUint32OrHash(uint32(startNumber)), 1, requestedData, Ascending), + } + } + + numRequests := diff / MaxBlocksInResponse + // we should check if the diff is in the maxResponseSize bounds + // otherwise we should increase the numRequests by one, take this + // example, we want to sync from 0 to 259, the diff is 259 + // then the num of requests is 2 (uint(259)/uint(128)) however two requests will + // retrieve only 256 blocks (each request can retrieve a max of 128 blocks), so we should + // create one more request to retrieve those missing blocks, 3 in this example. + missingBlocks := diff % MaxBlocksInResponse + if missingBlocks != 0 { + numRequests++ + } + + reqs := make([]*BlockRequestMessage, numRequests) + for i := uint(0); i < numRequests; i++ { + max := uint32(MaxBlocksInResponse) + + lastIteration := numRequests - 1 + if i == lastIteration && missingBlocks != 0 { + max = uint32(missingBlocks) + } + + start := variadic.MustNewUint32OrHash(startNumber) + reqs[i] = NewBlockRequest(*start, max, requestedData, Ascending) + startNumber += uint(max) + } + + return reqs +} diff --git a/dot/network/message_test.go b/dot/network/message_test.go index 44aed673a9..8498ee4426 100644 --- a/dot/network/message_test.go +++ b/dot/network/message_test.go @@ -422,3 +422,134 @@ func TestDecodeConsensusMessage(t *testing.T) { require.NoError(t, err) require.Equal(t, encMsg, encodedMessage) } + +func TestAscendingBlockRequest(t *testing.T) { + one := uint32(1) + three := uint32(3) + maxResponseSize := uint32(MaxBlocksInResponse) + cases := map[string]struct { + startNumber, targetNumber uint + expectedBlockRequestMessage []*BlockRequestMessage + expectedTotalOfBlocksRequested uint32 + }{ + "start_greater_than_target": { + startNumber: 10, + targetNumber: 0, + expectedBlockRequestMessage: []*BlockRequestMessage{}, + expectedTotalOfBlocksRequested: 0, + }, + + "no_difference_between_start_and_target": { + startNumber: 10, + targetNumber: 10, + expectedBlockRequestMessage: []*BlockRequestMessage{ + { + RequestedData: BootstrapRequestData, + StartingBlock: *variadic.MustNewUint32OrHash(uint32(10)), + Direction: Ascending, + Max: &one, + }, + }, + expectedTotalOfBlocksRequested: 1, + }, + + "requesting_128_blocks": { + startNumber: 1, + targetNumber: 128, + expectedTotalOfBlocksRequested: 128, + expectedBlockRequestMessage: []*BlockRequestMessage{ + { + RequestedData: BootstrapRequestData, + StartingBlock: *variadic.MustNewUint32OrHash(uint32(1)), + Direction: Ascending, + Max: &maxResponseSize, + }, + }, + }, + + "requesting_4_chunks_of_128_blocks": { + startNumber: 1, + targetNumber: 128 * 4, // 512 + expectedTotalOfBlocksRequested: 512, + expectedBlockRequestMessage: []*BlockRequestMessage{ + { + RequestedData: BootstrapRequestData, + StartingBlock: *variadic.MustNewUint32OrHash(uint32(1)), + Direction: Ascending, + Max: &maxResponseSize, + }, + { + RequestedData: BootstrapRequestData, + StartingBlock: *variadic.MustNewUint32OrHash(uint32(129)), + Direction: Ascending, + Max: &maxResponseSize, + }, + { + RequestedData: BootstrapRequestData, + StartingBlock: *variadic.MustNewUint32OrHash(uint32(257)), + Direction: Ascending, + Max: &maxResponseSize, + }, + { + RequestedData: BootstrapRequestData, + StartingBlock: *variadic.MustNewUint32OrHash(uint32(385)), + Direction: Ascending, + Max: &maxResponseSize, + }, + }, + }, + + "requesting_4_chunks_of_128_plus_3_blocks": { + startNumber: 1, + targetNumber: (128 * 4) + 3, + expectedTotalOfBlocksRequested: 515, + expectedBlockRequestMessage: []*BlockRequestMessage{ + { + RequestedData: BootstrapRequestData, + StartingBlock: *variadic.MustNewUint32OrHash(uint32(1)), + Direction: Ascending, + Max: &maxResponseSize, + }, + { + RequestedData: BootstrapRequestData, + StartingBlock: *variadic.MustNewUint32OrHash(uint32(129)), + Direction: Ascending, + Max: &maxResponseSize, + }, + { + RequestedData: BootstrapRequestData, + StartingBlock: *variadic.MustNewUint32OrHash(uint32(257)), + Direction: Ascending, + Max: &maxResponseSize, + }, + { + RequestedData: BootstrapRequestData, + StartingBlock: *variadic.MustNewUint32OrHash(uint32(385)), + Direction: Ascending, + Max: &maxResponseSize, + }, + { + RequestedData: BootstrapRequestData, + StartingBlock: *variadic.MustNewUint32OrHash(uint32(513)), + Direction: Ascending, + Max: &three, + }, + }, + }, + } + + for tname, tt := range cases { + tt := tt + + t.Run(tname, func(t *testing.T) { + requests := NewAscendingBlockRequests(tt.startNumber, tt.targetNumber, BootstrapRequestData) + require.Equal(t, tt.expectedBlockRequestMessage, requests) + + acc := uint32(0) + for _, r := range requests { + acc += *r.Max + } + require.Equal(t, tt.expectedTotalOfBlocksRequested, acc) + }) + } +} diff --git a/dot/network/request_response.go b/dot/network/request_response.go index 1671bca2f2..09956a6d27 100644 --- a/dot/network/request_response.go +++ b/dot/network/request_response.go @@ -70,7 +70,7 @@ func (rrp *RequestResponseProtocol) receiveResponse(stream libp2pnetwork.Stream, } if n == 0 { - return fmt.Errorf("received empty message") + return ErrReceivedEmptyMessage } err = msg.Decode(buf[:n]) diff --git a/dot/network/service.go b/dot/network/service.go index e2da8d43d3..fa139d3616 100644 --- a/dot/network/service.go +++ b/dot/network/service.go @@ -605,6 +605,11 @@ func (s *Service) NetworkState() common.NetworkState { } } +// AllConnectedPeersIDs returns all the connected to the node instance +func (s *Service) AllConnectedPeersIDs() []peer.ID { + return s.host.p2pHost.Network().Peers() +} + // Peers returns information about connected peers needed for the rpc server func (s *Service) Peers() []common.PeerInfo { var peers []common.PeerInfo diff --git a/dot/network/sync.go b/dot/network/sync.go index efef281203..ce96ae9d70 100644 --- a/dot/network/sync.go +++ b/dot/network/sync.go @@ -4,16 +4,10 @@ package network import ( - "time" - libp2pnetwork "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" ) -var ( - BlockRequestTimeout = time.Second * 20 -) - // handleSyncStream handles streams with the /sync/2 protocol ID func (s *Service) handleSyncStream(stream libp2pnetwork.Stream) { if stream == nil { diff --git a/dot/node_integration_test.go b/dot/node_integration_test.go index 64f3097b97..bf8931eda4 100644 --- a/dot/node_integration_test.go +++ b/dot/node_integration_test.go @@ -57,7 +57,6 @@ func TestNewNode(t *testing.T) { initConfig.Account.Key = "alice" initConfig.Core.Role = common.FullNodeRole initConfig.Core.WasmInterpreter = wazero_runtime.Name - initConfig.Log.Digest = "critical" networkConfig := &network.Config{ diff --git a/dot/services.go b/dot/services.go index 11682556ff..010b961efd 100644 --- a/dot/services.go +++ b/dot/services.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "strings" + "time" cfg "github.com/ChainSafe/gossamer/config" @@ -493,6 +494,13 @@ func (nodeBuilder) newSyncService(config *cfg.Config, st *state.Service, fg Bloc if err != nil { return nil, fmt.Errorf("failed to parse sync log level: %w", err) } + + const blockRequestTimeout = time.Second * 20 + requestMaker := net.GetRequestResponseProtocol( + network.SyncID, + blockRequestTimeout, + network.MaxBlockResponseSize) + syncCfg := &sync.Config{ LogLvl: syncLogLevel, Network: net, @@ -507,12 +515,10 @@ func (nodeBuilder) newSyncService(config *cfg.Config, st *state.Service, fg Bloc SlotDuration: slotDuration, Telemetry: telemetryMailer, BadBlocks: genesisData.BadBlocks, + RequestMaker: requestMaker, } - blockReqRes := net.GetRequestResponseProtocol(network.SyncID, network.BlockRequestTimeout, - network.MaxBlockResponseSize) - - return sync.NewService(syncCfg, blockReqRes) + return sync.NewService(syncCfg) } func (nodeBuilder) createDigestHandler(st *state.Service) (*digest.Handler, error) { diff --git a/dot/sync/benchmark.go b/dot/sync/benchmark.go deleted file mode 100644 index 018cb8b1e4..0000000000 --- a/dot/sync/benchmark.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2021 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package sync - -import ( - "container/ring" - "time" -) - -type syncBenchmarker struct { - start time.Time - startBlock uint - blocksPerSecond *ring.Ring - samplesToKeep int -} - -func newSyncBenchmarker(samplesToKeep int) *syncBenchmarker { - if samplesToKeep == 0 { - panic("cannot have 0 samples to keep") - } - - return &syncBenchmarker{ - blocksPerSecond: ring.New(samplesToKeep), - samplesToKeep: samplesToKeep, - } -} - -func (b *syncBenchmarker) begin(now time.Time, block uint) { - b.start = now - b.startBlock = block -} - -func (b *syncBenchmarker) end(now time.Time, block uint) { - duration := now.Sub(b.start) - blocks := block - b.startBlock - bps := float64(blocks) / duration.Seconds() - b.blocksPerSecond.Value = bps - b.blocksPerSecond = b.blocksPerSecond.Next() -} - -func (b *syncBenchmarker) average() float64 { - var sum float64 - var elementsSet int - b.blocksPerSecond.Do(func(x interface{}) { - if x == nil { - return - } - bps := x.(float64) - sum += bps - elementsSet++ - }) - - if elementsSet == 0 { - return 0 - } - - return sum / float64(elementsSet) -} - -func (b *syncBenchmarker) mostRecentAverage() float64 { - value := b.blocksPerSecond.Prev().Value - if value == nil { - return 0 - } - return value.(float64) -} diff --git a/dot/sync/benchmark_test.go b/dot/sync/benchmark_test.go deleted file mode 100644 index eae329300e..0000000000 --- a/dot/sync/benchmark_test.go +++ /dev/null @@ -1,239 +0,0 @@ -// Copyright 2022 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package sync - -import ( - "container/ring" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func Test_newSyncBenchmarker(t *testing.T) { - t.Parallel() - - t.Run("10_samples_to_keep", func(t *testing.T) { - t.Parallel() - const samplesToKeep = 10 - actual := newSyncBenchmarker(samplesToKeep) - - expected := &syncBenchmarker{ - blocksPerSecond: ring.New(samplesToKeep), - samplesToKeep: samplesToKeep, - } - - assert.Equal(t, expected, actual) - }) - - t.Run("panics_on_0_sample_to_keep", func(t *testing.T) { - t.Parallel() - const samplesToKeep = 0 - assert.PanicsWithValue(t, "cannot have 0 samples to keep", func() { - newSyncBenchmarker(samplesToKeep) - }) - }) -} - -func Test_syncBenchmarker_begin(t *testing.T) { - t.Parallel() - - const startSec = 1000 - start := time.Unix(startSec, 0) - const startBlock = 10 - - b := syncBenchmarker{} - b.begin(start, startBlock) - - expected := syncBenchmarker{ - start: start, - startBlock: startBlock, - } - - assert.Equal(t, expected, b) -} - -func Test_syncBenchmarker_end(t *testing.T) { - t.Parallel() - - const startSec = 1000 - start := time.Unix(startSec, 0) - - const nowSec = 1010 - now := time.Unix(nowSec, 0) - - const ( - startBlock = 10 - endBlock = 12 - ) - - const ringCap = 3 - - blocksPerSecond := ring.New(ringCap) - blocksPerSecond.Value = 1.00 - blocksPerSecond = blocksPerSecond.Next() - - b := syncBenchmarker{ - start: start, - startBlock: startBlock, - blocksPerSecond: blocksPerSecond, - } - b.end(now, endBlock) - - expectedBlocksPerSecond := ring.New(ringCap) - expectedBlocksPerSecond.Value = 1.00 - expectedBlocksPerSecond = expectedBlocksPerSecond.Next() - expectedBlocksPerSecond.Value = 0.2 - expectedBlocksPerSecond = expectedBlocksPerSecond.Next() - - expected := syncBenchmarker{ - start: start, - startBlock: startBlock, - blocksPerSecond: expectedBlocksPerSecond, - } - - assert.Equal(t, expected, b) -} - -func Test_syncBenchmarker_average(t *testing.T) { - t.Parallel() - - testCases := map[string]struct { - values []float64 - ringCap int - average float64 - }{ - // zero size ring is not possible due to constructor check - "empty_ring": { - ringCap: 1, - }, - "single_element_in_one-size_ring": { - values: []float64{1.1}, - ringCap: 1, - average: 1.1, - }, - "single_element_in_two-size_ring": { - values: []float64{1.1}, - ringCap: 2, - average: 1.1, - }, - "two_elements_in_two-size_ring": { - values: []float64{1.0, 2.0}, - ringCap: 2, - average: 1.5, - }, - } - - for name, testCase := range testCases { - testCase := testCase - t.Run(name, func(t *testing.T) { - t.Parallel() - - blocksPerSecond := ring.New(testCase.ringCap) - for _, value := range testCase.values { - blocksPerSecond.Value = value - blocksPerSecond = blocksPerSecond.Next() - } - - benchmarker := syncBenchmarker{ - blocksPerSecond: blocksPerSecond, - samplesToKeep: testCase.ringCap, - } - - avg := benchmarker.average() - - assert.Equal(t, testCase.average, avg) - }) - } -} - -func Test_syncBenchmarker_mostRecentAverage(t *testing.T) { - t.Parallel() - - testCases := map[string]struct { - values []float64 - ringCap int - average float64 - }{ - // zero size ring is not possible due to constructor check - "empty_ring": { - ringCap: 1, - }, - "single_element_in_one-size_ring": { - values: []float64{1.1}, - ringCap: 1, - average: 1.1, - }, - "single_element_in_two-size_ring": { - values: []float64{1.1}, - ringCap: 2, - average: 1.1, - }, - "two_elements_in_two-size_ring": { - values: []float64{1.0, 2.0}, - ringCap: 2, - average: 2.0, - }, - "three_elements_in_two-size_ring": { - values: []float64{1.0, 2.0, 3.0}, - ringCap: 2, - average: 3.0, - }, - } - - for name, testCase := range testCases { - testCase := testCase - t.Run(name, func(t *testing.T) { - t.Parallel() - - blocksPerSecond := ring.New(testCase.ringCap) - for _, value := range testCase.values { - blocksPerSecond.Value = value - blocksPerSecond = blocksPerSecond.Next() - } - - benchmarker := syncBenchmarker{ - blocksPerSecond: blocksPerSecond, - } - - avg := benchmarker.mostRecentAverage() - - assert.Equal(t, testCase.average, avg) - }) - } -} - -func Test_syncBenchmarker(t *testing.T) { - t.Parallel() - - const samplesToKeep = 5 - benchmarker := newSyncBenchmarker(samplesToKeep) - - const initialBlock = 10 - timeZero := time.Unix(0, 0) - const timeIncrement = time.Second - const baseBlocksIncrement uint = 1 - - startTime := timeZero - endTime := startTime.Add(timeIncrement) - var block uint = initialBlock - - const samples = 10 - for i := 0; i < samples; i++ { - benchmarker.begin(startTime, block) - block += baseBlocksIncrement + uint(i) - benchmarker.end(endTime, block) - - startTime = startTime.Add(timeIncrement) - endTime = startTime.Add(timeIncrement) - } - - avg := benchmarker.average() - const expectedAvg = 8.0 - assert.Equal(t, expectedAvg, avg) - - mostRecentAvg := benchmarker.mostRecentAverage() - const expectedMostRecentAvg = 10.0 - assert.Equal(t, expectedMostRecentAvg, mostRecentAvg) -} diff --git a/dot/sync/block_queue.go b/dot/sync/block_queue.go deleted file mode 100644 index 9b5a81d597..0000000000 --- a/dot/sync/block_queue.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2021 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package sync - -import ( - "context" - "sync" - - "github.com/ChainSafe/gossamer/dot/types" - "github.com/ChainSafe/gossamer/lib/common" -) - -type blockQueue struct { - queue chan *types.BlockData - hashesSet map[common.Hash]struct{} - hashesSetMutex sync.RWMutex -} - -// newBlockQueue initialises a queue of *types.BlockData with the given capacity. -func newBlockQueue(capacity int) *blockQueue { - return &blockQueue{ - queue: make(chan *types.BlockData, capacity), - hashesSet: make(map[common.Hash]struct{}, capacity), - } -} - -// push pushes an item into the queue. It blocks if the queue is at capacity. -func (bq *blockQueue) push(blockData *types.BlockData) { - bq.hashesSetMutex.Lock() - bq.hashesSet[blockData.Hash] = struct{}{} - bq.hashesSetMutex.Unlock() - - bq.queue <- blockData -} - -// pop pops the next item from the queue. It blocks if the queue is empty -// until the context is cancelled. If the context is canceled, it returns -// the error from the context. -func (bq *blockQueue) pop(ctx context.Context) (blockData *types.BlockData, err error) { - select { - case <-ctx.Done(): - return blockData, ctx.Err() - case blockData = <-bq.queue: - } - bq.hashesSetMutex.Lock() - delete(bq.hashesSet, blockData.Hash) - bq.hashesSetMutex.Unlock() - return blockData, nil -} - -func (bq *blockQueue) has(blockHash common.Hash) (has bool) { - bq.hashesSetMutex.RLock() - defer bq.hashesSetMutex.RUnlock() - _, has = bq.hashesSet[blockHash] - return has -} diff --git a/dot/sync/block_queue_test.go b/dot/sync/block_queue_test.go deleted file mode 100644 index cff9b181b3..0000000000 --- a/dot/sync/block_queue_test.go +++ /dev/null @@ -1,252 +0,0 @@ -// Copyright 2021 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package sync - -import ( - "context" - "sync" - "testing" - "time" - - "github.com/ChainSafe/gossamer/dot/types" - "github.com/ChainSafe/gossamer/lib/common" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func Test_newBlockQueue(t *testing.T) { - t.Parallel() - - const capacity = 1 - bq := newBlockQueue(capacity) - - require.NotNil(t, bq.queue) - assert.Equal(t, 1, cap(bq.queue)) - assert.Equal(t, 0, len(bq.queue)) - bq.queue = nil - - expectedBlockQueue := &blockQueue{ - hashesSet: make(map[common.Hash]struct{}, capacity), - } - assert.Equal(t, expectedBlockQueue, bq) -} - -func Test_blockQueue_push(t *testing.T) { - t.Parallel() - - const capacity = 1 - bq := newBlockQueue(capacity) - blockData := &types.BlockData{ - Hash: common.Hash{1}, - } - - bq.push(blockData) - - // cannot compare channels - require.NotNil(t, bq.queue) - assert.Len(t, bq.queue, 1) - - receivedBlockData := <-bq.queue - expectedBlockData := &types.BlockData{ - Hash: common.Hash{1}, - } - assert.Equal(t, expectedBlockData, receivedBlockData) - - bq.queue = nil - expectedBlockQueue := &blockQueue{ - hashesSet: map[common.Hash]struct{}{{1}: {}}, - } - assert.Equal(t, expectedBlockQueue, bq) -} - -func Test_blockQueue_pop(t *testing.T) { - t.Parallel() - - t.Run("context_canceled", func(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - cancel() - - const capacity = 1 - bq := newBlockQueue(capacity) - - blockData, err := bq.pop(ctx) - assert.Nil(t, blockData) - assert.ErrorIs(t, err, context.Canceled) - }) - - t.Run("get_block_data_after_waiting", func(t *testing.T) { - t.Parallel() - - ctx := context.Background() - - const capacity = 1 - bq := newBlockQueue(capacity) - - const afterDuration = 5 * time.Millisecond - time.AfterFunc(afterDuration, func() { - blockData := &types.BlockData{ - Hash: common.Hash{1}, - } - bq.push(blockData) - }) - - blockData, err := bq.pop(ctx) - - expectedBlockData := &types.BlockData{ - Hash: common.Hash{1}, - } - assert.Equal(t, expectedBlockData, blockData) - assert.NoError(t, err) - - assert.Len(t, bq.queue, 0) - bq.queue = nil - expectedBlockQueue := &blockQueue{ - hashesSet: map[common.Hash]struct{}{}, - } - assert.Equal(t, expectedBlockQueue, bq) - }) -} - -func Test_blockQueue_has(t *testing.T) { - t.Parallel() - - testCases := map[string]struct { - blockQueue *blockQueue - blockHash common.Hash - has bool - }{ - "absent": { - blockQueue: &blockQueue{ - hashesSet: map[common.Hash]struct{}{}, - }, - blockHash: common.Hash{1}, - }, - "exists": { - blockQueue: &blockQueue{ - hashesSet: map[common.Hash]struct{}{{1}: {}}, - }, - blockHash: common.Hash{1}, - has: true, - }, - } - - for name, tc := range testCases { - testCase := tc - t.Run(name, func(t *testing.T) { - t.Parallel() - - has := testCase.blockQueue.has(testCase.blockHash) - assert.Equal(t, testCase.has, has) - }) - } -} - -func Test_lockQueue_endToEnd(t *testing.T) { - t.Parallel() - - const capacity = 10 - blockQueue := newBlockQueue(capacity) - - newBlockData := func(i byte) *types.BlockData { - return &types.BlockData{ - Hash: common.Hash{i}, - } - } - - blockQueue.push(newBlockData(1)) - blockQueue.push(newBlockData(2)) - blockQueue.push(newBlockData(3)) - - blockData, err := blockQueue.pop(context.Background()) - assert.Equal(t, newBlockData(1), blockData) - assert.NoError(t, err) - - has := blockQueue.has(newBlockData(2).Hash) - assert.True(t, has) - has = blockQueue.has(newBlockData(3).Hash) - assert.True(t, has) - - blockQueue.push(newBlockData(4)) - - has = blockQueue.has(newBlockData(4).Hash) - assert.True(t, has) - - blockData, err = blockQueue.pop(context.Background()) - assert.Equal(t, newBlockData(2), blockData) - assert.NoError(t, err) - - // drain queue - for len(blockQueue.queue) > 0 { - <-blockQueue.queue - } -} - -func Test_lockQueue_threadSafety(t *testing.T) { - // This test consists in checking for concurrent access - // using the -race detector. - t.Parallel() - - var startWg, endWg sync.WaitGroup - ctx, cancel := context.WithCancel(context.Background()) - - const operations = 3 - const parallelism = 3 - const goroutines = parallelism * operations - startWg.Add(goroutines) - endWg.Add(goroutines) - - const testDuration = 50 * time.Millisecond - go func() { - timer := time.NewTimer(time.Hour) - startWg.Wait() - _ = timer.Reset(testDuration) - <-timer.C - cancel() - }() - - runInLoop := func(f func()) { - defer endWg.Done() - startWg.Done() - startWg.Wait() - for ctx.Err() == nil { - f() - } - } - - const capacity = 10 - blockQueue := newBlockQueue(capacity) - blockData := &types.BlockData{ - Hash: common.Hash{1}, - } - blockHash := common.Hash{1} - - endWg.Add(1) - go func() { - defer endWg.Done() - <-ctx.Done() - // Empty queue channel to make sure `push` does not block - // when the context is cancelled. - for len(blockQueue.queue) > 0 { - <-blockQueue.queue - } - }() - - for i := 0; i < parallelism; i++ { - go runInLoop(func() { - blockQueue.push(blockData) - }) - - go runInLoop(func() { - _, _ = blockQueue.pop(ctx) - }) - - go runInLoop(func() { - _ = blockQueue.has(blockHash) - }) - } - - endWg.Wait() -} diff --git a/dot/sync/bootstrap_syncer.go b/dot/sync/bootstrap_syncer.go deleted file mode 100644 index f6b912b0df..0000000000 --- a/dot/sync/bootstrap_syncer.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2021 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package sync - -import ( - "errors" - - "github.com/ChainSafe/gossamer/dot/network" - "github.com/ChainSafe/gossamer/lib/common" -) - -var _ workHandler = &bootstrapSyncer{} - -var bootstrapRequestData = network.RequestedDataHeader + network.RequestedDataBody + network.RequestedDataJustification - -// bootstrapSyncer handles worker logic for bootstrap mode -type bootstrapSyncer struct { - blockState BlockState -} - -func newBootstrapSyncer(blockState BlockState) *bootstrapSyncer { - return &bootstrapSyncer{ - blockState: blockState, - } -} - -func (s *bootstrapSyncer) handleNewPeerState(ps *peerState) (*worker, error) { - head, err := s.blockState.BestBlockHeader() - if err != nil { - return nil, err - } - - if ps.number <= head.Number { - return nil, nil - } - - return &worker{ - startNumber: uintPtr(head.Number + 1), - targetHash: ps.hash, - targetNumber: uintPtr(ps.number), - requestData: bootstrapRequestData, - direction: network.Ascending, - }, nil -} - -func (s *bootstrapSyncer) handleWorkerResult(res *worker) ( - workerToRetry *worker, err error) { - // if there is an error, potentially retry the worker - if res.err == nil { - return nil, nil - } - - // new worker should update start block and re-dispatch - head, err := s.blockState.BestBlockHeader() - if err != nil { - return nil, err - } - - // we've reached the target, return - if *res.targetNumber <= head.Number { - return nil, nil - } - - startNumber := head.Number + 1 - - // in the case we started a block producing node, we might have produced blocks - // before fully syncing (this should probably be fixed by connecting sync into BABE) - if errors.Is(res.err.err, errUnknownParent) { - fin, err := s.blockState.GetHighestFinalisedHeader() - if err != nil { - return nil, err - } - - startNumber = fin.Number - } - - return &worker{ - startHash: common.Hash{}, // for bootstrap, just use number - startNumber: uintPtr(startNumber), - targetHash: res.targetHash, - targetNumber: res.targetNumber, - requestData: res.requestData, - direction: res.direction, - }, nil -} - -func (*bootstrapSyncer) hasCurrentWorker(_ *worker, workers map[uint64]*worker) bool { - // we're in bootstrap mode, and there already is a worker, we don't need to dispatch another - return len(workers) != 0 -} - -func (*bootstrapSyncer) handleTick() ([]*worker, error) { - return nil, nil -} diff --git a/dot/sync/bootstrap_syncer_integration_test.go b/dot/sync/bootstrap_syncer_integration_test.go deleted file mode 100644 index 20a9e32d3e..0000000000 --- a/dot/sync/bootstrap_syncer_integration_test.go +++ /dev/null @@ -1,135 +0,0 @@ -//go:build integration - -// Copyright 2021 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package sync - -import ( - "testing" - - "github.com/ChainSafe/gossamer/dot/types" - "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" - - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/require" -) - -func newTestBootstrapSyncer(t *testing.T) *bootstrapSyncer { - header := types.NewHeader( - common.NewHash([]byte{0}), trie.EmptyHash, - trie.EmptyHash, 100, types.NewDigest()) - - finHeader := types.NewHeader( - common.NewHash([]byte{0}), trie.EmptyHash, - trie.EmptyHash, 200, types.NewDigest()) - - ctrl := gomock.NewController(t) - bs := NewMockBlockState(ctrl) - bs.EXPECT().BestBlockHeader().Return(header, nil).AnyTimes() - bs.EXPECT().GetHighestFinalisedHeader().Return(finHeader, nil).AnyTimes() - - return newBootstrapSyncer(bs) -} - -func TestBootstrapSyncer_handleWork(t *testing.T) { - s := newTestBootstrapSyncer(t) - - // peer's state is equal or lower than ours - // should not create a worker for bootstrap mode - w, err := s.handleNewPeerState(&peerState{ - number: 100, - }) - require.NoError(t, err) - require.Nil(t, w) - - w, err = s.handleNewPeerState(&peerState{ - number: 99, - }) - require.NoError(t, err) - require.Nil(t, w) - - // if peer's number is highest, return worker w/ their block as target - expected := &worker{ - requestData: bootstrapRequestData, - startNumber: uintPtr(101), - targetHash: common.NewHash([]byte{1}), - targetNumber: uintPtr(101), - } - w, err = s.handleNewPeerState(&peerState{ - number: 101, - hash: common.NewHash([]byte{1}), - }) - require.NoError(t, err) - require.Equal(t, expected, w) - - expected = &worker{ - requestData: bootstrapRequestData, - startNumber: uintPtr(101), - targetHash: common.NewHash([]byte{1}), - targetNumber: uintPtr(9999), - } - w, err = s.handleNewPeerState(&peerState{ - number: 9999, - hash: common.NewHash([]byte{1}), - }) - require.NoError(t, err) - require.Equal(t, expected, w) -} - -func TestBootstrapSyncer_handleWorkerResult(t *testing.T) { - s := newTestBootstrapSyncer(t) - - // if the worker error is nil, then this function should do nothing - res := &worker{} - w, err := s.handleWorkerResult(res) - require.NoError(t, err) - require.Nil(t, w) - - // if there was a worker error, this should return a worker with - // startNumber = bestBlockNumber + 1 and the same target as previously - expected := &worker{ - requestData: bootstrapRequestData, - startNumber: uintPtr(101), - targetHash: common.NewHash([]byte{1}), - targetNumber: uintPtr(201), - } - - res = &worker{ - requestData: bootstrapRequestData, - targetHash: common.NewHash([]byte{1}), - targetNumber: uintPtr(201), - err: &workerError{}, - } - - w, err = s.handleWorkerResult(res) - require.NoError(t, err) - require.Equal(t, expected, w) -} - -func TestBootstrapSyncer_handleWorkerResult_errUnknownParent(t *testing.T) { - s := newTestBootstrapSyncer(t) - - // if there was a worker error, this should return a worker with - // startNumber = bestBlockNumber + 1 and the same target as previously - expected := &worker{ - requestData: bootstrapRequestData, - startNumber: uintPtr(200), - targetHash: common.NewHash([]byte{1}), - targetNumber: uintPtr(300), - } - - res := &worker{ - requestData: bootstrapRequestData, - targetHash: common.NewHash([]byte{1}), - targetNumber: uintPtr(300), - err: &workerError{ - err: errUnknownParent, - }, - } - - w, err := s.handleWorkerResult(res) - require.NoError(t, err) - require.Equal(t, expected, w) -} diff --git a/dot/sync/bootstrap_syncer_test.go b/dot/sync/bootstrap_syncer_test.go deleted file mode 100644 index 9d59f8dd27..0000000000 --- a/dot/sync/bootstrap_syncer_test.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2021 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package sync - -import ( - "errors" - "testing" - - "github.com/ChainSafe/gossamer/dot/types" - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" -) - -func Test_bootstrapSyncer_handleWorkerResult(t *testing.T) { - t.Parallel() - mockError := errors.New("mock testing error") - - tests := map[string]struct { - blockStateBuilder func(ctrl *gomock.Controller) BlockState - worker *worker - wantWorkerToRetry *worker - err error - }{ - "nil_worker.err_returns_nil": { - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - return NewMockBlockState(ctrl) - }, - worker: &worker{}, - }, - "best_block_header_error": { - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().BestBlockHeader().Return(nil, - mockError) - return mockBlockState - }, - worker: &worker{ - err: &workerError{}, - targetNumber: uintPtr(0), - }, - err: mockError, - }, - "targetNumber_<_bestBlockHeader_number_returns_nil": { - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().BestBlockHeader().Return(&types.Header{Number: 2}, nil) - return mockBlockState - }, - worker: &worker{ - err: &workerError{}, - targetNumber: uintPtr(0), - }, - }, - "targetNumber_>_bestBlockHeader_number_worker_errUnknownParent,_error_GetHighestFinalisedHeader": { - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().BestBlockHeader().Return(&types.Header{Number: 2}, nil) - mockBlockState.EXPECT().GetHighestFinalisedHeader().Return(nil, mockError) - return mockBlockState - }, - worker: &worker{ - err: &workerError{err: errUnknownParent}, - targetNumber: uintPtr(3), - }, - err: mockError, - }, - "targetNumber_>_bestBlockHeader_number_worker_errUnknownParent_returns_worker": { - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().BestBlockHeader().Return(&types.Header{Number: 2}, nil) - mockBlockState.EXPECT().GetHighestFinalisedHeader().Return(&types.Header{Number: 1}, nil) - return mockBlockState - }, - worker: &worker{ - err: &workerError{err: errUnknownParent}, - targetNumber: uintPtr(3), - }, - wantWorkerToRetry: &worker{ - startNumber: uintPtr(1), - targetNumber: uintPtr(3), - }, - }, - "targetNumber_>_bestBlockHeader_number_returns_worker": { - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().BestBlockHeader().Return(&types.Header{Number: 2}, nil) - return mockBlockState - }, - worker: &worker{ - err: &workerError{}, - targetNumber: uintPtr(3), - }, - wantWorkerToRetry: &worker{ - startNumber: uintPtr(3), - targetNumber: uintPtr(3), - }, - }, - } - for testName, tt := range tests { - tt := tt - t.Run(testName, func(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - s := &bootstrapSyncer{ - blockState: tt.blockStateBuilder(ctrl), - } - gotWorkerToRetry, err := s.handleWorkerResult(tt.worker) - assert.ErrorIs(t, err, tt.err) - assert.Equal(t, tt.wantWorkerToRetry, gotWorkerToRetry) - }) - } -} diff --git a/dot/sync/chain_processor.go b/dot/sync/chain_processor.go deleted file mode 100644 index c90e9b7159..0000000000 --- a/dot/sync/chain_processor.go +++ /dev/null @@ -1,300 +0,0 @@ -// Copyright 2021 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package sync - -import ( - "bytes" - "context" - "errors" - "fmt" - - "github.com/ChainSafe/gossamer/dot/telemetry" - "github.com/ChainSafe/gossamer/dot/types" - "github.com/ChainSafe/gossamer/lib/blocktree" -) - -// ChainProcessor processes ready blocks. -// it is implemented by *chainProcessor -type ChainProcessor interface { - processReadyBlocks() - stop() -} - -type chainProcessor struct { - ctx context.Context - cancel context.CancelFunc - - chainSync ChainSync - - // blocks that are ready for processing. ie. their parent is known, or their parent is ahead - // of them within this channel and thus will be processed first - readyBlocks *blockQueue - - // set of block not yet ready to be processed. - // blocks are placed here if they fail to be processed due to missing parent block - pendingBlocks DisjointBlockSet - - blockState BlockState - storageState StorageState - transactionState TransactionState - babeVerifier BabeVerifier - finalityGadget FinalityGadget - blockImportHandler BlockImportHandler - telemetry Telemetry -} - -type chainProcessorConfig struct { - readyBlocks *blockQueue - pendingBlocks DisjointBlockSet - syncer ChainSync - blockState BlockState - storageState StorageState - transactionState TransactionState - babeVerifier BabeVerifier - finalityGadget FinalityGadget - blockImportHandler BlockImportHandler - telemetry Telemetry - badBlocks []string -} - -func newChainProcessor(cfg chainProcessorConfig) *chainProcessor { - ctx, cancel := context.WithCancel(context.Background()) - - return &chainProcessor{ - ctx: ctx, - cancel: cancel, - readyBlocks: cfg.readyBlocks, - pendingBlocks: cfg.pendingBlocks, - chainSync: cfg.syncer, - blockState: cfg.blockState, - storageState: cfg.storageState, - transactionState: cfg.transactionState, - babeVerifier: cfg.babeVerifier, - finalityGadget: cfg.finalityGadget, - blockImportHandler: cfg.blockImportHandler, - telemetry: cfg.telemetry, - } -} - -func (s *chainProcessor) stop() { - s.cancel() -} - -func (s *chainProcessor) processReadyBlocks() { - for { - bd, err := s.readyBlocks.pop(s.ctx) - if err != nil { - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return - } - panic(fmt.Sprintf("unhandled error: %s", err)) - } - - if err := s.processBlockData(*bd); err != nil { - // depending on the error, we might want to save this block for later - if !errors.Is(err, errFailedToGetParent) && !errors.Is(err, blocktree.ErrParentNotFound) { - logger.Errorf("block data processing for block with hash %s failed: %s", bd.Hash, err) - continue - } - - logger.Tracef("block data processing for block with hash %s failed: %s", bd.Hash, err) - if err := s.pendingBlocks.addBlock(&types.Block{ - Header: *bd.Header, - Body: *bd.Body, - }); err != nil { - logger.Debugf("failed to re-add block to pending blocks: %s", err) - } - } - } -} - -// processBlockData processes the BlockData from a BlockResponse and -// returns the index of the last BlockData it handled on success, -// or the index of the block data that errored on failure. -func (c *chainProcessor) processBlockData(blockData types.BlockData) error { - logger.Debugf("processing block data with hash %s", blockData.Hash) - - headerInState, err := c.blockState.HasHeader(blockData.Hash) - if err != nil { - return fmt.Errorf("checking if block state has header: %w", err) - } - - bodyInState, err := c.blockState.HasBlockBody(blockData.Hash) - if err != nil { - return fmt.Errorf("checking if block state has body: %w", err) - } - - // while in bootstrap mode we don't need to broadcast block announcements - announceImportedBlock := c.chainSync.syncState() == tip - if headerInState && bodyInState { - err = c.processBlockDataWithStateHeaderAndBody(blockData, announceImportedBlock) - if err != nil { - return fmt.Errorf("processing block data with header and "+ - "body in block state: %w", err) - } - return nil - } - - if blockData.Header != nil { - if blockData.Body != nil { - err = c.processBlockDataWithHeaderAndBody(blockData, announceImportedBlock) - if err != nil { - return fmt.Errorf("processing block data with header and body: %w", err) - } - logger.Debugf("block with hash %s processed", blockData.Hash) - } - - if blockData.Justification != nil && len(*blockData.Justification) > 0 { - err = c.handleJustification(blockData.Header, *blockData.Justification) - if err != nil { - return fmt.Errorf("handling justification: %w", err) - } - } - } - - err = c.blockState.CompareAndSetBlockData(&blockData) - if err != nil { - return fmt.Errorf("comparing and setting block data: %w", err) - } - - return nil -} - -func (c *chainProcessor) processBlockDataWithStateHeaderAndBody(blockData types.BlockData, - announceImportedBlock bool) (err error) { - // TODO: fix this; sometimes when the node shuts down the "best block" isn't stored properly, - // so when the node restarts it has blocks higher than what it thinks is the best, causing it not to sync - // if we update the node to only store finalised blocks in the database, this should be fixed and the entire - // code block can be removed (#1784) - block, err := c.blockState.GetBlockByHash(blockData.Hash) - if err != nil { - return fmt.Errorf("getting block by hash: %w", err) - } - - err = c.blockState.AddBlockToBlockTree(block) - if errors.Is(err, blocktree.ErrBlockExists) { - logger.Debugf( - "block number %d with hash %s already exists in block tree, skipping it.", - block.Header.Number, blockData.Hash) - return nil - } else if err != nil { - return fmt.Errorf("adding block to blocktree: %w", err) - } - - if blockData.Justification != nil && len(*blockData.Justification) > 0 { - err = c.handleJustification(&block.Header, *blockData.Justification) - if err != nil { - return fmt.Errorf("handling justification: %w", err) - } - } - - // TODO: this is probably unnecessary, since the state is already in the database - // however, this case shouldn't be hit often, since it's only hit if the node state - // is rewinded or if the node shuts down unexpectedly (#1784) - state, err := c.storageState.TrieState(&block.Header.StateRoot) - if err != nil { - return fmt.Errorf("loading trie state: %w", err) - } - - err = c.blockImportHandler.HandleBlockImport(block, state, announceImportedBlock) - if err != nil { - return fmt.Errorf("handling block import: %w", err) - } - - return nil -} - -func (c *chainProcessor) processBlockDataWithHeaderAndBody(blockData types.BlockData, - announceImportedBlock bool) (err error) { - err = c.babeVerifier.VerifyBlock(blockData.Header) - if err != nil { - return fmt.Errorf("babe verifying block: %w", err) - } - - c.handleBody(blockData.Body) - - block := &types.Block{ - Header: *blockData.Header, - Body: *blockData.Body, - } - - err = c.handleBlock(block, announceImportedBlock) - if err != nil { - return fmt.Errorf("handling block: %w", err) - } - - return nil -} - -// handleHeader handles block bodies included in BlockResponses -func (s *chainProcessor) handleBody(body *types.Body) { - for _, ext := range *body { - s.transactionState.RemoveExtrinsic(ext) - } -} - -// handleHeader handles blocks (header+body) included in BlockResponses -func (s *chainProcessor) handleBlock(block *types.Block, announceImportedBlock bool) error { - parent, err := s.blockState.GetHeader(block.Header.ParentHash) - if err != nil { - return fmt.Errorf("%w: %s", errFailedToGetParent, err) - } - - s.storageState.Lock() - defer s.storageState.Unlock() - - ts, err := s.storageState.TrieState(&parent.StateRoot) - if err != nil { - return err - } - - root := ts.MustRoot() - if !bytes.Equal(parent.StateRoot[:], root[:]) { - panic("parent state root does not match snapshot state root") - } - - rt, err := s.blockState.GetRuntime(parent.Hash()) - if err != nil { - return err - } - - rt.SetContextStorage(ts) - - _, err = rt.ExecuteBlock(block) - if err != nil { - return fmt.Errorf("failed to execute block %d: %w", block.Header.Number, err) - } - - if err = s.blockImportHandler.HandleBlockImport(block, ts, announceImportedBlock); err != nil { - return err - } - - logger.Debugf("🔗 imported block number %d with hash %s", block.Header.Number, block.Header.Hash()) - - blockHash := block.Header.Hash() - s.telemetry.SendMessage(telemetry.NewBlockImport( - &blockHash, - block.Header.Number, - "NetworkInitialSync")) - - return nil -} - -func (s *chainProcessor) handleJustification(header *types.Header, justification []byte) (err error) { - logger.Debugf("handling justification for block %d...", header.Number) - - headerHash := header.Hash() - err = s.finalityGadget.VerifyBlockJustification(headerHash, justification) - if err != nil { - return fmt.Errorf("verifying block number %d justification: %w", header.Number, err) - } - - err = s.blockState.SetJustification(headerHash, justification) - if err != nil { - return fmt.Errorf("setting justification for block number %d: %w", header.Number, err) - } - - logger.Infof("🔨 finalised block number %d with hash %s", header.Number, headerHash) - return nil -} diff --git a/dot/sync/chain_processor_integration_test.go b/dot/sync/chain_processor_integration_test.go deleted file mode 100644 index 7164cd75d2..0000000000 --- a/dot/sync/chain_processor_integration_test.go +++ /dev/null @@ -1,348 +0,0 @@ -//go:build integration - -// Copyright 2021 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package sync - -import ( - "errors" - "testing" - "time" - - "github.com/ChainSafe/gossamer/dot/network" - "github.com/ChainSafe/gossamer/dot/state" - "github.com/ChainSafe/gossamer/dot/types" - "github.com/ChainSafe/gossamer/lib/babe/inherents" - "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/common/variadic" - runtime "github.com/ChainSafe/gossamer/lib/runtime" - "github.com/ChainSafe/gossamer/lib/transaction" - "github.com/ChainSafe/gossamer/pkg/scale" - - "github.com/stretchr/testify/require" -) - -func buildBlockWithSlotAndTimestamp(t *testing.T, instance runtime.Instance, - parent *types.Header, currentSlot, timestamp uint64) *types.Block { - t.Helper() - - digest := types.NewDigest() - prd, err := types.NewBabeSecondaryPlainPreDigest(0, currentSlot).ToPreRuntimeDigest() - require.NoError(t, err) - err = digest.Add(*prd) - require.NoError(t, err) - header := &types.Header{ - ParentHash: parent.Hash(), - StateRoot: common.Hash{}, - ExtrinsicsRoot: common.Hash{}, - Number: parent.Number + 1, - Digest: digest, - } - - err = instance.InitializeBlock(header) - require.NoError(t, err) - - inherentData := types.NewInherentData() - err = inherentData.SetInherent(types.Timstap0, timestamp) - require.NoError(t, err) - - err = inherentData.SetInherent(types.Babeslot, currentSlot) - require.NoError(t, err) - - parachainInherent := inherents.ParachainInherentData{ - ParentHeader: *parent, - } - - err = inherentData.SetInherent(types.Parachn0, parachainInherent) - require.NoError(t, err) - - err = inherentData.SetInherent(types.Newheads, []byte{0}) - require.NoError(t, err) - - encodedInherentData, err := inherentData.Encode() - require.NoError(t, err) - - // Call BlockBuilder_inherent_extrinsics which returns the inherents as encoded extrinsics - encodedInherentExtrinsics, err := instance.InherentExtrinsics(encodedInherentData) - require.NoError(t, err) - - var inherentExtrinsics [][]byte - err = scale.Unmarshal(encodedInherentExtrinsics, &inherentExtrinsics) - require.NoError(t, err) - - for _, inherent := range inherentExtrinsics { - encodedInherent, err := scale.Marshal(inherent) - require.NoError(t, err) - - applyExtrinsicResult, err := instance.ApplyExtrinsic(encodedInherent) - require.NoError(t, err) - require.Equal(t, applyExtrinsicResult, []byte{0, 0}) - } - - finalisedHeader, err := instance.FinalizeBlock() - require.NoError(t, err) - - body := types.Body(types.BytesArrayToExtrinsics(inherentExtrinsics)) - - finalisedHeader.Number = header.Number - finalisedHeader.Hash() - - return &types.Block{ - Header: *finalisedHeader, - Body: body, - } -} - -func buildAndAddBlocksToState(t *testing.T, - runtime runtime.Instance, blockState *state.BlockState, amount uint) { - - t.Helper() - - parent, err := blockState.BestBlockHeader() - require.NoError(t, err) - - babeConfig, err := runtime.BabeConfiguration() - require.NoError(t, err) - - timestamp := uint64(time.Now().Unix()) - slotDuration := babeConfig.SlotDuration - - for i := uint(0); i < amount; i++ { - // calculate the exact slot for each produced block - currentSlot := timestamp / slotDuration - - block := buildBlockWithSlotAndTimestamp(t, runtime, parent, currentSlot, timestamp) - err = blockState.AddBlock(block) - require.NoError(t, err) - parent = &block.Header - - // increase the timestamp by the slot duration - // so we will get a different slot for the next block - timestamp += slotDuration - } - -} - -func TestChainProcessor_HandleBlockResponse_ValidChain(t *testing.T) { - syncer := newTestSyncer(t) - responder := newTestSyncer(t) - - bestBlockHash := responder.blockState.(*state.BlockState).BestBlockHash() - runtimeInstance, err := responder.blockState.GetRuntime(bestBlockHash) - require.NoError(t, err) - - buildAndAddBlocksToState(t, runtimeInstance, - responder.blockState.(*state.BlockState), maxResponseSize*2) - - // syncer makes request for chain - startNum := 1 - start, err := variadic.NewUint32OrHash(startNum) - require.NoError(t, err) - - req := &network.BlockRequestMessage{ - RequestedData: network.RequestedDataHeader + network.RequestedDataBody, - StartingBlock: *start, - } - - // get response - resp, err := responder.CreateBlockResponse(req) - require.NoError(t, err) - - // process response - for _, bd := range resp.BlockData { - err = syncer.chainProcessor.(*chainProcessor).processBlockData(*bd) - require.NoError(t, err) - } - - // syncer makes request for chain again (block 129+) - startNum = 129 - start, err = variadic.NewUint32OrHash(startNum) - require.NoError(t, err) - - req = &network.BlockRequestMessage{ - RequestedData: network.RequestedDataHeader + network.RequestedDataBody, - StartingBlock: *start, - } - - // get response - resp, err = responder.CreateBlockResponse(req) - require.NoError(t, err) - - // process response - for _, bd := range resp.BlockData { - err = syncer.chainProcessor.(*chainProcessor).processBlockData(*bd) - require.NoError(t, err) - } -} - -func TestChainProcessor_HandleBlockResponse_MissingBlocks(t *testing.T) { - syncer := newTestSyncer(t) - - bestBlockHash := syncer.blockState.(*state.BlockState).BestBlockHash() - syncerRuntime, err := syncer.blockState.GetRuntime(bestBlockHash) - require.NoError(t, err) - - const syncerAmountOfBlocks = 4 - buildAndAddBlocksToState(t, syncerRuntime, syncer.blockState.(*state.BlockState), syncerAmountOfBlocks) - - responder := newTestSyncer(t) - responderRuntime, err := responder.blockState.GetRuntime(bestBlockHash) - require.NoError(t, err) - - const responderAmountOfBlocks = 16 - buildAndAddBlocksToState(t, responderRuntime, responder.blockState.(*state.BlockState), responderAmountOfBlocks) - - startNum := 15 - start, err := variadic.NewUint32OrHash(startNum) - require.NoError(t, err) - - req := &network.BlockRequestMessage{ - RequestedData: 3, - StartingBlock: *start, - } - - // resp contains blocks 15 to 15 + maxResponseSize) - resp, err := responder.CreateBlockResponse(req) - require.NoError(t, err) - - for _, bd := range resp.BlockData { - err = syncer.chainProcessor.(*chainProcessor).processBlockData(*bd) - require.True(t, errors.Is(err, errFailedToGetParent)) - } -} - -func TestChainProcessor_handleBody_ShouldRemoveIncludedExtrinsics(t *testing.T) { - syncer := newTestSyncer(t) - - ext := []byte("nootwashere") - tx := &transaction.ValidTransaction{ - Extrinsic: ext, - Validity: &transaction.Validity{Priority: 1}, - } - - _, err := syncer.chainProcessor.(*chainProcessor).transactionState.(*state.TransactionState).Push(tx) - require.NoError(t, err) - - body := types.NewBody([]types.Extrinsic{ext}) - syncer.chainProcessor.(*chainProcessor).handleBody(body) - - inQueue := syncer.chainProcessor.(*chainProcessor).transactionState.(*state.TransactionState).Pop() - require.Nil(t, inQueue, "queue should be empty") -} - -func TestChainProcessor_HandleBlockResponse_BlockData(t *testing.T) { - syncer := newTestSyncer(t) - - parent, err := syncer.blockState.(*state.BlockState).BestBlockHeader() - require.NoError(t, err) - - runtimeInstance, err := syncer.blockState.GetRuntime(parent.Hash()) - require.NoError(t, err) - - babeConfig, err := runtimeInstance.BabeConfiguration() - require.NoError(t, err) - - timestamp := uint64(time.Now().Unix()) - slotDuration := babeConfig.SlotDuration - - // calculate the exact slot for each produced block - currentSlot := timestamp / slotDuration - block := buildBlockWithSlotAndTimestamp(t, runtimeInstance, parent, currentSlot, timestamp) - - bd := []*types.BlockData{{ - Hash: block.Header.Hash(), - Header: &block.Header, - Body: &block.Body, - Receipt: nil, - MessageQueue: nil, - Justification: nil, - }} - msg := &network.BlockResponseMessage{ - BlockData: bd, - } - - for _, bd := range msg.BlockData { - err = syncer.chainProcessor.(*chainProcessor).processBlockData(*bd) - require.NoError(t, err) - } -} - -func TestChainProcessor_ExecuteBlock(t *testing.T) { - syncer := newTestSyncer(t) - - parent, err := syncer.blockState.(*state.BlockState).BestBlockHeader() - require.NoError(t, err) - - bestBlockHash := syncer.blockState.(*state.BlockState).BestBlockHash() - runtimeInstance, err := syncer.blockState.GetRuntime(bestBlockHash) - require.NoError(t, err) - - babeConfig, err := runtimeInstance.BabeConfiguration() - require.NoError(t, err) - - timestamp := uint64(time.Now().Unix()) - slotDuration := babeConfig.SlotDuration - - // calculate the exact slot for each produced block - currentSlot := timestamp / slotDuration - block := buildBlockWithSlotAndTimestamp(t, runtimeInstance, parent, currentSlot, timestamp) - - // reset parentState - parentState, err := syncer.chainProcessor.(*chainProcessor).storageState.TrieState(&parent.StateRoot) - require.NoError(t, err) - runtimeInstance.SetContextStorage(parentState) - - _, err = runtimeInstance.ExecuteBlock(block) - require.NoError(t, err) -} - -func TestChainProcessor_HandleJustification(t *testing.T) { - syncer := newTestSyncer(t) - - d, err := types.NewBabeSecondaryPlainPreDigest(0, 1).ToPreRuntimeDigest() - require.NoError(t, err) - digest := types.NewDigest() - err = digest.Add(*d) - require.NoError(t, err) - - header := &types.Header{ - ParentHash: syncer.blockState.(*state.BlockState).GenesisHash(), - Number: 1, - Digest: digest, - } - - just := []byte("testjustification") - - err = syncer.blockState.(*state.BlockState).AddBlock(&types.Block{ - Header: *header, - Body: types.Body{}, - }) - require.NoError(t, err) - - err = syncer.chainProcessor.(*chainProcessor).handleJustification(header, just) - require.NoError(t, err) - - res, err := syncer.blockState.GetJustification(header.Hash()) - require.NoError(t, err) - require.Equal(t, just, res) -} - -func TestChainProcessor_processReadyBlocks_errFailedToGetParent(t *testing.T) { - syncer := newTestSyncer(t) - processor := syncer.chainProcessor.(*chainProcessor) - go processor.processReadyBlocks() - defer processor.cancel() - - header := &types.Header{ - Number: 1, - } - - processor.readyBlocks.push(&types.BlockData{ - Header: header, - Body: &types.Body{}, - }) - - time.Sleep(time.Millisecond * 100) - require.True(t, processor.pendingBlocks.(*disjointBlockSet).hasBlock(header.Hash())) -} diff --git a/dot/sync/chain_processor_test.go b/dot/sync/chain_processor_test.go deleted file mode 100644 index 8e794767ab..0000000000 --- a/dot/sync/chain_processor_test.go +++ /dev/null @@ -1,1181 +0,0 @@ -// Copyright 2021 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package sync - -import ( - "context" - "errors" - "testing" - - "github.com/ChainSafe/gossamer/dot/telemetry" - "github.com/ChainSafe/gossamer/dot/types" - "github.com/ChainSafe/gossamer/lib/blocktree" - "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/runtime/storage" - "github.com/ChainSafe/gossamer/lib/trie" - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" -) - -func Test_chainProcessor_handleBlock(t *testing.T) { - t.Parallel() - mockError := errors.New("test mock error") - testHash := common.MustHexToHash("0x03170a2e7597b7b7e3d84c05391d139a62b157e78786d8c082f29dcf4c111314") - testParentHash := common.MustHexToHash("0x7db9db5ed9967b80143100189ba69d9e4deab85ac3570e5df25686cabe32964a") - - tests := map[string]struct { - chainProcessorBuilder func(ctrl *gomock.Controller) chainProcessor - block *types.Block - announce bool - wantErr error - }{ - "handle_getHeader_error": { - chainProcessorBuilder: func(ctrl *gomock.Controller) (chainProcessor chainProcessor) { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().GetHeader(common.Hash{}).Return(nil, mockError) - chainProcessor.blockState = mockBlockState - return - }, - block: &types.Block{ - Body: types.Body{}, - }, - wantErr: errFailedToGetParent, - }, - "handle_trieState_error": { - chainProcessorBuilder: func(ctrl *gomock.Controller) (chainProcessor chainProcessor) { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().GetHeader(common.Hash{}).Return(&types.Header{}, nil) - chainProcessor.blockState = mockBlockState - mockStorageState := NewMockStorageState(ctrl) - mockStorageState.EXPECT().Lock() - mockStorageState.EXPECT().TrieState(&common.Hash{}).Return(nil, mockError) - mockStorageState.EXPECT().Unlock() - chainProcessor.storageState = mockStorageState - return - }, - block: &types.Block{ - Body: types.Body{}, - }, - wantErr: mockError, - }, - "handle_getRuntime_error": { - chainProcessorBuilder: func(ctrl *gomock.Controller) (chainProcessor chainProcessor) { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().GetHeader(common.Hash{}).Return(&types.Header{ - StateRoot: testHash, - }, nil) - mockBlockState.EXPECT().GetRuntime(testParentHash).Return(nil, mockError) - chainProcessor.blockState = mockBlockState - trieState := storage.NewTrieState(nil) - mockStorageState := NewMockStorageState(ctrl) - mockStorageState.EXPECT().Lock() - mockStorageState.EXPECT().TrieState(&testHash).Return(trieState, nil) - mockStorageState.EXPECT().Unlock() - chainProcessor.storageState = mockStorageState - return - }, - block: &types.Block{ - Body: types.Body{}, - }, - wantErr: mockError, - }, - "handle_runtime_ExecuteBlock_error": { - chainProcessorBuilder: func(ctrl *gomock.Controller) (chainProcessor chainProcessor) { - trieState := storage.NewTrieState(nil) - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().GetHeader(common.Hash{}).Return(&types.Header{ - StateRoot: testHash, - }, nil) - mockInstance := NewMockInstance(ctrl) - mockInstance.EXPECT().SetContextStorage(trieState) - mockInstance.EXPECT().ExecuteBlock(&types.Block{Body: types.Body{}}).Return(nil, mockError) - mockBlockState.EXPECT().GetRuntime(testParentHash).Return(mockInstance, nil) - chainProcessor.blockState = mockBlockState - mockStorageState := NewMockStorageState(ctrl) - mockStorageState.EXPECT().Lock() - mockStorageState.EXPECT().TrieState(&testHash).Return(trieState, nil) - mockStorageState.EXPECT().Unlock() - chainProcessor.storageState = mockStorageState - return - }, - block: &types.Block{ - Body: types.Body{}, - }, - wantErr: mockError, - }, - "handle_block_import_error": { - chainProcessorBuilder: func(ctrl *gomock.Controller) (chainProcessor chainProcessor) { - trieState := storage.NewTrieState(nil) - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().GetHeader(common.Hash{}).Return(&types.Header{ - StateRoot: testHash, - }, nil) - mockBlock := &types.Block{Body: types.Body{}} - mockInstance := NewMockInstance(ctrl) - mockInstance.EXPECT().SetContextStorage(trieState) - mockInstance.EXPECT().ExecuteBlock(mockBlock).Return(nil, nil) - mockBlockState.EXPECT().GetRuntime(testParentHash).Return(mockInstance, nil) - chainProcessor.blockState = mockBlockState - mockStorageState := NewMockStorageState(ctrl) - mockStorageState.EXPECT().Lock() - mockStorageState.EXPECT().TrieState(&testHash).Return(trieState, nil) - mockStorageState.EXPECT().Unlock() - chainProcessor.storageState = mockStorageState - mockBlockImportHandler := NewMockBlockImportHandler(ctrl) - mockBlockImportHandler.EXPECT().HandleBlockImport(mockBlock, - trieState, false).Return(mockError) - chainProcessor.blockImportHandler = mockBlockImportHandler - return - }, - block: &types.Block{ - Body: types.Body{}, - }, - wantErr: mockError, - }, - "base_case": { - chainProcessorBuilder: func(ctrl *gomock.Controller) (chainProcessor chainProcessor) { - mockBlock := &types.Block{ - Body: types.Body{}, // empty slice of extrinsics - } - trieState := storage.NewTrieState(nil) - mockBlockState := NewMockBlockState(ctrl) - mockHeader := &types.Header{ - Number: 0, - StateRoot: trie.EmptyHash, - } - mockHeaderHash := mockHeader.Hash() - mockBlockState.EXPECT().GetHeader(common.Hash{}).Return(mockHeader, nil) - - mockInstance := NewMockInstance(ctrl) - mockInstance.EXPECT().SetContextStorage(trieState) - mockInstance.EXPECT().ExecuteBlock(mockBlock).Return(nil, nil) - mockBlockState.EXPECT().GetRuntime(mockHeaderHash).Return(mockInstance, nil) - chainProcessor.blockState = mockBlockState - mockStorageState := NewMockStorageState(ctrl) - mockStorageState.EXPECT().Lock() - mockStorageState.EXPECT().Unlock() - mockStorageState.EXPECT().TrieState(&trie.EmptyHash).Return(trieState, nil) - chainProcessor.storageState = mockStorageState - mockBlockImportHandler := NewMockBlockImportHandler(ctrl) - mockBlockImportHandler.EXPECT().HandleBlockImport(mockBlock, trieState, false).Return(nil) - chainProcessor.blockImportHandler = mockBlockImportHandler - mockTelemetry := NewMockTelemetry(ctrl) - mockTelemetry.EXPECT().SendMessage(gomock.Any()) - chainProcessor.telemetry = mockTelemetry - return - }, - block: &types.Block{ - Header: types.Header{ - Number: 0, - }, - Body: types.Body{}, - }, - }, - "import_block_and_announce": { - announce: true, - chainProcessorBuilder: func(ctrl *gomock.Controller) (chainProcessor chainProcessor) { - mockBlock := &types.Block{ - Body: types.Body{}, // empty slice of extrinsics - } - trieState := storage.NewTrieState(nil) - mockBlockState := NewMockBlockState(ctrl) - mockHeader := &types.Header{ - Number: 0, - StateRoot: trie.EmptyHash, - } - mockHeaderHash := mockHeader.Hash() - mockBlockState.EXPECT().GetHeader(common.Hash{}).Return(mockHeader, nil) - - mockInstance := NewMockInstance(ctrl) - mockInstance.EXPECT().SetContextStorage(trieState) - mockInstance.EXPECT().ExecuteBlock(mockBlock).Return(nil, nil) - mockBlockState.EXPECT().GetRuntime(mockHeaderHash).Return(mockInstance, nil) - chainProcessor.blockState = mockBlockState - mockStorageState := NewMockStorageState(ctrl) - mockStorageState.EXPECT().Lock() - mockStorageState.EXPECT().Unlock() - mockStorageState.EXPECT().TrieState(&trie.EmptyHash).Return(trieState, nil) - chainProcessor.storageState = mockStorageState - mockBlockImportHandler := NewMockBlockImportHandler(ctrl) - mockBlockImportHandler.EXPECT().HandleBlockImport(mockBlock, trieState, true).Return(nil) - chainProcessor.blockImportHandler = mockBlockImportHandler - mockTelemetry := NewMockTelemetry(ctrl) - mockTelemetry.EXPECT().SendMessage(gomock.Any()) - chainProcessor.telemetry = mockTelemetry - return - }, - block: &types.Block{ - Header: types.Header{ - Number: 0, - }, - Body: types.Body{}, - }, - }, - } - for name, tt := range tests { - tt := tt - t.Run(name, func(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - s := tt.chainProcessorBuilder(ctrl) - - err := s.handleBlock(tt.block, tt.announce) - assert.ErrorIs(t, err, tt.wantErr) - }) - } - t.Run("panics_on_different_parent_state_root", func(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - bock := &types.Block{ - Header: types.Header{ - ParentHash: common.Hash{1}, - }, - } - blockState := NewMockBlockState(ctrl) - blockState.EXPECT().GetHeader(common.Hash{1}). - Return(&types.Header{StateRoot: common.Hash{2}}, nil) - trieState := storage.NewTrieState(nil) - storageState := NewMockStorageState(ctrl) - lockCall := storageState.EXPECT().Lock() - trieStateCall := storageState.EXPECT().TrieState(&common.Hash{2}). - Return(trieState, nil).After(lockCall) - storageState.EXPECT().Unlock().After(trieStateCall) - chainProcessor := &chainProcessor{ - blockState: blockState, - storageState: storageState, - } - const expectedPanicValue = "parent state root does not match snapshot state root" - assert.PanicsWithValue(t, expectedPanicValue, func() { - _ = chainProcessor.handleBlock(bock, false) - }) - }) -} - -func Test_chainProcessor_handleBody(t *testing.T) { - t.Parallel() - - testExtrinsics := []types.Extrinsic{{1, 2, 3}, {7, 8, 9, 0}, {0xa, 0xb}} - testBody := types.NewBody(testExtrinsics) - - t.Run("base_case", func(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - - mockTransactionState := NewMockTransactionState(ctrl) - mockTransactionState.EXPECT().RemoveExtrinsic(testExtrinsics[0]) - mockTransactionState.EXPECT().RemoveExtrinsic(testExtrinsics[1]) - mockTransactionState.EXPECT().RemoveExtrinsic(testExtrinsics[2]) - processor := chainProcessor{ - transactionState: mockTransactionState, - } - processor.handleBody(testBody) - }) -} - -func Test_chainProcessor_handleJustification(t *testing.T) { - t.Parallel() - - header := &types.Header{ - Number: 2, - } - headerHash := header.Hash() - errTest := errors.New("test error") - - type args struct { - header *types.Header - justification []byte - } - tests := map[string]struct { - chainProcessorBuilder func(ctrl *gomock.Controller) chainProcessor - args args - sentinelError error - errorMessage string - }{ - "invalid_justification": { - chainProcessorBuilder: func(ctrl *gomock.Controller) chainProcessor { - mockFinalityGadget := NewMockFinalityGadget(ctrl) - mockFinalityGadget.EXPECT().VerifyBlockJustification(headerHash, - []byte(`x`)).Return(errTest) - return chainProcessor{ - finalityGadget: mockFinalityGadget, - } - }, - args: args{ - header: header, - justification: []byte(`x`), - }, - sentinelError: errTest, - errorMessage: "verifying block number 2 justification: test error", - }, - "set_justification_error": { - chainProcessorBuilder: func(ctrl *gomock.Controller) chainProcessor { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().SetJustification(headerHash, []byte(`xx`)).Return(errTest) - mockFinalityGadget := NewMockFinalityGadget(ctrl) - mockFinalityGadget.EXPECT().VerifyBlockJustification(headerHash, []byte(`xx`)).Return(nil) - return chainProcessor{ - blockState: mockBlockState, - finalityGadget: mockFinalityGadget, - } - }, - args: args{ - header: header, - justification: []byte(`xx`), - }, - sentinelError: errTest, - errorMessage: "setting justification for block number 2: test error", - }, - "base_case_set": { - chainProcessorBuilder: func(ctrl *gomock.Controller) chainProcessor { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().SetJustification(headerHash, []byte(`1234`)).Return(nil) - mockFinalityGadget := NewMockFinalityGadget(ctrl) - mockFinalityGadget.EXPECT().VerifyBlockJustification(headerHash, []byte(`1234`)).Return(nil) - return chainProcessor{ - blockState: mockBlockState, - finalityGadget: mockFinalityGadget, - } - }, - args: args{ - header: header, - justification: []byte(`1234`), - }, - }, - } - for name, tt := range tests { - tt := tt - t.Run(name, func(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - - processor := tt.chainProcessorBuilder(ctrl) - - err := processor.handleJustification(tt.args.header, tt.args.justification) - - assert.ErrorIs(t, err, tt.sentinelError) - if tt.sentinelError != nil { - assert.EqualError(t, err, tt.errorMessage) - } - }) - } -} - -func Test_chainProcessor_processBlockData(t *testing.T) { - t.Parallel() - - mockError := errors.New("mock test error") - - tests := map[string]struct { - chainProcessorBuilder func(ctrl *gomock.Controller) chainProcessor - blockData types.BlockData - expectedError error - }{ - "handle_has_header_error": { - chainProcessorBuilder: func(ctrl *gomock.Controller) chainProcessor { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().HasHeader(common.Hash{}).Return(false, mockError) - - return chainProcessor{ - blockState: mockBlockState, - } - }, - blockData: types.BlockData{}, - expectedError: mockError, - }, - "handle_has_block_body_error": { - chainProcessorBuilder: func(ctrl *gomock.Controller) chainProcessor { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().HasHeader(common.Hash{}).Return(false, nil) - mockBlockState.EXPECT().HasBlockBody(common.Hash{}).Return(false, mockError) - return chainProcessor{ - blockState: mockBlockState, - } - }, - blockData: types.BlockData{}, - expectedError: mockError, - }, - "handle_getBlockByHash_error": { - chainProcessorBuilder: func(ctrl *gomock.Controller) chainProcessor { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().HasHeader(common.Hash{}).Return(true, nil) - mockBlockState.EXPECT().HasBlockBody(common.Hash{}).Return(true, nil) - mockBlockState.EXPECT().GetBlockByHash(common.Hash{}).Return(nil, mockError) - - mockChainSync := NewMockChainSync(ctrl) - mockChainSync.EXPECT().syncState().Return(bootstrap) - return chainProcessor{ - blockState: mockBlockState, - chainSync: mockChainSync, - } - }, - blockData: types.BlockData{}, - expectedError: mockError, - }, - "handle_block_data_justification_!=_nil": { - chainProcessorBuilder: func(ctrl *gomock.Controller) chainProcessor { - mockBlock := &types.Block{ - Header: types.Header{ - Number: uint(1), - }, - } - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().HasHeader(common.Hash{}).Return(true, nil) - mockBlockState.EXPECT().HasBlockBody(common.Hash{}).Return(true, nil) - mockBlockState.EXPECT().GetBlockByHash(common.Hash{}).Return(mockBlock, nil) - mockBlockState.EXPECT().AddBlockToBlockTree(&types.Block{ - Header: types.Header{Number: 1}}).Return(nil) - mockBlockState.EXPECT().SetJustification(common.MustHexToHash( - "0x6443a0b46e0412e626363028115a9f2cf963eeed526b8b33e5316f08b50d0dc3"), []byte{1, 2, 3}) - mockFinalityGadget := NewMockFinalityGadget(ctrl) - mockFinalityGadget.EXPECT().VerifyBlockJustification(common.MustHexToHash( - "0x6443a0b46e0412e626363028115a9f2cf963eeed526b8b33e5316f08b50d0dc3"), []byte{1, 2, - 3}).Return(nil) - mockStorageState := NewMockStorageState(ctrl) - mockStorageState.EXPECT().TrieState(&common.Hash{}).Return(nil, nil) - - // given our current chain sync state is `tip` - // the `HandleBlockImport` method should expect - // true as the announce parameter - mockChainSync := NewMockChainSync(ctrl) - mockChainSync.EXPECT().syncState().Return(tip) - - mockBlockImportHandler := NewMockBlockImportHandler(ctrl) - mockBlockImportHandler.EXPECT().HandleBlockImport(mockBlock, - nil, true).Return(nil) - - return chainProcessor{ - chainSync: mockChainSync, - blockState: mockBlockState, - finalityGadget: mockFinalityGadget, - storageState: mockStorageState, - blockImportHandler: mockBlockImportHandler, - } - }, - blockData: types.BlockData{ - Justification: &[]byte{1, 2, 3}, - }, - }, - "handle_babe_verify_block_error": { - chainProcessorBuilder: func(ctrl *gomock.Controller) chainProcessor { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().HasHeader(common.Hash{}).Return(false, nil) - mockBlockState.EXPECT().HasBlockBody(common.Hash{}).Return(false, nil) - mockBabeVerifier := NewMockBabeVerifier(ctrl) - mockBabeVerifier.EXPECT().VerifyBlock(&types.Header{}).Return(mockError) - - mockChainSync := NewMockChainSync(ctrl) - mockChainSync.EXPECT().syncState().Return(bootstrap) - - return chainProcessor{ - chainSync: mockChainSync, - blockState: mockBlockState, - babeVerifier: mockBabeVerifier, - } - }, - blockData: types.BlockData{ - Header: &types.Header{}, - Body: &types.Body{}, - }, - expectedError: mockError, - }, - "no_header_and_body_-_fail_to_handle_justification": { - chainProcessorBuilder: func(ctrl *gomock.Controller) chainProcessor { - blockState := NewMockBlockState(ctrl) - blockState.EXPECT().HasHeader(common.Hash{1}).Return(false, nil) - blockState.EXPECT().HasBlockBody(common.Hash{1}).Return(true, nil) - - finalityGadget := NewMockFinalityGadget(ctrl) - expectedBlockDataHeader := &types.Header{Number: 2} - expectedBlockDataHeaderHash := expectedBlockDataHeader.Hash() - finalityGadget.EXPECT(). - VerifyBlockJustification(expectedBlockDataHeaderHash, []byte{1, 2, 3}). - Return(mockError) - - mockChainSync := NewMockChainSync(ctrl) - mockChainSync.EXPECT().syncState().Return(bootstrap) - - return chainProcessor{ - chainSync: mockChainSync, - blockState: blockState, - finalityGadget: finalityGadget, - } - }, - blockData: types.BlockData{ - Hash: common.Hash{1}, - Header: &types.Header{Number: 2}, - Justification: &[]byte{1, 2, 3}, - }, - expectedError: mockError, - }, - "handle_compareAndSetBlockData_error": { - chainProcessorBuilder: func(ctrl *gomock.Controller) chainProcessor { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().HasHeader(common.Hash{}).Return(false, nil) - mockBlockState.EXPECT().HasBlockBody(common.Hash{}).Return(false, nil) - mockBlockState.EXPECT().CompareAndSetBlockData(&types.BlockData{}).Return(mockError) - - mockChainSync := NewMockChainSync(ctrl) - mockChainSync.EXPECT().syncState().Return(bootstrap) - return chainProcessor{ - chainSync: mockChainSync, - blockState: mockBlockState, - } - }, - blockData: types.BlockData{}, - expectedError: mockError, - }, - "success_with_justification": { - chainProcessorBuilder: func(ctrl *gomock.Controller) chainProcessor { - stateRootHash := common.MustHexToHash("0x03170a2e7597b7b7e3d84c05391d139a62b157e78786d8c082f29dcf4c111314") - runtimeHash := common.MustHexToHash("0x7db9db5ed9967b80143100189ba69d9e4deab85ac3570e5df25686cabe32964a") - mockTrieState := storage.NewTrieState(nil) - mockBlock := &types.Block{Header: types.Header{}, Body: types.Body{}} - - mockInstance := NewMockInstance(ctrl) - mockInstance.EXPECT().SetContextStorage(mockTrieState) - mockInstance.EXPECT().ExecuteBlock(mockBlock).Return(nil, nil) - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().HasHeader(common.Hash{}).Return(false, nil) - mockBlockState.EXPECT().HasBlockBody(common.Hash{}).Return(false, nil) - mockBlockState.EXPECT().GetHeader(common.Hash{}).Return(&types.Header{ - Number: 0, - StateRoot: stateRootHash, - }, nil) - mockBlockState.EXPECT().SetJustification( - common.MustHexToHash("0xdcdd89927d8a348e00257e1ecc8617f45edb5118efff3ea2f9961b2ad9b7690a"), []byte{1, 2, 3}) - mockBlockState.EXPECT().CompareAndSetBlockData(gomock.AssignableToTypeOf(&types.BlockData{})) - mockBlockState.EXPECT().GetRuntime(runtimeHash).Return(mockInstance, nil) - mockBabeVerifier := NewMockBabeVerifier(ctrl) - mockBabeVerifier.EXPECT().VerifyBlock(&types.Header{}) - mockStorageState := NewMockStorageState(ctrl) - mockStorageState.EXPECT().Lock() - mockStorageState.EXPECT().TrieState(&stateRootHash).Return(mockTrieState, nil) - mockStorageState.EXPECT().Unlock() - - mockChainSync := NewMockChainSync(ctrl) - mockChainSync.EXPECT().syncState().Return(bootstrap) - - mockBlockImportHandler := NewMockBlockImportHandler(ctrl) - mockBlockImportHandler.EXPECT().HandleBlockImport(mockBlock, mockTrieState, false) - - mockTelemetry := NewMockTelemetry(ctrl) - mockTelemetry.EXPECT().SendMessage(gomock.Any()) - mockFinalityGadget := NewMockFinalityGadget(ctrl) - mockFinalityGadget.EXPECT().VerifyBlockJustification( - common.MustHexToHash("0xdcdd89927d8a348e00257e1ecc8617f45edb5118efff3ea2f9961b2ad9b7690a"), - []byte{1, 2, 3}).Return(nil) - return chainProcessor{ - chainSync: mockChainSync, - blockState: mockBlockState, - babeVerifier: mockBabeVerifier, - storageState: mockStorageState, - blockImportHandler: mockBlockImportHandler, - telemetry: mockTelemetry, - finalityGadget: mockFinalityGadget, - } - }, - blockData: types.BlockData{ - Header: &types.Header{ - Number: 0, - }, - Body: &types.Body{}, - Justification: &[]byte{1, 2, 3}, - }, - }, - } - - for name, tt := range tests { - tt := tt - t.Run(name, func(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - processor := tt.chainProcessorBuilder(ctrl) - err := processor.processBlockData(tt.blockData) - assert.ErrorIs(t, err, tt.expectedError) - }) - } -} - -func Test_chainProcessor_processBlockDataWithStateHeaderAndBody(t *testing.T) { - t.Parallel() - - errTest := errors.New("test error") - - testCases := map[string]struct { - chainProcessorBuilder func(ctrl *gomock.Controller) chainProcessor - blockData types.BlockData - announceImportedBlock bool - sentinelError error - errorMessage string - }{ - "get_block_by_hash_error": { - chainProcessorBuilder: func(ctrl *gomock.Controller) chainProcessor { - blockState := NewMockBlockState(ctrl) - blockState.EXPECT().GetBlockByHash(common.Hash{1}). - Return(nil, errTest) - return chainProcessor{ - blockState: blockState, - } - }, - blockData: types.BlockData{Hash: common.Hash{1}}, - sentinelError: errTest, - errorMessage: "getting block by hash: test error", - }, - "block_already_exists_in_blocktree": { - chainProcessorBuilder: func(ctrl *gomock.Controller) chainProcessor { - blockState := NewMockBlockState(ctrl) - block := &types.Block{Header: types.Header{Number: 2}} - blockState.EXPECT().GetBlockByHash(common.Hash{1}).Return(block, nil) - blockState.EXPECT().AddBlockToBlockTree(block).Return(blocktree.ErrBlockExists) - return chainProcessor{ - blockState: blockState, - } - }, - blockData: types.BlockData{Hash: common.Hash{1}}, - }, - "add_block_to_blocktree_error": { - chainProcessorBuilder: func(ctrl *gomock.Controller) chainProcessor { - blockState := NewMockBlockState(ctrl) - block := &types.Block{Header: types.Header{Number: 2}} - blockState.EXPECT().GetBlockByHash(common.Hash{1}).Return(block, nil) - blockState.EXPECT().AddBlockToBlockTree(block).Return(errTest) - return chainProcessor{ - blockState: blockState, - } - }, - blockData: types.BlockData{Hash: common.Hash{1}}, - sentinelError: errTest, - errorMessage: "adding block to blocktree: test error", - }, - "handle_justification_error": { - chainProcessorBuilder: func(ctrl *gomock.Controller) chainProcessor { - blockState := NewMockBlockState(ctrl) - blockHeader := types.Header{Number: 2} - blockHeaderHash := blockHeader.Hash() - block := &types.Block{Header: blockHeader} - blockState.EXPECT().GetBlockByHash(common.Hash{1}).Return(block, nil) - blockState.EXPECT().AddBlockToBlockTree(block).Return(nil) - - finalityGadget := NewMockFinalityGadget(ctrl) - finalityGadget.EXPECT(). - VerifyBlockJustification(blockHeaderHash, []byte{3}). - Return(errTest) - - return chainProcessor{ - blockState: blockState, - finalityGadget: finalityGadget, - } - }, - blockData: types.BlockData{ - Hash: common.Hash{1}, - Justification: &[]byte{3}, - }, - sentinelError: errTest, - errorMessage: "handling justification: verifying block number 2 justification: test error", - }, - "trie_state_error": { - chainProcessorBuilder: func(ctrl *gomock.Controller) chainProcessor { - blockState := NewMockBlockState(ctrl) - blockHeader := types.Header{StateRoot: common.Hash{2}} - block := &types.Block{Header: blockHeader} - blockState.EXPECT().GetBlockByHash(common.Hash{1}).Return(block, nil) - blockState.EXPECT().AddBlockToBlockTree(block).Return(nil) - - storageState := NewMockStorageState(ctrl) - storageState.EXPECT().TrieState(&common.Hash{2}). - Return(nil, errTest) - - return chainProcessor{ - blockState: blockState, - storageState: storageState, - } - }, - blockData: types.BlockData{ - Hash: common.Hash{1}, - }, - sentinelError: errTest, - errorMessage: "loading trie state: test error", - }, - "handle_block_import_error": { - chainProcessorBuilder: func(ctrl *gomock.Controller) chainProcessor { - blockState := NewMockBlockState(ctrl) - blockHeader := types.Header{StateRoot: common.Hash{2}} - block := &types.Block{Header: blockHeader} - blockState.EXPECT().GetBlockByHash(common.Hash{1}).Return(block, nil) - blockState.EXPECT().AddBlockToBlockTree(block).Return(nil) - - storageState := NewMockStorageState(ctrl) - trieState := storage.NewTrieState(nil) - storageState.EXPECT().TrieState(&common.Hash{2}). - Return(trieState, nil) - - blockImportHandler := NewMockBlockImportHandler(ctrl) - const announceImportedBlock = true - blockImportHandler.EXPECT().HandleBlockImport(block, trieState, announceImportedBlock). - Return(errTest) - - return chainProcessor{ - blockState: blockState, - storageState: storageState, - blockImportHandler: blockImportHandler, - } - }, - blockData: types.BlockData{ - Hash: common.Hash{1}, - }, - announceImportedBlock: true, - sentinelError: errTest, - errorMessage: "handling block import: test error", - }, - "success": { - chainProcessorBuilder: func(ctrl *gomock.Controller) chainProcessor { - blockState := NewMockBlockState(ctrl) - blockHeader := types.Header{StateRoot: common.Hash{2}} - block := &types.Block{Header: blockHeader} - blockState.EXPECT().GetBlockByHash(common.Hash{1}).Return(block, nil) - blockState.EXPECT().AddBlockToBlockTree(block).Return(nil) - - storageState := NewMockStorageState(ctrl) - trieState := storage.NewTrieState(nil) - storageState.EXPECT().TrieState(&common.Hash{2}). - Return(trieState, nil) - - blockImportHandler := NewMockBlockImportHandler(ctrl) - const announceImportedBlock = true - blockImportHandler.EXPECT().HandleBlockImport(block, trieState, announceImportedBlock). - Return(nil) - - return chainProcessor{ - blockState: blockState, - storageState: storageState, - blockImportHandler: blockImportHandler, - } - }, - blockData: types.BlockData{ - Hash: common.Hash{1}, - }, - announceImportedBlock: true, - }, - } - - for name, testCase := range testCases { - testCase := testCase - t.Run(name, func(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - - processor := testCase.chainProcessorBuilder(ctrl) - - err := processor.processBlockDataWithStateHeaderAndBody( - testCase.blockData, testCase.announceImportedBlock) - - assert.ErrorIs(t, err, testCase.sentinelError) - if testCase.sentinelError != nil { - assert.EqualError(t, err, testCase.errorMessage) - } - }) - } -} - -func Test_chainProcessor_processBlockDataWithHeaderAndBody(t *testing.T) { - t.Parallel() - - errTest := errors.New("test error") - - testCases := map[string]struct { - chainProcessorBuilder func(ctrl *gomock.Controller) chainProcessor - blockData types.BlockData - announceImportedBlock bool - sentinelError error - errorMessage string - }{ - "verify_block_error": { - chainProcessorBuilder: func(ctrl *gomock.Controller) chainProcessor { - babeVerifier := NewMockBabeVerifier(ctrl) - babeVerifier.EXPECT().VerifyBlock(&types.Header{Number: 1}). - Return(errTest) - - return chainProcessor{ - babeVerifier: babeVerifier, - } - }, - blockData: types.BlockData{ - Header: &types.Header{Number: 1}, - }, - sentinelError: errTest, - errorMessage: "babe verifying block: test error", - }, - "handle_block_error": { - chainProcessorBuilder: func(ctrl *gomock.Controller) chainProcessor { - babeVerifier := NewMockBabeVerifier(ctrl) - expectedHeader := &types.Header{ParentHash: common.Hash{1}} - babeVerifier.EXPECT().VerifyBlock(expectedHeader). - Return(nil) - - transactionState := NewMockTransactionState(ctrl) - transactionState.EXPECT().RemoveExtrinsic(types.Extrinsic{2}) - - blockState := NewMockBlockState(ctrl) - blockState.EXPECT().GetHeader(common.Hash{1}). - Return(nil, errTest) - - return chainProcessor{ - babeVerifier: babeVerifier, - transactionState: transactionState, - blockState: blockState, - } - }, - blockData: types.BlockData{ - Header: &types.Header{ParentHash: common.Hash{1}}, - Body: &types.Body{{2}}, - }, - sentinelError: errFailedToGetParent, - errorMessage: "handling block: failed to get parent header: test error", - }, - "success": { - chainProcessorBuilder: func(ctrl *gomock.Controller) chainProcessor { - babeVerifier := NewMockBabeVerifier(ctrl) - expectedHeader := &types.Header{ - ParentHash: common.Hash{1}, - Number: 5, - } - babeVerifier.EXPECT().VerifyBlock(expectedHeader). - Return(nil) - - transactionState := NewMockTransactionState(ctrl) - transactionState.EXPECT().RemoveExtrinsic(types.Extrinsic{2}) - - blockState := NewMockBlockState(ctrl) - parentHeader := &types.Header{StateRoot: trie.EmptyHash} - blockState.EXPECT().GetHeader(common.Hash{1}). - Return(parentHeader, nil) - - storageState := NewMockStorageState(ctrl) - lockCall := storageState.EXPECT().Lock() - storageState.EXPECT().Unlock().After(lockCall) - trieState := storage.NewTrieState(nil) - storageState.EXPECT().TrieState(&trie.EmptyHash). - Return(trieState, nil) - - parentHeaderHash := parentHeader.Hash() - instance := NewMockInstance(ctrl) - blockState.EXPECT().GetRuntime(parentHeaderHash). - Return(instance, nil) - - instance.EXPECT().SetContextStorage(trieState) - block := &types.Block{ - Header: *expectedHeader, - Body: types.Body{{2}}, - } - instance.EXPECT().ExecuteBlock(block).Return(nil, nil) - - blockImportHandler := NewMockBlockImportHandler(ctrl) - const announceImportedBlock = true - blockImportHandler.EXPECT().HandleBlockImport(block, trieState, announceImportedBlock). - Return(nil) - - telemetryClient := NewMockTelemetry(ctrl) - headerHash := common.MustHexToHash("0x18d21d2901e4a4ac6a8c6431da2dfee1b8701f31a9e49283a082e6c744d4117c") - message := telemetry.NewBlockImport(&headerHash, expectedHeader.Number, "NetworkInitialSync") - telemetryClient.EXPECT().SendMessage(message) - - return chainProcessor{ - babeVerifier: babeVerifier, - transactionState: transactionState, - blockState: blockState, - storageState: storageState, - blockImportHandler: blockImportHandler, - telemetry: telemetryClient, - } - }, - blockData: types.BlockData{ - Header: &types.Header{ - ParentHash: common.Hash{1}, - Number: 5, - }, - Body: &types.Body{{2}}, - }, - announceImportedBlock: true, - }, - } - - for name, testCase := range testCases { - testCase := testCase - t.Run(name, func(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - - processor := testCase.chainProcessorBuilder(ctrl) - - err := processor.processBlockDataWithHeaderAndBody( - testCase.blockData, testCase.announceImportedBlock) - - assert.ErrorIs(t, err, testCase.sentinelError) - if testCase.sentinelError != nil { - assert.EqualError(t, err, testCase.errorMessage) - } - }) - } -} - -func Test_chainProcessor_processReadyBlocks(t *testing.T) { - t.Parallel() - mockError := errors.New("test mock error") - tests := map[string]struct { - chainSyncBuilder func(ctrl *gomock.Controller) ChainSync - blockStateBuilder func(ctrl *gomock.Controller, done chan struct{}) BlockState - blockData *types.BlockData - babeVerifierBuilder func(ctrl *gomock.Controller) BabeVerifier - pendingBlockBuilder func(ctrl *gomock.Controller, done chan struct{}) DisjointBlockSet - storageStateBuilder func(ctrl *gomock.Controller, done chan struct{}) StorageState - }{ - "base_case": { - chainSyncBuilder: func(ctrl *gomock.Controller) ChainSync { - cs := NewMockChainSync(ctrl) - cs.EXPECT().syncState().Return(bootstrap) - return cs - }, - blockStateBuilder: func(ctrl *gomock.Controller, done chan struct{}) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().HasHeader(common.Hash{}).Return(false, nil) - mockBlockState.EXPECT().HasBlockBody(common.Hash{}).Return(false, nil) - mockBlockState.EXPECT().CompareAndSetBlockData(&types.BlockData{}).DoAndReturn(func(*types. - BlockData) error { - close(done) - return nil - }) - return mockBlockState - }, - blockData: &types.BlockData{ - Hash: common.Hash{}, - }, - babeVerifierBuilder: func(ctrl *gomock.Controller) BabeVerifier { - return nil - }, - pendingBlockBuilder: func(ctrl *gomock.Controller, done chan struct{}) DisjointBlockSet { - return nil - }, - storageStateBuilder: func(ctrl *gomock.Controller, done chan struct{}) StorageState { - return nil - }, - }, - "add_block": { - chainSyncBuilder: func(ctrl *gomock.Controller) ChainSync { - cs := NewMockChainSync(ctrl) - cs.EXPECT().syncState().Return(bootstrap) - return cs - }, - blockStateBuilder: func(ctrl *gomock.Controller, done chan struct{}) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().HasHeader(common.Hash{}).Return(false, nil) - mockBlockState.EXPECT().HasBlockBody(common.Hash{}).Return(false, nil) - mockBlockState.EXPECT().GetHeader(common.Hash{}).Return(nil, mockError) - return mockBlockState - }, - blockData: &types.BlockData{ - Hash: common.Hash{}, - Header: &types.Header{}, - Body: &types.Body{}, - }, - babeVerifierBuilder: func(ctrl *gomock.Controller) BabeVerifier { - mockBabeVerifier := NewMockBabeVerifier(ctrl) - mockBabeVerifier.EXPECT().VerifyBlock(&types.Header{}).Return(nil) - return mockBabeVerifier - }, - pendingBlockBuilder: func(ctrl *gomock.Controller, done chan struct{}) DisjointBlockSet { - mockDisjointBlockSet := NewMockDisjointBlockSet(ctrl) - mockDisjointBlockSet.EXPECT().addBlock(&types.Block{ - Header: types.Header{}, - Body: types.Body{}, - }).DoAndReturn(func(block *types.Block) error { - close(done) - return nil - }) - return mockDisjointBlockSet - }, - storageStateBuilder: func(ctrl *gomock.Controller, done chan struct{}) StorageState { - return nil - }, - }, - "error_in_process_block": { - chainSyncBuilder: func(ctrl *gomock.Controller) ChainSync { - cs := NewMockChainSync(ctrl) - cs.EXPECT().syncState().Return(bootstrap) - return cs - }, - blockStateBuilder: func(ctrl *gomock.Controller, done chan struct{}) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().HasHeader(common.Hash{}).Return(false, nil) - mockBlockState.EXPECT().HasBlockBody(common.Hash{}).Return(false, nil) - mockBlockState.EXPECT().GetHeader(common.Hash{}).Return(&types.Header{}, nil) - return mockBlockState - }, - blockData: &types.BlockData{ - Hash: common.Hash{}, - Header: &types.Header{}, - Body: &types.Body{}, - }, - babeVerifierBuilder: func(ctrl *gomock.Controller) BabeVerifier { - mockBabeVerifier := NewMockBabeVerifier(ctrl) - mockBabeVerifier.EXPECT().VerifyBlock(&types.Header{}).Return(nil) - return mockBabeVerifier - }, - pendingBlockBuilder: func(ctrl *gomock.Controller, done chan struct{}) DisjointBlockSet { - return nil - }, - storageStateBuilder: func(ctrl *gomock.Controller, done chan struct{}) StorageState { - mockStorageState := NewMockStorageState(ctrl) - mockStorageState.EXPECT().Lock() - mockStorageState.EXPECT().Unlock() - mockStorageState.EXPECT().TrieState(&common.Hash{}).DoAndReturn(func(hash *common.Hash) (*storage. - TrieState, error) { - close(done) - return nil, mockError - }) - return mockStorageState - }, - }, - "add_block_error": { - chainSyncBuilder: func(ctrl *gomock.Controller) ChainSync { - cs := NewMockChainSync(ctrl) - cs.EXPECT().syncState().Return(bootstrap) - return cs - }, - blockStateBuilder: func(ctrl *gomock.Controller, done chan struct{}) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().HasHeader(common.Hash{}).Return(false, nil) - mockBlockState.EXPECT().HasBlockBody(common.Hash{}).Return(false, nil) - mockBlockState.EXPECT().GetHeader(common.Hash{}).Return(nil, mockError) - return mockBlockState - }, - blockData: &types.BlockData{ - Hash: common.Hash{}, - Header: &types.Header{}, - Body: &types.Body{}, - }, - babeVerifierBuilder: func(ctrl *gomock.Controller) BabeVerifier { - mockBabeVerifier := NewMockBabeVerifier(ctrl) - mockBabeVerifier.EXPECT().VerifyBlock(&types.Header{}).Return(nil) - return mockBabeVerifier - }, - pendingBlockBuilder: func(ctrl *gomock.Controller, done chan struct{}) DisjointBlockSet { - mockDisjointBlockSet := NewMockDisjointBlockSet(ctrl) - mockDisjointBlockSet.EXPECT().addBlock(&types.Block{ - Header: types.Header{}, - Body: types.Body{}, - }).DoAndReturn(func(block *types.Block) error { - close(done) - return mockError - }) - return mockDisjointBlockSet - }, - storageStateBuilder: func(ctrl *gomock.Controller, done chan struct{}) StorageState { - return nil - }, - }, - } - for name, tt := range tests { - tt := tt - t.Run(name, func(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - ctx, cancel := context.WithCancel(context.Background()) - readyBlock := newBlockQueue(5) - done := make(chan struct{}) - - s := &chainProcessor{ - ctx: ctx, - cancel: cancel, - readyBlocks: readyBlock, - chainSync: tt.chainSyncBuilder(ctrl), - blockState: tt.blockStateBuilder(ctrl, done), - babeVerifier: tt.babeVerifierBuilder(ctrl), - pendingBlocks: tt.pendingBlockBuilder(ctrl, done), - storageState: tt.storageStateBuilder(ctrl, done), - } - - go s.processReadyBlocks() - - readyBlock.push(tt.blockData) - <-done - s.cancel() - }) - } -} - -func Test_newChainProcessor(t *testing.T) { - t.Parallel() - - mockReadyBlock := newBlockQueue(5) - mockDisjointBlockSet := NewMockDisjointBlockSet(nil) - mockBlockState := NewMockBlockState(nil) - mockStorageState := NewMockStorageState(nil) - mockTransactionState := NewMockTransactionState(nil) - mockBabeVerifier := NewMockBabeVerifier(nil) - mockFinalityGadget := NewMockFinalityGadget(nil) - mockBlockImportHandler := NewMockBlockImportHandler(nil) - - type args struct { - readyBlocks *blockQueue - pendingBlocks DisjointBlockSet - blockState BlockState - storageState StorageState - transactionState TransactionState - babeVerifier BabeVerifier - finalityGadget FinalityGadget - blockImportHandler BlockImportHandler - } - tests := []struct { - name string - args args - want *chainProcessor - }{ - { - name: "with_args", - args: args{ - readyBlocks: mockReadyBlock, - pendingBlocks: mockDisjointBlockSet, - blockState: mockBlockState, - storageState: mockStorageState, - transactionState: mockTransactionState, - babeVerifier: mockBabeVerifier, - finalityGadget: mockFinalityGadget, - blockImportHandler: mockBlockImportHandler, - }, - want: &chainProcessor{ - readyBlocks: mockReadyBlock, - pendingBlocks: mockDisjointBlockSet, - blockState: mockBlockState, - storageState: mockStorageState, - transactionState: mockTransactionState, - babeVerifier: mockBabeVerifier, - finalityGadget: mockFinalityGadget, - blockImportHandler: mockBlockImportHandler, - }, - }, - } - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - cpCfg := chainProcessorConfig{ - readyBlocks: tt.args.readyBlocks, - pendingBlocks: tt.args.pendingBlocks, - blockState: tt.args.blockState, - storageState: tt.args.storageState, - transactionState: tt.args.transactionState, - babeVerifier: tt.args.babeVerifier, - finalityGadget: tt.args.finalityGadget, - blockImportHandler: tt.args.blockImportHandler, - } - - got := newChainProcessor(cpCfg) - assert.NotNil(t, got.ctx) - got.ctx = nil - assert.NotNil(t, got.cancel) - got.cancel = nil - assert.Equal(t, tt.want, got) - }) - } -} diff --git a/dot/sync/chain_sync.go b/dot/sync/chain_sync.go index 48e5c1232e..3d64993130 100644 --- a/dot/sync/chain_sync.go +++ b/dot/sync/chain_sync.go @@ -4,13 +4,13 @@ package sync import ( - "context" - "crypto/rand" + "bytes" "errors" "fmt" "math/big" "strings" "sync" + "sync/atomic" "time" "github.com/ChainSafe/chaindb" @@ -21,18 +21,14 @@ import ( "github.com/ChainSafe/gossamer/dot/network" "github.com/ChainSafe/gossamer/dot/peerset" + "github.com/ChainSafe/gossamer/dot/telemetry" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/blocktree" "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/common/variadic" ) -const ( - // maxWorkers is the maximum number of parallel sync workers - maxWorkers = 12 -) - -var _ ChainSync = &chainSync{} +var _ ChainSync = (*chainSync)(nil) type chainSyncState byte @@ -53,692 +49,705 @@ func (s chainSyncState) String() string { } var ( - pendingBlocksLimit = maxResponseSize * 32 + pendingBlocksLimit = network.MaxBlocksInResponse * 32 isSyncedGauge = promauto.NewGauge(prometheus.GaugeOpts{ Namespace: "gossamer_network_syncer", Name: "is_synced", Help: "bool representing whether the node is synced to the head of the chain", }) + + blockSizeGauge = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: "gossamer_sync", + Name: "block_size", + Help: "represent the size of blocks synced", + }) ) -// peerState tracks our peers's best reported blocks -type peerState struct { +// peerView tracks our peers's best reported blocks +type peerView struct { who peer.ID hash common.Hash number uint } -// workHandler handles new potential work (ie. reported peer state, block announces), results from dispatched workers, -// and stored pending work (ie. pending blocks set) -// workHandler should be implemented by `bootstrapSync` and `tipSync` -type workHandler interface { - // handleNewPeerState returns a new worker based on a peerState. - // The worker may be nil in which case we do nothing. - handleNewPeerState(*peerState) (*worker, error) - - // handleWorkerResult handles the result of a worker, which may be - // nil or error. It optionally returns a new worker to be dispatched. - handleWorkerResult(w *worker) (workerToRetry *worker, err error) - - // hasCurrentWorker is called before a worker is to be dispatched to - // check whether it is a duplicate. this function returns whether there is - // a worker that covers the scope of the proposed worker; if true, - // ignore the proposed worker - hasCurrentWorker(*worker, map[uint64]*worker) bool - - // handleTick handles a timer tick - handleTick() ([]*worker, error) -} - // ChainSync contains the methods used by the high-level service into the `chainSync` module type ChainSync interface { start() - stop() - - // called upon receiving a BlockAnnounce - setBlockAnnounce(from peer.ID, header *types.Header) error + stop() error // called upon receiving a BlockAnnounceHandshake - setPeerHead(p peer.ID, hash common.Hash, number uint) error + onBlockAnnounceHandshake(p peer.ID, hash common.Hash, number uint) error - // syncState returns the current syncing state - syncState() chainSyncState + // getSyncMode returns the current syncing state + getSyncMode() chainSyncState // getHighestBlock returns the highest block or an error getHighestBlock() (highestBlock uint, err error) + + onBlockAnnounce(announcedBlock) error +} + +type announcedBlock struct { + who peer.ID + header *types.Header } type chainSync struct { - ctx context.Context - cancel context.CancelFunc + wg sync.WaitGroup + stopCh chan struct{} blockState BlockState network Network - // queue of work created by setting peer heads - workQueue chan *peerState - - // workers are put here when they are completed so we can handle their result - resultQueue chan *worker + workerPool *syncWorkerPool // tracks the latest state we know of from our peers, // ie. their best block hash and number - sync.RWMutex - peerState map[peer.ID]*peerState - ignorePeers map[peer.ID]struct{} - - // current workers that are attempting to obtain blocks - workerState *workerState - - // blocks which are ready to be processed are put into this queue - // the `chainProcessor` will read from this channel and process the blocks - // note: blocks must not be put into this channel unless their parent is known - // - // there is a case where we request and process "duplicate" blocks, which is where there - // are some blocks in this queue, and at the same time, the bootstrap worker errors and dispatches - // a new worker with start=(current best head), which results in the blocks in the queue - // getting re-requested (as they have not been processed yet) - // to fix this, we track the blocks that are in the queue - readyBlocks *blockQueue + peerViewLock sync.RWMutex + peerView map[peer.ID]peerView // disjoint set of blocks which are known but not ready to be processed // ie. we only know the hash, number, or the parent block is unknown, or the body is unknown // note: the block may have empty fields, as some data about it may be unknown - pendingBlocks DisjointBlockSet - pendingBlockDoneCh chan<- struct{} - - // bootstrap or tip (near-head) - state chainSyncState + pendingBlocks DisjointBlockSet - // handler is set to either `bootstrapSyncer` or `tipSyncer`, depending on the current - // chain sync state - handler workHandler - - benchmarker *syncBenchmarker + syncMode atomic.Value finalisedCh <-chan *types.FinalisationInfo - minPeers int - maxWorkerRetries uint16 - slotDuration time.Duration - - logSyncTicker *time.Ticker - logSyncTickerC <-chan time.Time // channel as field for unit testing - logSyncStarted bool - logSyncDone chan struct{} - badBlocks []string + minPeers int + slotDuration time.Duration - blockReqRes network.RequestMaker + storageState StorageState + transactionState TransactionState + babeVerifier BabeVerifier + finalityGadget FinalityGadget + blockImportHandler BlockImportHandler + telemetry Telemetry + badBlocks []string + requestMaker network.RequestMaker } type chainSyncConfig struct { bs BlockState net Network - readyBlocks *blockQueue + requestMaker network.RequestMaker pendingBlocks DisjointBlockSet minPeers, maxPeers int slotDuration time.Duration + storageState StorageState + transactionState TransactionState + babeVerifier BabeVerifier + finalityGadget FinalityGadget + blockImportHandler BlockImportHandler + telemetry Telemetry badBlocks []string } -func newChainSync(cfg chainSyncConfig, blockReqRes network.RequestMaker) *chainSync { - ctx, cancel := context.WithCancel(context.Background()) - const syncSamplesToKeep = 30 - const logSyncPeriod = 5 * time.Second - logSyncTicker := time.NewTicker(logSyncPeriod) - +func newChainSync(cfg chainSyncConfig) *chainSync { + atomicState := atomic.Value{} + atomicState.Store(tip) return &chainSync{ - ctx: ctx, - cancel: cancel, - blockState: cfg.bs, - network: cfg.net, - workQueue: make(chan *peerState, 1024), - resultQueue: make(chan *worker, 1024), - peerState: make(map[peer.ID]*peerState), - ignorePeers: make(map[peer.ID]struct{}), - workerState: newWorkerState(), - readyBlocks: cfg.readyBlocks, - pendingBlocks: cfg.pendingBlocks, - state: bootstrap, - handler: newBootstrapSyncer(cfg.bs), - benchmarker: newSyncBenchmarker(syncSamplesToKeep), - finalisedCh: cfg.bs.GetFinalisedNotifierChannel(), - minPeers: cfg.minPeers, - maxWorkerRetries: uint16(cfg.maxPeers), - slotDuration: cfg.slotDuration, - logSyncTicker: logSyncTicker, - logSyncTickerC: logSyncTicker.C, - logSyncDone: make(chan struct{}), - badBlocks: cfg.badBlocks, - blockReqRes: blockReqRes, + stopCh: make(chan struct{}), + storageState: cfg.storageState, + transactionState: cfg.transactionState, + babeVerifier: cfg.babeVerifier, + finalityGadget: cfg.finalityGadget, + blockImportHandler: cfg.blockImportHandler, + telemetry: cfg.telemetry, + blockState: cfg.bs, + network: cfg.net, + peerView: make(map[peer.ID]peerView), + pendingBlocks: cfg.pendingBlocks, + syncMode: atomicState, + finalisedCh: cfg.bs.GetFinalisedNotifierChannel(), + minPeers: cfg.minPeers, + slotDuration: cfg.slotDuration, + workerPool: newSyncWorkerPool(cfg.net, cfg.requestMaker), + badBlocks: cfg.badBlocks, + requestMaker: cfg.requestMaker, } } func (cs *chainSync) start() { - // wait until we have received at least `minPeers` peer heads + // since the default status from sync mode is syncMode(tip) + isSyncedGauge.Set(1) + + // wait until we have a minimal workers in the sync worker pool for { - cs.RLock() - n := len(cs.peerState) - cs.RUnlock() - if n >= cs.minPeers { + totalAvailable := cs.workerPool.totalWorkers() + if totalAvailable >= uint(cs.minPeers) { break } - time.Sleep(time.Millisecond * 100) - } - isSyncedGauge.Set(float64(cs.state)) - - pendingBlockDoneCh := make(chan struct{}) - cs.pendingBlockDoneCh = pendingBlockDoneCh - go cs.pendingBlocks.run(pendingBlockDoneCh) - go cs.sync() - cs.logSyncStarted = true - go cs.logSyncSpeed() -} - -func (cs *chainSync) stop() { - if cs.pendingBlockDoneCh != nil { - close(cs.pendingBlockDoneCh) - } - cs.cancel() - if cs.logSyncStarted { - <-cs.logSyncDone + // TODO: https://github.com/ChainSafe/gossamer/issues/3402 + time.Sleep(time.Second) } -} -func (cs *chainSync) syncState() chainSyncState { - return cs.state + cs.wg.Add(1) + go cs.pendingBlocks.run(cs.finalisedCh, cs.stopCh, &cs.wg) } -func (cs *chainSync) setBlockAnnounce(from peer.ID, header *types.Header) error { - // check if we already know of this block, if not, - // add to pendingBlocks set - has, err := cs.blockState.HasHeader(header.Hash()) +func (cs *chainSync) stop() error { + err := cs.workerPool.stop() if err != nil { - return err + return fmt.Errorf("stopping worker poll: %w", err) } - if has { - return blocktree.ErrBlockExists - } + close(cs.stopCh) + allStopCh := make(chan struct{}) + go func() { + defer close(allStopCh) + cs.wg.Wait() + }() - if err = cs.pendingBlocks.addHeader(header); err != nil { - return err - } + timeoutTimer := time.NewTimer(30 * time.Second) - // we assume that if a peer sends us a block announce for a certain block, - // that is also has the chain up until and including that block. - // this may not be a valid assumption, but perhaps we can assume that - // it is likely they will receive this block and its ancestors before us. - return cs.setPeerHead(from, header.Hash(), header.Number) + select { + case <-allStopCh: + if !timeoutTimer.Stop() { + <-timeoutTimer.C + } + return nil + case <-timeoutTimer.C: + return ErrStopTimeout + } } -// setPeerHead sets a peer's best known block and potentially adds the peer's state to the workQueue -func (cs *chainSync) setPeerHead(p peer.ID, hash common.Hash, number uint) error { - ps := &peerState{ - who: p, - hash: hash, - number: number, +func (cs *chainSync) isBootstrap() (bestBlockHeader *types.Header, syncTarget uint, + isBootstrap bool, err error) { + syncTarget, err = cs.getTarget() + if err != nil { + return nil, syncTarget, false, fmt.Errorf("getting target: %w", err) } - cs.Lock() - cs.peerState[p] = ps - cs.Unlock() - // if the peer reports a lower or equal best block number than us, - // check if they are on a fork or not - head, err := cs.blockState.BestBlockHeader() + bestBlockHeader, err = cs.blockState.BestBlockHeader() if err != nil { - return fmt.Errorf("best block header: %w", err) + return nil, syncTarget, false, fmt.Errorf("getting best block header: %w", err) } - if ps.number <= head.Number { - // check if our block hash for that number is the same, if so, do nothing - // as we already have that block - ourHash, err := cs.blockState.GetHashByNumber(ps.number) - if err != nil { - return fmt.Errorf("get block hash by number: %w", err) - } + bestBlockNumber := bestBlockHeader.Number + isBootstrap = bestBlockNumber+network.MaxBlocksInResponse < syncTarget + return bestBlockHeader, syncTarget, isBootstrap, nil +} - if ourHash == ps.hash { - return nil - } +func (cs *chainSync) bootstrapSync() { + defer cs.wg.Done() - // check if their best block is on an invalid chain, if it is, - // potentially downscore them - // for now, we can remove them from the syncing peers set - fin, err := cs.blockState.GetHighestFinalisedHeader() - if err != nil { - return fmt.Errorf("get highest finalised header: %w", err) + for { + select { + case <-cs.stopCh: + logger.Warn("ending bootstrap sync, chain sync stop channel triggered") + return + default: } - // their block hash doesn't match ours for that number (ie. they are on a different - // chain), and also the highest finalised block is higher than that number. - // thus the peer is on an invalid chain - if fin.Number >= ps.number { - // TODO: downscore this peer, or temporarily don't sync from them? (#1399) - // perhaps we need another field in `peerState` to mark whether the state is valid or not - cs.network.ReportPeer(peerset.ReputationChange{ - Value: peerset.BadBlockAnnouncementValue, - Reason: peerset.BadBlockAnnouncementReason, - }, p) - return fmt.Errorf("%w: for peer %s and block number %d", - errPeerOnInvalidFork, p, ps.number) + bestBlockHeader, syncTarget, isFarFromTarget, err := cs.isBootstrap() + if err != nil && !errors.Is(err, errNoPeerViews) { + logger.Criticalf("ending bootstrap sync, checking target distance: %s", err) + return } - // peer is on a fork, check if we have processed the fork already or not - // ie. is their block written to our db? - has, err := cs.blockState.HasHeader(ps.hash) + finalisedHeader, err := cs.blockState.GetHighestFinalisedHeader() if err != nil { - return fmt.Errorf("has header: %w", err) + logger.Criticalf("getting highest finalized header: %w", err) + return } - // if so, do nothing, as we already have their fork - if has { - return nil + logger.Infof( + "🚣 currently syncing, %d peers connected, "+ + "%d available workers, "+ + "target block number %d, "+ + "finalised block number %d with hash %s", + len(cs.network.Peers()), + cs.workerPool.totalWorkers(), + syncTarget, finalisedHeader.Number, finalisedHeader.Hash()) + + if isFarFromTarget { + cs.workerPool.useConnectedPeers() + err = cs.requestMaxBlocksFrom(bestBlockHeader) + if err != nil { + logger.Errorf("requesting max blocks from best block header: %s", err) + } + } else { + // we are less than 128 blocks behind the target we can use tip sync + cs.syncMode.Store(tip) + isSyncedGauge.Set(1) + logger.Debugf("switched sync mode to %s", tip.String()) + return } } +} + +func (cs *chainSync) getSyncMode() chainSyncState { + return cs.syncMode.Load().(chainSyncState) +} + +// onBlockAnnounceHandshake sets a peer's best known block +func (cs *chainSync) onBlockAnnounceHandshake(who peer.ID, bestHash common.Hash, bestNumber uint) error { + cs.workerPool.fromBlockAnnounce(who) + + cs.peerViewLock.Lock() + cs.peerView[who] = peerView{ + who: who, + hash: bestHash, + number: bestNumber, + } + cs.peerViewLock.Unlock() + + if cs.getSyncMode() == bootstrap { + return nil + } - // the peer has a higher best block than us, or they are on some fork we are not aware of - // add it to the disjoint block set - if err = cs.pendingBlocks.addHashAndNumber(ps.hash, ps.number); err != nil { - return fmt.Errorf("add hash and number: %w", err) + _, _, isFarFromTarget, err := cs.isBootstrap() + if err != nil && !errors.Is(err, errNoPeerViews) { + return fmt.Errorf("checking target distance: %w", err) } - cs.workQueue <- ps - logger.Debugf("set peer %s head with block number %d and hash %s", p, number, hash) + if !isFarFromTarget { + return nil + } + + // we are more than 128 blocks behind the head, switch to bootstrap + cs.syncMode.Store(bootstrap) + isSyncedGauge.Set(0) + logger.Debugf("switched sync mode to %s", bootstrap.String()) + + cs.wg.Add(1) + go cs.bootstrapSync() return nil } -func (cs *chainSync) logSyncSpeed() { - defer close(cs.logSyncDone) - defer cs.logSyncTicker.Stop() +func (cs *chainSync) onBlockAnnounce(announced announcedBlock) error { + // TODO: https://github.com/ChainSafe/gossamer/issues/3432 + cs.workerPool.fromBlockAnnounce(announced.who) - for { - before, err := cs.blockState.BestBlockHeader() - if err != nil { - continue - } + if cs.pendingBlocks.hasBlock(announced.header.Hash()) { + return fmt.Errorf("%w: block %s (#%d)", + errAlreadyInDisjointSet, announced.header.Hash(), announced.header.Number) + } - if cs.state == bootstrap { - cs.benchmarker.begin(time.Now(), before.Number) - } + err := cs.pendingBlocks.addHeader(announced.header) + if err != nil { + return fmt.Errorf("while adding pending block header: %w", err) + } - select { - case <-cs.logSyncTickerC: // channel of cs.logSyncTicker - case <-cs.ctx.Done(): - return - } + if cs.getSyncMode() == bootstrap { + return nil + } - finalised, err := cs.blockState.GetHighestFinalisedHeader() - if err != nil { - continue - } + _, _, isFarFromTarget, err := cs.isBootstrap() + if err != nil && !errors.Is(err, errNoPeerViews) { + return fmt.Errorf("checking target distance: %w", err) + } - after, err := cs.blockState.BestBlockHeader() - if err != nil { - continue - } + if !isFarFromTarget { + return cs.requestAnnouncedBlock(announced) + } + + return nil +} - switch cs.state { - case bootstrap: - cs.benchmarker.end(time.Now(), after.Number) - target := cs.getTarget() - - logger.Infof( - "🔗 imported blocks from %d to %d (hashes [%s ... %s])", - before.Number, after.Number, before.Hash(), after.Hash()) - - logger.Infof( - "🚣 currently syncing, %d peers connected, "+ - "target block number %d, %.2f average blocks/second, "+ - "%.2f overall average, finalised block number %d with hash %s", - len(cs.network.Peers()), - target, cs.benchmarker.mostRecentAverage(), - cs.benchmarker.average(), finalised.Number, finalised.Hash()) - case tip: - logger.Infof( - "💤 node waiting, %d peers connected, "+ - "head block number %d with hash %s, "+ - "finalised block number %d with hash %s", - len(cs.network.Peers()), - after.Number, after.Hash(), - finalised.Number, finalised.Hash()) +func (cs *chainSync) requestAnnouncedBlock(announce announcedBlock) error { + peerWhoAnnounced := announce.who + announcedHash := announce.header.Hash() + announcedNumber := announce.header.Number + + has, err := cs.blockState.HasHeader(announcedHash) + if err != nil { + return fmt.Errorf("checking if header exists: %s", err) + } + + if has { + return nil + } + + bestBlockHeader, err := cs.blockState.BestBlockHeader() + if err != nil { + return fmt.Errorf("getting best block header: %w", err) + } + + highestFinalizedHeader, err := cs.blockState.GetHighestFinalisedHeader() + if err != nil { + return fmt.Errorf("getting highest finalized header") + } + + // if the announced block contains a lower number than our best + // block header, let's check if it is greater than our latests + // finalized header, if so this block belongs to a fork chain + if announcedNumber < bestBlockHeader.Number { + // ignore the block if it has the same or lower number + // TODO: is it following the protocol to send a blockAnnounce with number < highestFinalized number? + if announcedNumber <= highestFinalizedHeader.Number { + return nil } + + return cs.requestForkBlocks(bestBlockHeader, highestFinalizedHeader, announce.header, announce.who) } -} -func (cs *chainSync) ignorePeer(who peer.ID) { - if err := who.Validate(); err != nil { - return + err = cs.requestChainBlocks(announce.header, bestBlockHeader, peerWhoAnnounced) + if err != nil { + return fmt.Errorf("requesting chain blocks: %w", err) } - cs.Lock() - cs.ignorePeers[who] = struct{}{} - cs.Unlock() + err = cs.requestPendingBlocks(highestFinalizedHeader) + if err != nil { + return fmt.Errorf("while requesting pending blocks") + } + + return nil } -func (cs *chainSync) sync() { - // set to slot time - ticker := time.NewTicker(cs.slotDuration) +func (cs *chainSync) requestChainBlocks(announcedHeader, bestBlockHeader *types.Header, + peerWhoAnnounced peer.ID) error { + gapLength := uint32(announcedHeader.Number - bestBlockHeader.Number) + startAtBlock := announcedHeader.Number + totalBlocks := uint32(1) - for { - select { - case ps := <-cs.workQueue: - cs.maybeSwitchMode() + var request *network.BlockRequestMessage + startingBlock := *variadic.MustNewUint32OrHash(announcedHeader.Hash()) - if err := cs.handleWork(ps); err != nil { - logger.Errorf("failed to handle chain sync work: %s", err) - } - case res := <-cs.resultQueue: - if err := cs.handleResult(res); err != nil { - logger.Errorf("failed to handle chain sync result: %s", err) - } - case <-ticker.C: - cs.maybeSwitchMode() + if gapLength > 1 { + request = network.NewBlockRequest(startingBlock, gapLength, + network.BootstrapRequestData, network.Descending) - workers, err := cs.handler.handleTick() - if err != nil { - logger.Errorf("failed to handle tick: %s", err) - continue - } + startAtBlock = announcedHeader.Number - uint(*request.Max) + 1 + totalBlocks = *request.Max - for _, worker := range workers { - cs.tryDispatchWorker(worker) - } - case fin := <-cs.finalisedCh: - // on finalised block, call pendingBlocks.removeLowerBlocks() to remove blocks on - // invalid forks from the pending blocks set - cs.pendingBlocks.removeLowerBlocks(fin.Header.Number) - case <-cs.ctx.Done(): - return - } + logger.Debugf("received a block announce from %s, requesting %d blocks, descending request from %s (#%d)", + peerWhoAnnounced, gapLength, announcedHeader.Hash(), announcedHeader.Number) + } else { + request = network.NewBlockRequest(startingBlock, 1, network.BootstrapRequestData, network.Descending) + logger.Debugf("received a block announce from %s, requesting a single block %s (#%d)", + peerWhoAnnounced, announcedHeader.Hash(), announcedHeader.Number) } -} -func (cs *chainSync) maybeSwitchMode() { - head, err := cs.blockState.BestBlockHeader() + resultsQueue := make(chan *syncTaskResult) + cs.workerPool.submitRequest(request, &peerWhoAnnounced, resultsQueue) + err := cs.handleWorkersResults(resultsQueue, startAtBlock, totalBlocks) if err != nil { - logger.Errorf("failed to get best block header: %s", err) - return + return fmt.Errorf("while handling workers results: %w", err) } - target := cs.getTarget() - switch { - case head.Number+maxResponseSize < target: - // we are at least 128 blocks behind the head, switch to bootstrap - cs.setMode(bootstrap) - case head.Number >= target: - // bootstrap complete, switch state to tip if not already - // and begin near-head fork-sync - cs.setMode(tip) - default: - // head is between (target-128, target), and we don't want to switch modes. - } + return nil } -func (cs *chainSync) handleResult(resultWorker *worker) error { - // delete worker from workers map - cs.workerState.delete(resultWorker.id) +func (cs *chainSync) requestForkBlocks(bestBlockHeader, highestFinalizedHeader, announcedHeader *types.Header, + peerWhoAnnounced peer.ID) error { + logger.Debugf("block announce lower than best block %s (#%d) and greater highest finalized %s (#%d)", + bestBlockHeader.Hash(), bestBlockHeader.Number, highestFinalizedHeader.Hash(), highestFinalizedHeader.Number) - // handle results from worker - // if there is an error, potentially retry the worker - if resultWorker.err == nil || resultWorker.ctx.Err() != nil { - return nil //nolint:nilerr + parentExists, err := cs.blockState.HasHeader(announcedHeader.ParentHash) + if err != nil && !errors.Is(err, chaindb.ErrKeyNotFound) { + return fmt.Errorf("while checking header exists: %w", err) } - logger.Debugf("worker id %d failed: %s", resultWorker.id, resultWorker.err.err) + gapLength := uint32(1) + startAtBlock := announcedHeader.Number + announcedHash := announcedHeader.Hash() + var request *network.BlockRequestMessage + startingBlock := *variadic.MustNewUint32OrHash(announcedHash) - // handle errors. in the case that a peer did not respond to us in time, - // temporarily add them to the ignore list. - switch { - case errors.Is(resultWorker.err.err, context.Canceled): - return nil - case errors.Is(resultWorker.err.err, errNoPeers): - logger.Debugf("worker id %d not able to sync with any peer", resultWorker.id) - return nil - case errors.Is(resultWorker.err.err, context.DeadlineExceeded): - cs.network.ReportPeer(peerset.ReputationChange{ - Value: peerset.TimeOutValue, - Reason: peerset.TimeOutReason, - }, resultWorker.err.who) - cs.ignorePeer(resultWorker.err.who) - case strings.Contains(resultWorker.err.err.Error(), "dial backoff"): - cs.ignorePeer(resultWorker.err.who) - return nil - case resultWorker.err.err.Error() == "protocol not supported": - cs.network.ReportPeer(peerset.ReputationChange{ - Value: peerset.BadProtocolValue, - Reason: peerset.BadProtocolReason, - }, resultWorker.err.who) - cs.ignorePeer(resultWorker.err.who) - return nil + if parentExists { + request = network.NewBlockRequest(startingBlock, 1, network.BootstrapRequestData, network.Descending) + } else { + gapLength = uint32(announcedHeader.Number - highestFinalizedHeader.Number) + startAtBlock = highestFinalizedHeader.Number + 1 + request = network.NewBlockRequest(startingBlock, gapLength, network.BootstrapRequestData, network.Descending) } - worker, err := cs.handler.handleWorkerResult(resultWorker) + logger.Debugf("requesting %d fork blocks, starting at %s (#%d)", + peerWhoAnnounced, gapLength, announcedHash, announcedHeader.Number) + + resultsQueue := make(chan *syncTaskResult) + cs.workerPool.submitRequest(request, &peerWhoAnnounced, resultsQueue) + + err = cs.handleWorkersResults(resultsQueue, startAtBlock, gapLength) if err != nil { - logger.Errorf("failed to handle worker result: %s", err) - return err - } else if worker == nil { - return nil + return fmt.Errorf("while handling workers results: %w", err) } - worker.retryCount = resultWorker.retryCount + 1 - if worker.retryCount > cs.maxWorkerRetries { - logger.Debugf( - "discarding worker id %d: maximum retry count %d reached", - worker.id, cs.maxWorkerRetries) - - // if this worker was triggered due to a block in the pending blocks set, - // we want to remove it from the set, as we asked all our peers for it - // and none replied with the info we need. - if worker.pendingBlock != nil { - cs.pendingBlocks.removeBlock(worker.pendingBlock.hash) - } + return nil +} + +func (cs *chainSync) requestPendingBlocks(highestFinalizedHeader *types.Header) error { + pendingBlocksTotal := cs.pendingBlocks.size() + logger.Infof("total of pending blocks: %d", pendingBlocksTotal) + if pendingBlocksTotal < 1 { return nil } - // if we've already tried a peer and there was an error, - // then we shouldn't try them again. - if resultWorker.peersTried != nil { - worker.peersTried = resultWorker.peersTried - } else { - worker.peersTried = make(map[peer.ID]struct{}) + pendingBlocks := cs.pendingBlocks.getBlocks() + for _, pendingBlock := range pendingBlocks { + if pendingBlock.number <= highestFinalizedHeader.Number { + cs.pendingBlocks.removeBlock(pendingBlock.hash) + continue + } + + parentExists, err := cs.blockState.HasHeader(pendingBlock.header.ParentHash) + if err != nil { + return fmt.Errorf("getting pending block parent header: %w", err) + } + + if parentExists { + err := cs.handleReadyBlock(pendingBlock.toBlockData()) + if err != nil { + return fmt.Errorf("handling ready block: %w", err) + } + continue + } + + gapLength := pendingBlock.number - highestFinalizedHeader.Number + if gapLength > 128 { + logger.Warnf("gap of %d blocks, max expected: 128 block", gapLength) + gapLength = 128 + } + + descendingGapRequest := network.NewBlockRequest(*variadic.MustNewUint32OrHash(pendingBlock.hash), + uint32(gapLength), network.BootstrapRequestData, network.Descending) + startAtBlock := pendingBlock.number - uint(*descendingGapRequest.Max) + 1 + + // the `requests` in the tip sync are not related necessarily + // this is why we need to treat them separately + resultsQueue := make(chan *syncTaskResult) + cs.workerPool.submitRequest(descendingGapRequest, nil, resultsQueue) + + // TODO: we should handle the requests concurrently + // a way of achieve that is by constructing a new `handleWorkersResults` for + // handling only tip sync requests + err = cs.handleWorkersResults(resultsQueue, startAtBlock, *descendingGapRequest.Max) + if err != nil { + return fmt.Errorf("while handling workers results: %w", err) + } } - worker.peersTried[resultWorker.err.who] = struct{}{} - cs.tryDispatchWorker(worker) return nil } -// setMode stops all existing workers and clears the worker set and switches the `handler` -// based on the new mode, if the mode is different than previous -func (cs *chainSync) setMode(mode chainSyncState) { - if cs.state == mode { - return +func (cs *chainSync) requestMaxBlocksFrom(bestBlockHeader *types.Header) error { + startRequestAt := bestBlockHeader.Number + 1 + + // targetBlockNumber is the virtual target we will request, however + // we should bound it to the real target which is collected through + // block announces received from other peers + targetBlockNumber := startRequestAt + maxRequestsAllowed*128 + realTarget, err := cs.getTarget() + if err != nil { + return fmt.Errorf("while getting target: %w", err) } - // stop all current workers and clear set - cs.workerState.reset() + if targetBlockNumber > realTarget { + targetBlockNumber = realTarget + } - // update handler to respective mode - switch mode { - case bootstrap: - cs.handler = newBootstrapSyncer(cs.blockState) - case tip: - cs.handler = newTipSyncer(cs.blockState, cs.pendingBlocks, cs.readyBlocks, cs.handleReadyBlock) + requests := network.NewAscendingBlockRequests(startRequestAt, targetBlockNumber, + network.BootstrapRequestData) + + var expectedAmountOfBlocks uint32 + for _, request := range requests { + if request.Max != nil { + expectedAmountOfBlocks += *request.Max + } + } + + resultsQueue := cs.workerPool.submitRequests(requests) + err = cs.handleWorkersResults(resultsQueue, startRequestAt, expectedAmountOfBlocks) + if err != nil { + return fmt.Errorf("while handling workers results: %w", err) } - cs.state = mode - isSyncedGauge.Set(float64(cs.state)) - logger.Debugf("switched sync mode to %d", mode) + return nil } // getTarget takes the average of all peer heads // TODO: should we just return the highest? could be an attack vector potentially, if a peer reports some very large // head block number, it would leave us in bootstrap mode forever // it would be better to have some sort of standard deviation calculation and discard any outliers (#1861) -func (cs *chainSync) getTarget() uint { - cs.RLock() - defer cs.RUnlock() +func (cs *chainSync) getTarget() (uint, error) { + cs.peerViewLock.RLock() + defer cs.peerViewLock.RUnlock() // in practice, this shouldn't happen, as we only start the module once we have some peer states - if len(cs.peerState) == 0 { - // return max uint32 instead of 0, as returning 0 would switch us to tip mode unexpectedly - return uint(1<<32 - 1) + if len(cs.peerView) == 0 { + return 0, errNoPeerViews } // we are going to sort the data and remove the outliers then we will return the avg of all the valid elements - uintArr := make([]uint, 0, len(cs.peerState)) - for _, ps := range cs.peerState { + uintArr := make([]uint, 0, len(cs.peerView)) + for _, ps := range cs.peerView { uintArr = append(uintArr, ps.number) } sum, count := nonOutliersSumCount(uintArr) quotientBigInt := big.NewInt(0).Div(sum, big.NewInt(int64(count))) - return uint(quotientBigInt.Uint64()) + return uint(quotientBigInt.Uint64()), nil } -// handleWork handles potential new work that may be triggered on receiving a peer's state -// in bootstrap mode, this begins the bootstrap process -// in tip mode, this adds the peer's state to the pendingBlocks set and potentially starts -// a fork sync -func (cs *chainSync) handleWork(ps *peerState) error { - logger.Tracef("handling potential work for target block number %d and hash %s", ps.number, ps.hash) - worker, err := cs.handler.handleNewPeerState(ps) - if err != nil { - return err - } else if worker != nil { - cs.tryDispatchWorker(worker) - } +// handleWorkersResults, every time we submit requests to workers they results should be computed here +// and every cicle we should endup with a complete chain, whenever we identify +// any error from a worker we should evaluate the error and re-insert the request +// in the queue and wait for it to completes +// TODO: handle only justification requests +func (cs *chainSync) handleWorkersResults( + workersResults chan *syncTaskResult, startAtBlock uint, expectedSyncedBlocks uint32) error { - return nil -} + startTime := time.Now() + defer func() { + totalSyncAndImportSeconds := time.Since(startTime).Seconds() + bps := float64(expectedSyncedBlocks) / totalSyncAndImportSeconds + logger.Debugf("⛓️ synced %d blocks, "+ + "took: %.2f seconds, bps: %.2f blocks/second", + expectedSyncedBlocks, totalSyncAndImportSeconds, bps) + }() -func (cs *chainSync) tryDispatchWorker(w *worker) { - // if we already have the maximum number of workers, don't dispatch another - if len(cs.workerState.workers) >= maxWorkers { - logger.Trace("reached max workers, ignoring potential work") - return - } + syncingChain := make([]*types.BlockData, expectedSyncedBlocks) + // the total numbers of blocks is missing in the syncing chain + waitingBlocks := expectedSyncedBlocks - // check current worker set for workers already working on these blocks - // if there are none, dispatch new worker - if cs.handler.hasCurrentWorker(w, cs.workerState.workers) { - return - } +taskResultLoop: + for waitingBlocks > 0 { + // in a case where we don't handle workers results we should check the pool + idleDuration := time.Minute + idleTimer := time.NewTimer(idleDuration) - cs.workerState.add(w) - go cs.dispatchWorker(w) -} + select { + case <-cs.stopCh: + return nil -// dispatchWorker begins making requests to the network and attempts to receive responses up until the target -// if it fails due to any reason, it sets the worker `err` and returns -// this function always places the worker into the `resultCh` for result handling upon return -func (cs *chainSync) dispatchWorker(w *worker) { - if w.targetNumber == nil || w.startNumber == nil { - return - } + case <-idleTimer.C: + logger.Warnf("idle ticker triggered! checking pool") + cs.workerPool.useConnectedPeers() + continue - logger.Debugf("dispatching sync worker id %d, "+ - "start number %d, target number %d, "+ - "start hash %s, target hash %s, "+ - "request data %d, direction %s", - w.id, - *w.startNumber, *w.targetNumber, - w.startHash, w.targetHash, - w.requestData, w.direction) + case taskResult := <-workersResults: + if !idleTimer.Stop() { + <-idleTimer.C + } - start := time.Now() - defer func() { - end := time.Now() - w.duration = end.Sub(start) - outcome := "success" - if w.err != nil { - outcome = "failure" - } - logger.Debugf( - "sync worker completed in %s with %s for worker id %d", - w.duration, outcome, w.id) - cs.resultQueue <- w - }() + who := taskResult.who + request := taskResult.request + response := taskResult.response - reqs, err := workerToRequests(w) - if err != nil { - // if we are creating valid workers, this should not happen - logger.Criticalf("failed to create requests from worker id %d: %s", w.id, err) - w.err = &workerError{ - err: err, - } - return - } + logger.Debugf("task result: peer(%s), with error: %v, with response: %v", + taskResult.who, taskResult.err != nil, taskResult.response != nil) - for _, req := range reqs { - // TODO: if we find a good peer, do sync with them, right now it re-selects a peer each time (#1399) - if err := cs.doSync(req, w.peersTried); err != nil { - // failed to sync, set worker error and put into result queue - w.err = err - return - } - } -} + if taskResult.err != nil { + logger.Errorf("task result: peer(%s) error: %s", + taskResult.who, taskResult.err) + + if !errors.Is(taskResult.err, network.ErrReceivedEmptyMessage) { + if strings.Contains(taskResult.err.Error(), "protocols not supported") { + cs.network.ReportPeer(peerset.ReputationChange{ + Value: peerset.BadProtocolValue, + Reason: peerset.BadProtocolReason, + }, who) + } + } -func (cs *chainSync) doSync(req *network.BlockRequestMessage, peersTried map[peer.ID]struct{}) *workerError { - // determine which peers have the blocks we want to request - peers := cs.determineSyncPeers(req, peersTried) + // TODO: avoid the same peer to get the same task + cs.workerPool.submitRequest(request, nil, workersResults) + continue + } - if len(peers) == 0 { - return &workerError{ - err: errNoPeers, - } - } + if request.Direction == network.Descending { + // reverse blocks before pre-validating and placing in ready queue + reverseBlockData(response.BlockData) + } - // send out request and potentially receive response, error if timeout - logger.Tracef("sending out block request: %s", req) + err := validateResponseFields(request.RequestedData, response.BlockData) + if err != nil { + logger.Criticalf("validating fields: %s", err) + // TODO: check the reputation change for nil body in response + // and nil justification in response + if errors.Is(err, errNilHeaderInResponse) { + cs.network.ReportPeer(peerset.ReputationChange{ + Value: peerset.IncompleteHeaderValue, + Reason: peerset.IncompleteHeaderReason, + }, who) + } - // TODO: use scoring to determine what peer to try to sync from first (#1399) - idx, _ := rand.Int(rand.Reader, big.NewInt(int64(len(peers)))) - who := peers[idx.Int64()] + cs.workerPool.submitRequest(taskResult.request, nil, workersResults) + continue taskResultLoop + } - resp := new(network.BlockResponseMessage) + isChain := isResponseAChain(response.BlockData) + if !isChain { + logger.Criticalf("response from %s is not a chain", who) + cs.workerPool.submitRequest(taskResult.request, nil, workersResults) + continue taskResultLoop + } - err := cs.blockReqRes.Do(who, req, resp) - if err != nil { - return &workerError{ - err: err, - who: who, - } - } + grows := doResponseGrowsTheChain(response.BlockData, syncingChain, + startAtBlock, expectedSyncedBlocks) + if !grows { + logger.Criticalf("response from %s does not grows the ongoing chain", who) + cs.workerPool.submitRequest(taskResult.request, nil, workersResults) + continue taskResultLoop + } - if req.Direction == network.Descending { - // reverse blocks before pre-validating and placing in ready queue - reverseBlockData(resp.BlockData) - } + for _, blockInResponse := range response.BlockData { + if slices.Contains(cs.badBlocks, blockInResponse.Hash.String()) { + logger.Criticalf("%s sent a known bad block: %s (#%d)", + who, blockInResponse.Hash.String(), blockInResponse.Number()) + + cs.network.ReportPeer(peerset.ReputationChange{ + Value: peerset.BadBlockAnnouncementValue, + Reason: peerset.BadBlockAnnouncementReason, + }, who) + + cs.workerPool.ignorePeerAsWorker(taskResult.who) + cs.workerPool.submitRequest(taskResult.request, nil, workersResults) + continue taskResultLoop + } + + blockExactIndex := blockInResponse.Header.Number - startAtBlock + syncingChain[blockExactIndex] = blockInResponse + } + + // we need to check if we've filled all positions + // otherwise we should wait for more responses + waitingBlocks -= uint32(len(response.BlockData)) + + // we received a response without the desired amount of blocks + // we should include a new request to retrieve the missing blocks + if len(response.BlockData) < int(*request.Max) { + difference := uint32(int(*request.Max) - len(response.BlockData)) + lastItem := response.BlockData[len(response.BlockData)-1] - // perform some pre-validation of response, error if failure - if err := cs.validateResponse(req, resp, who); err != nil { - return &workerError{ - err: err, - who: who, + startRequestNumber := uint32(lastItem.Header.Number + 1) + startAt, err := variadic.NewUint32OrHash(startRequestNumber) + if err != nil { + panic(err) + } + + taskResult.request = &network.BlockRequestMessage{ + RequestedData: network.BootstrapRequestData, + StartingBlock: *startAt, + Direction: network.Ascending, + Max: &difference, + } + cs.workerPool.submitRequest(taskResult.request, nil, workersResults) + continue taskResultLoop + } } } - logger.Trace("success! placing block response data in ready queue") + retreiveBlocksSeconds := time.Since(startTime).Seconds() + logger.Debugf("🔽 retrieved %d blocks, took: %.2f seconds, starting process...", + expectedSyncedBlocks, retreiveBlocksSeconds) // response was validated! place into ready block queue - for _, bd := range resp.BlockData { + for _, bd := range syncingChain { // block is ready to be processed! - cs.handleReadyBlock(bd) + if err := cs.handleReadyBlock(bd); err != nil { + return fmt.Errorf("while handling ready block: %w", err) + } } return nil } -func (cs *chainSync) handleReadyBlock(bd *types.BlockData) { - if cs.readyBlocks.has(bd.Hash) { - logger.Tracef("ignoring block %s in response, already in ready queue", bd.Hash) - return - } - +func (cs *chainSync) handleReadyBlock(bd *types.BlockData) error { // if header was not requested, get it from the pending set // if we're expecting headers, validate should ensure we have a header if bd.Header == nil { @@ -749,326 +758,345 @@ func (cs *chainSync) handleReadyBlock(bd *types.BlockData) { has, err := cs.blockState.HasHeader(bd.Hash) if err != nil && !errors.Is(err, chaindb.ErrKeyNotFound) { logger.Debugf("failed to check if header is known for hash %s: %s", bd.Hash, err) - return + return err } if has { logger.Tracef("ignoring block we've already processed, hash=%s", bd.Hash) - return + return err } // this is bad and shouldn't happen logger.Errorf("block with unknown header is ready: hash=%s", bd.Hash) - return + return err + } + + if block.header == nil { + logger.Errorf("new ready block number (unknown) with hash %s", bd.Hash) + return nil } bd.Header = block.header } - if bd.Header == nil { - logger.Errorf("new ready block number (unknown) with hash %s", bd.Hash) - return + err := cs.processBlockData(*bd) + if err != nil { + // depending on the error, we might want to save this block for later + logger.Errorf("block data processing for block with hash %s failed: %s", bd.Hash, err) + return err } - logger.Tracef("new ready block number %d with hash %s", bd.Header.Number, bd.Hash) + cs.pendingBlocks.removeBlock(bd.Hash) + return nil +} - // see if there are any descendents in the pending queue that are now ready to be processed, - // as we have just become aware of their parent block - ready := []*types.BlockData{bd} - ready = cs.pendingBlocks.getReadyDescendants(bd.Hash, ready) +// processBlockData processes the BlockData from a BlockResponse and +// returns the index of the last BlockData it handled on success, +// or the index of the block data that errored on failure. +func (cs *chainSync) processBlockData(blockData types.BlockData) error { + headerInState, err := cs.blockState.HasHeader(blockData.Hash) + if err != nil { + return fmt.Errorf("checking if block state has header: %w", err) + } - for _, rb := range ready { - cs.pendingBlocks.removeBlock(rb.Hash) - cs.readyBlocks.push(rb) + bodyInState, err := cs.blockState.HasBlockBody(blockData.Hash) + if err != nil { + return fmt.Errorf("checking if block state has body: %w", err) } -} -// determineSyncPeers returns a list of peers that likely have the blocks in the given block request. -func (cs *chainSync) determineSyncPeers(req *network.BlockRequestMessage, peersTried map[peer.ID]struct{}) []peer.ID { - var start uint32 - if req.StartingBlock.IsUint32() { - start = req.StartingBlock.Uint32() + // while in bootstrap mode we don't need to broadcast block announcements + announceImportedBlock := cs.getSyncMode() == tip + if headerInState && bodyInState { + err = cs.processBlockDataWithStateHeaderAndBody(blockData, announceImportedBlock) + if err != nil { + return fmt.Errorf("processing block data with header and "+ + "body in block state: %w", err) + } + return nil } - cs.RLock() - defer cs.RUnlock() + if blockData.Header != nil { + if blockData.Body != nil { + err = cs.processBlockDataWithHeaderAndBody(blockData, announceImportedBlock) + if err != nil { + return fmt.Errorf("processing block data with header and body: %w", err) + } + } - // if we're currently ignoring all our peers, clear out the list. - if len(cs.peerState) == len(cs.ignorePeers) { - cs.RUnlock() - cs.Lock() - for p := range cs.ignorePeers { - delete(cs.ignorePeers, p) + if blockData.Justification != nil && len(*blockData.Justification) > 0 { + logger.Infof("handling justification for block %s (#%d)", blockData.Hash.Short(), blockData.Number()) + err = cs.handleJustification(blockData.Header, *blockData.Justification) + if err != nil { + return fmt.Errorf("handling justification: %w", err) + } } - cs.Unlock() - cs.RLock() } - peers := make([]peer.ID, 0, len(cs.peerState)) + err = cs.blockState.CompareAndSetBlockData(&blockData) + if err != nil { + return fmt.Errorf("comparing and setting block data: %w", err) + } - for p, state := range cs.peerState { - if _, has := cs.ignorePeers[p]; has { - continue - } + return nil +} - if _, has := peersTried[p]; has { - continue - } +func (cs *chainSync) processBlockDataWithStateHeaderAndBody(blockData types.BlockData, + announceImportedBlock bool) (err error) { + // TODO: fix this; sometimes when the node shuts down the "best block" isn't stored properly, + // so when the node restarts it has blocks higher than what it thinks is the best, causing it not to sync + // if we update the node to only store finalised blocks in the database, this should be fixed and the entire + // code block can be removed (#1784) + block, err := cs.blockState.GetBlockByHash(blockData.Hash) + if err != nil { + return fmt.Errorf("getting block by hash: %w", err) + } - // if peer definitely doesn't have any blocks we want in the request, - // don't request from them - if start > 0 && uint32(state.number) < start { - continue + err = cs.blockState.AddBlockToBlockTree(block) + if errors.Is(err, blocktree.ErrBlockExists) { + logger.Debugf( + "block number %d with hash %s already exists in block tree, skipping it.", + block.Header.Number, blockData.Hash) + return nil + } else if err != nil { + return fmt.Errorf("adding block to blocktree: %w", err) + } + + if blockData.Justification != nil && len(*blockData.Justification) > 0 { + err = cs.handleJustification(&block.Header, *blockData.Justification) + if err != nil { + return fmt.Errorf("handling justification: %w", err) } + } - peers = append(peers, p) + // TODO: this is probably unnecessary, since the state is already in the database + // however, this case shouldn't be hit often, since it's only hit if the node state + // is rewinded or if the node shuts down unexpectedly (#1784) + state, err := cs.storageState.TrieState(&block.Header.StateRoot) + if err != nil { + return fmt.Errorf("loading trie state: %w", err) } - return peers -} + err = cs.blockImportHandler.HandleBlockImport(block, state, announceImportedBlock) + if err != nil { + return fmt.Errorf("handling block import: %w", err) + } -// validateResponse performs pre-validation of a block response before placing it into either the -// pendingBlocks or readyBlocks set. -// It checks the following: -// - the response is not empty -// - the response contains all the expected fields -// - the block is not contained in the bad block list -// - each block has the correct parent, ie. the response constitutes a valid chain -func (cs *chainSync) validateResponse(req *network.BlockRequestMessage, - resp *network.BlockResponseMessage, p peer.ID) error { - if resp == nil || len(resp.BlockData) == 0 { - return errEmptyBlockData - } - - logger.Tracef("validating block response starting at block hash %s", resp.BlockData[0].Hash) - - var ( - prev, curr *types.Header - err error - ) - headerRequested := (req.RequestedData & network.RequestedDataHeader) == 1 - - for i, bd := range resp.BlockData { - if err = cs.validateBlockData(req, bd, p); err != nil { - return err - } + return nil +} - if headerRequested { - curr = bd.Header - } else { - // if this is a justification-only request, make sure we have the block for the justification - if err = cs.validateJustification(bd); err != nil { - cs.network.ReportPeer(peerset.ReputationChange{ - Value: peerset.BadJustificationValue, - Reason: peerset.BadJustificationReason, - }, p) - return err - } - continue - } +func (cs *chainSync) processBlockDataWithHeaderAndBody(blockData types.BlockData, + announceImportedBlock bool) (err error) { + err = cs.babeVerifier.VerifyBlock(blockData.Header) + if err != nil { + return fmt.Errorf("babe verifying block: %w", err) + } - // check that parent of first block in response is known (either in our db or in the ready queue) - if i == 0 { - prev = curr + cs.handleBody(blockData.Body) - // check that we know the parent of the first block (or it's in the ready queue) - has, _ := cs.blockState.HasHeader(curr.ParentHash) - if has { - continue - } + block := &types.Block{ + Header: *blockData.Header, + Body: *blockData.Body, + } - if cs.readyBlocks.has(curr.ParentHash) { - continue - } + err = cs.handleBlock(block, announceImportedBlock) + if err != nil { + return fmt.Errorf("handling block: %w", err) + } - // parent unknown, add to pending blocks - if err := cs.pendingBlocks.addBlock(&types.Block{ - Header: *curr, - Body: *bd.Body, - }); err != nil { - return err - } + return nil +} - if bd.Justification != nil { - if err := cs.pendingBlocks.addJustification(bd.Hash, *bd.Justification); err != nil { - return err - } - } +// handleHeader handles block bodies included in BlockResponses +func (cs *chainSync) handleBody(body *types.Body) { + acc := 0 + for _, ext := range *body { + acc += len(ext) + cs.transactionState.RemoveExtrinsic(ext) + } - return errUnknownParent - } + blockSizeGauge.Set(float64(acc)) +} - // otherwise, check that this response forms a chain - // ie. curr's parent hash is hash of previous header, and curr's number is previous number + 1 - if prev.Hash() != curr.ParentHash || curr.Number != prev.Number+1 { - // the response is missing some blocks, place blocks from curr onwards into pending blocks set - for _, bd := range resp.BlockData[i:] { - if err := cs.pendingBlocks.addBlock(&types.Block{ - Header: *curr, - Body: *bd.Body, - }); err != nil { - return err - } +func (cs *chainSync) handleJustification(header *types.Header, justification []byte) (err error) { + logger.Debugf("handling justification for block %d...", header.Number) - if bd.Justification != nil { - if err := cs.pendingBlocks.addJustification(bd.Hash, *bd.Justification); err != nil { - return err - } - } - } - return errResponseIsNotChain - } + headerHash := header.Hash() + err = cs.finalityGadget.VerifyBlockJustification(headerHash, justification) + if err != nil { + return fmt.Errorf("verifying block number %d justification: %w", header.Number, err) + } - prev = curr + err = cs.blockState.SetJustification(headerHash, justification) + if err != nil { + return fmt.Errorf("setting justification for block number %d: %w", header.Number, err) } + logger.Infof("🔨 finalised block number %d with hash %s", header.Number, headerHash) return nil } -// validateBlockData checks that the expected fields are in the block data -func (cs *chainSync) validateBlockData(req *network.BlockRequestMessage, bd *types.BlockData, p peer.ID) error { - if bd == nil { - return errNilBlockData +// handleHeader handles blocks (header+body) included in BlockResponses +func (cs *chainSync) handleBlock(block *types.Block, announceImportedBlock bool) error { + parent, err := cs.blockState.GetHeader(block.Header.ParentHash) + if err != nil { + return fmt.Errorf("%w: %s", errFailedToGetParent, err) } - requestedData := req.RequestedData + cs.storageState.Lock() + defer cs.storageState.Unlock() - if slices.Contains(cs.badBlocks, bd.Hash.String()) { - logger.Errorf("Rejecting known bad block Number: %d Hash: %s", bd.Number(), bd.Hash) - return errBadBlock + ts, err := cs.storageState.TrieState(&parent.StateRoot) + if err != nil { + return err } - if (requestedData&network.RequestedDataHeader) == 1 && bd.Header == nil { - cs.network.ReportPeer(peerset.ReputationChange{ - Value: peerset.IncompleteHeaderValue, - Reason: peerset.IncompleteHeaderReason, - }, p) - return errNilHeaderInResponse + root := ts.MustRoot() + if !bytes.Equal(parent.StateRoot[:], root[:]) { + panic("parent state root does not match snapshot state root") } - if (requestedData&network.RequestedDataBody>>1) == 1 && bd.Body == nil { - return fmt.Errorf("%w: hash=%s", errNilBodyInResponse, bd.Hash) + rt, err := cs.blockState.GetRuntime(parent.Hash()) + if err != nil { + return err } - return nil -} + rt.SetContextStorage(ts) -func (cs *chainSync) validateJustification(bd *types.BlockData) error { - if bd == nil { - return errNilBlockData + _, err = rt.ExecuteBlock(block) + if err != nil { + return fmt.Errorf("failed to execute block %d: %w", block.Header.Number, err) } - // this is ok, since the remote peer doesn't need to provide the info we request from them - // especially with justifications, it's common that they don't have them. - if bd.Justification == nil { - return nil + if err = cs.blockImportHandler.HandleBlockImport(block, ts, announceImportedBlock); err != nil { + return err } - has, _ := cs.blockState.HasHeader(bd.Hash) - if !has { - return errUnknownBlockForJustification - } + blockHash := block.Header.Hash() + cs.telemetry.SendMessage(telemetry.NewBlockImport( + &blockHash, + block.Header.Number, + "NetworkInitialSync")) return nil } -func (cs *chainSync) getHighestBlock() (highestBlock uint, err error) { - cs.RLock() - defer cs.RUnlock() +// validateResponseFields checks that the expected fields are in the block data +func validateResponseFields(requestedData byte, blocks []*types.BlockData) error { + for _, bd := range blocks { + if bd == nil { + return errNilBlockData + } - if len(cs.peerState) == 0 { - return 0, errNoPeers - } + if (requestedData&network.RequestedDataHeader) == network.RequestedDataHeader && bd.Header == nil { + return fmt.Errorf("%w: %s", errNilHeaderInResponse, bd.Hash) + } - for _, ps := range cs.peerState { - if ps.number < highestBlock { - continue + if (requestedData&network.RequestedDataBody) == network.RequestedDataBody && bd.Body == nil { + return fmt.Errorf("%w: %s", errNilBodyInResponse, bd.Hash) + } + + // if we requested strictly justification + if (requestedData|network.RequestedDataJustification) == network.RequestedDataJustification && + bd.Justification == nil { + return fmt.Errorf("%w: %s", errNilJustificationInResponse, bd.Hash) } - highestBlock = ps.number } - return highestBlock, nil + return nil } -func workerToRequests(w *worker) ([]*network.BlockRequestMessage, error) { - diff := int(*w.targetNumber) - int(*w.startNumber) - if diff < 0 && w.direction != network.Descending { - return nil, errInvalidDirection +func isResponseAChain(responseBlockData []*types.BlockData) bool { + if len(responseBlockData) < 2 { + return true } - if diff > 0 && w.direction != network.Ascending { - return nil, errInvalidDirection - } + previousBlockData := responseBlockData[0] + for _, currBlockData := range responseBlockData[1:] { + previousHash := previousBlockData.Header.Hash() + isParent := previousHash == currBlockData.Header.ParentHash + if !isParent { + return false + } - // start and end block are the same, just request 1 block - if diff == 0 { - diff = 1 + previousBlockData = currBlockData } - // to deal with descending requests (ie. target may be lower than start) which are used in tip mode, - // take absolute value of difference between start and target - numBlocks := diff - if numBlocks < 0 { - numBlocks = -numBlocks + return true +} + +// doResponseGrowsTheChain will check if the acquired blocks grows the current chain +// matching their parent hashes +func doResponseGrowsTheChain(response, ongoingChain []*types.BlockData, startAtBlock uint, expectedTotal uint32) bool { + // the ongoing chain does not have any element, we can safely insert an item in it + if len(ongoingChain) < 1 { + return true } - numRequests := uint(numBlocks) / maxResponseSize - if numBlocks%maxResponseSize != 0 { - numRequests++ + compareParentHash := func(parent, child *types.BlockData) bool { + return parent.Header.Hash() == child.Header.ParentHash } - startNumber := *w.startNumber - reqs := make([]*network.BlockRequestMessage, numRequests) + firstBlockInResponse := response[0] + firstBlockExactIndex := firstBlockInResponse.Header.Number - startAtBlock + if firstBlockExactIndex != 0 { + leftElement := ongoingChain[firstBlockExactIndex-1] + if leftElement != nil && !compareParentHash(leftElement, firstBlockInResponse) { + return false + } - for i := uint(0); i < numRequests; i++ { - // check if we want to specify a size - max := uint32(maxResponseSize) + } - if w.direction == network.Descending && i == numRequests-1 { - size := numBlocks % maxResponseSize - if size == 0 { - size = maxResponseSize + switch { + // if the response contains only one block then we should check both sides + // for example, if the response contains only one block called X we should + // check if its parent hash matches with the left element as well as we should + // check if the right element contains X hash as its parent hash + // ... W <- X -> Y ... + // we can skip left side comparison if X is in the 0 index and we can skip + // right side comparison if X is in the last index + case len(response) == 1: + if uint32(firstBlockExactIndex+1) < expectedTotal { + rightElement := ongoingChain[firstBlockExactIndex+1] + if rightElement != nil && !compareParentHash(firstBlockInResponse, rightElement) { + return false } - max = uint32(size) } - - var start *variadic.Uint32OrHash - if w.startHash.IsEmpty() { - // worker startHash is unspecified if we are in bootstrap mode - start = variadic.MustNewUint32OrHash(uint32(startNumber)) - } else { - // in tip-syncing mode, we know the hash of the block on the fork we wish to sync - start = variadic.MustNewUint32OrHash(w.startHash) - - // if we're doing descending requests and not at the last (highest starting) request, - // then use number as start block - if w.direction == network.Descending && i != numRequests-1 { - start = variadic.MustNewUint32OrHash(startNumber) + // if the response contains more than 1 block then we need to compare + // only the start and the end of the acquired response, for example + // let's say we receive a response [C, D, E] and we need to check + // if those values fits correctly: + // ... B <- C D E -> F + // we skip the left check if its index is equals to 0 and we skip the right + // check if it ends in the latest position of the ongoing array + case len(response) > 1: + lastBlockInResponse := response[len(response)-1] + lastBlockExactIndex := lastBlockInResponse.Header.Number - startAtBlock + + if uint32(lastBlockExactIndex+1) < expectedTotal { + rightElement := ongoingChain[lastBlockExactIndex+1] + if rightElement != nil && !compareParentHash(lastBlockInResponse, rightElement) { + return false } } + } - reqs[i] = &network.BlockRequestMessage{ - RequestedData: w.requestData, - StartingBlock: *start, - Direction: w.direction, - Max: &max, - } + return true +} - switch w.direction { - case network.Ascending: - startNumber += maxResponseSize - case network.Descending: - startNumber -= maxResponseSize - } +func (cs *chainSync) getHighestBlock() (highestBlock uint, err error) { + cs.peerViewLock.RLock() + defer cs.peerViewLock.RUnlock() + + if len(cs.peerView) == 0 { + return 0, errNoPeers } - // if our direction is descending, we want to send out the request with the lowest - // startNumber first - if w.direction == network.Descending { - for i, j := 0, len(reqs)-1; i < j; i, j = i+1, j-1 { - reqs[i], reqs[j] = reqs[j], reqs[i] + for _, ps := range cs.peerView { + if ps.number < highestBlock { + continue } + highestBlock = ps.number } - return reqs, nil + return highestBlock, nil } diff --git a/dot/sync/chain_sync_integration_test.go b/dot/sync/chain_sync_integration_test.go deleted file mode 100644 index 375059cb34..0000000000 --- a/dot/sync/chain_sync_integration_test.go +++ /dev/null @@ -1,88 +0,0 @@ -//go:build integration - -// Copyright 2021 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package sync - -import ( - "errors" - "testing" - - "github.com/ChainSafe/gossamer/dot/network" - "github.com/ChainSafe/gossamer/dot/peerset" - "github.com/ChainSafe/gossamer/dot/types" - "github.com/ChainSafe/gossamer/lib/common" - "github.com/golang/mock/gomock" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/stretchr/testify/require" -) - -func TestValidateBlockData(t *testing.T) { - ctrl := gomock.NewController(t) - cs := newTestChainSync(ctrl) - mockNetwork := NewMockNetwork(ctrl) - mockNetwork.EXPECT().ReportPeer(peerset.ReputationChange{ - Value: -1048576, - Reason: "Incomplete header", - }, peer.ID("")) - cs.network = mockNetwork - - req := &network.BlockRequestMessage{ - RequestedData: bootstrapRequestData, - } - - err := cs.validateBlockData(req, nil, "") - require.Equal(t, errNilBlockData, err) - - err = cs.validateBlockData(req, &types.BlockData{}, "") - require.Equal(t, errNilHeaderInResponse, err) - - err = cs.validateBlockData(req, &types.BlockData{ - Header: &types.Header{}, - }, "") - require.ErrorIs(t, err, errNilBodyInResponse) - - err = cs.validateBlockData(req, &types.BlockData{ - Header: &types.Header{}, - Body: &types.Body{}, - }, "") - require.NoError(t, err) -} - -func TestChainSync_validateResponse_firstBlock_Integration(t *testing.T) { - ctrl := gomock.NewController(t) - cs := newTestChainSync(ctrl) - bs := NewMockBlockState(ctrl) - bs.EXPECT().HasHeader(gomock.AssignableToTypeOf(common.Hash{})).Return(false, nil) - cs.blockState = bs - - req := &network.BlockRequestMessage{ - RequestedData: bootstrapRequestData, - } - - header := &types.Header{ - Number: 2, - } - - resp := &network.BlockResponseMessage{ - BlockData: []*types.BlockData{ - { - Hash: header.Hash(), - Header: &types.Header{ - Number: 2, - }, - Body: &types.Body{}, - Justification: &[]byte{0}, - }, - }, - } - - err := cs.validateResponse(req, resp, "") - require.True(t, errors.Is(err, errUnknownParent)) - require.True(t, cs.pendingBlocks.(*disjointBlockSet).hasBlock(header.Hash())) - bd := cs.pendingBlocks.getBlock(header.Hash()) - require.NotNil(t, bd.header) - require.NotNil(t, bd.body) - require.NotNil(t, bd.justification) -} diff --git a/dot/sync/chain_sync_test.go b/dot/sync/chain_sync_test.go index a53aa8ba4c..0344ab43f6 100644 --- a/dot/sync/chain_sync_test.go +++ b/dot/sync/chain_sync_test.go @@ -4,26 +4,27 @@ package sync import ( - "context" "errors" + "fmt" + "sync/atomic" "testing" "time" "github.com/ChainSafe/gossamer/dot/network" "github.com/ChainSafe/gossamer/dot/peerset" + "github.com/ChainSafe/gossamer/dot/telemetry" "github.com/ChainSafe/gossamer/dot/types" - "github.com/ChainSafe/gossamer/lib/blocktree" "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/common/variadic" + "github.com/ChainSafe/gossamer/lib/runtime/storage" "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/scale" "github.com/golang/mock/gomock" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -const defaultSlotDuration = 6 * time.Second - func Test_chainSyncState_String(t *testing.T) { t.Parallel() @@ -58,1604 +59,1618 @@ func Test_chainSyncState_String(t *testing.T) { } } -func Test_chainSync_setPeerHead(t *testing.T) { +func Test_chainSync_onBlockAnnounce(t *testing.T) { t.Parallel() + const somePeer = peer.ID("abc") errTest := errors.New("test error") - const somePeer = peer.ID("abc") - someHash := common.Hash{1, 2, 3, 4} + emptyTrieState := storage.NewTrieState(nil) + block1AnnounceHeader := types.NewHeader(common.Hash{}, emptyTrieState.MustRoot(), + common.Hash{}, 1, scale.VaryingDataTypeSlice{}) + block2AnnounceHeader := types.NewHeader(block1AnnounceHeader.Hash(), emptyTrieState.MustRoot(), + common.Hash{}, 2, scale.VaryingDataTypeSlice{}) testCases := map[string]struct { - chainSyncBuilder func(ctrl *gomock.Controller) *chainSync - peerID peer.ID - hash common.Hash - number uint - errWrapped error - errMessage string - expectedPeerIDToPeerState map[peer.ID]*peerState - expectedQueuedPeerStates []*peerState + waitBootstrapSync bool + chainSyncBuilder func(ctrl *gomock.Controller) *chainSync + peerID peer.ID + blockAnnounceHeader *types.Header + errWrapped error + errMessage string + expectedSyncMode chainSyncState }{ - "best_block_header_error": { - chainSyncBuilder: func(ctrl *gomock.Controller) *chainSync { - blockState := NewMockBlockState(ctrl) - blockState.EXPECT().BestBlockHeader().Return(nil, errTest) - return &chainSync{ - peerState: map[peer.ID]*peerState{}, - blockState: blockState, - } - }, - peerID: somePeer, - hash: someHash, - number: 1, - errWrapped: errTest, - errMessage: "best block header: test error", - expectedPeerIDToPeerState: map[peer.ID]*peerState{ - somePeer: { - who: somePeer, - hash: someHash, - number: 1, - }, - }, - }, - "number_smaller_than_best_block_number_get_hash_by_number_error": { + "announced_block_already_exists_in_disjoint_set": { chainSyncBuilder: func(ctrl *gomock.Controller) *chainSync { - blockState := NewMockBlockState(ctrl) - bestBlockHeader := &types.Header{Number: 2} - blockState.EXPECT().BestBlockHeader().Return(bestBlockHeader, nil) - blockState.EXPECT().GetHashByNumber(uint(1)). - Return(common.Hash{}, errTest) - return &chainSync{ - peerState: map[peer.ID]*peerState{}, - blockState: blockState, - } - }, - peerID: somePeer, - hash: someHash, - number: 1, - errWrapped: errTest, - errMessage: "get block hash by number: test error", - expectedPeerIDToPeerState: map[peer.ID]*peerState{ - somePeer: { - who: somePeer, - hash: someHash, - number: 1, - }, - }, - }, - "number_smaller_than_best_block_number_and_same_hash": { - chainSyncBuilder: func(ctrl *gomock.Controller) *chainSync { - blockState := NewMockBlockState(ctrl) - bestBlockHeader := &types.Header{Number: 2} - blockState.EXPECT().BestBlockHeader().Return(bestBlockHeader, nil) - blockState.EXPECT().GetHashByNumber(uint(1)).Return(someHash, nil) + pendingBlocks := NewMockDisjointBlockSet(ctrl) + pendingBlocks.EXPECT().hasBlock(block2AnnounceHeader.Hash()).Return(true) return &chainSync{ - peerState: map[peer.ID]*peerState{}, - blockState: blockState, + stopCh: make(chan struct{}), + pendingBlocks: pendingBlocks, + workerPool: newSyncWorkerPool(NewMockNetwork(nil), NewMockRequestMaker(nil)), } }, - peerID: somePeer, - hash: someHash, - number: 1, - expectedPeerIDToPeerState: map[peer.ID]*peerState{ - somePeer: { - who: somePeer, - hash: someHash, - number: 1, - }, - }, + peerID: somePeer, + blockAnnounceHeader: block2AnnounceHeader, + errWrapped: errAlreadyInDisjointSet, + errMessage: fmt.Sprintf("already in disjoint set: block %s (#%d)", + block2AnnounceHeader.Hash(), block2AnnounceHeader.Number), }, - "number_smaller_than_best_block_number_get_highest_finalised_header_error": { + "failed_to_add_announced_block_in_disjoint_set": { chainSyncBuilder: func(ctrl *gomock.Controller) *chainSync { - blockState := NewMockBlockState(ctrl) - bestBlockHeader := &types.Header{Number: 2} - blockState.EXPECT().BestBlockHeader().Return(bestBlockHeader, nil) - blockState.EXPECT().GetHashByNumber(uint(1)). - Return(common.Hash{2}, nil) // other hash than someHash - blockState.EXPECT().GetHighestFinalisedHeader().Return(nil, errTest) + pendingBlocks := NewMockDisjointBlockSet(ctrl) + pendingBlocks.EXPECT().hasBlock(block2AnnounceHeader.Hash()).Return(false) + pendingBlocks.EXPECT().addHeader(block2AnnounceHeader).Return(errTest) + return &chainSync{ - peerState: map[peer.ID]*peerState{}, - blockState: blockState, + stopCh: make(chan struct{}), + pendingBlocks: pendingBlocks, + workerPool: newSyncWorkerPool(NewMockNetwork(nil), NewMockRequestMaker(nil)), } }, - peerID: somePeer, - hash: someHash, - number: 1, - errWrapped: errTest, - errMessage: "get highest finalised header: test error", - expectedPeerIDToPeerState: map[peer.ID]*peerState{ - somePeer: { - who: somePeer, - hash: someHash, - number: 1, - }, - }, + peerID: somePeer, + blockAnnounceHeader: block2AnnounceHeader, + errWrapped: errTest, + errMessage: "while adding pending block header: test error", }, - "number_smaller_than_best_block_number_and_finalised_number_equal_than_number": { + "announced_block_while_in_bootstrap_mode": { chainSyncBuilder: func(ctrl *gomock.Controller) *chainSync { - blockState := NewMockBlockState(ctrl) - bestBlockHeader := &types.Header{Number: 2} - blockState.EXPECT().BestBlockHeader().Return(bestBlockHeader, nil) - blockState.EXPECT().GetHashByNumber(uint(1)). - Return(common.Hash{2}, nil) // other hash than someHash - finalisedBlockHeader := &types.Header{Number: 1} - blockState.EXPECT().GetHighestFinalisedHeader().Return(finalisedBlockHeader, nil) - network := NewMockNetwork(ctrl) - network.EXPECT().ReportPeer(peerset.ReputationChange{ - Value: peerset.BadBlockAnnouncementValue, - Reason: peerset.BadBlockAnnouncementReason, - }, somePeer) + pendingBlocks := NewMockDisjointBlockSet(ctrl) + pendingBlocks.EXPECT().hasBlock(block2AnnounceHeader.Hash()).Return(false) + pendingBlocks.EXPECT().addHeader(block2AnnounceHeader).Return(nil) + + state := atomic.Value{} + state.Store(bootstrap) + return &chainSync{ - peerState: map[peer.ID]*peerState{}, - blockState: blockState, - network: network, + stopCh: make(chan struct{}), + pendingBlocks: pendingBlocks, + syncMode: state, + workerPool: newSyncWorkerPool(NewMockNetwork(nil), NewMockRequestMaker(nil)), } }, - peerID: somePeer, - hash: someHash, - number: 1, - errWrapped: errPeerOnInvalidFork, - errMessage: "peer is on an invalid fork: for peer ZiCa and block number 1", - expectedPeerIDToPeerState: map[peer.ID]*peerState{ - somePeer: { - who: somePeer, - hash: someHash, - number: 1, - }, - }, + peerID: somePeer, + blockAnnounceHeader: block2AnnounceHeader, }, - "number_smaller_than_best_block_number_and_finalised_number_bigger_than_number": { + "announced_block_while_in_tip_mode": { chainSyncBuilder: func(ctrl *gomock.Controller) *chainSync { - blockState := NewMockBlockState(ctrl) - bestBlockHeader := &types.Header{Number: 2} - blockState.EXPECT().BestBlockHeader().Return(bestBlockHeader, nil) - blockState.EXPECT().GetHashByNumber(uint(1)). - Return(common.Hash{2}, nil) // other hash than someHash - finalisedBlockHeader := &types.Header{Number: 2} - blockState.EXPECT().GetHighestFinalisedHeader().Return(finalisedBlockHeader, nil) - network := NewMockNetwork(ctrl) - network.EXPECT().ReportPeer(peerset.ReputationChange{ - Value: peerset.BadBlockAnnouncementValue, - Reason: peerset.BadBlockAnnouncementReason, - }, somePeer) - return &chainSync{ - peerState: map[peer.ID]*peerState{}, - blockState: blockState, - network: network, + pendingBlocksMock := NewMockDisjointBlockSet(ctrl) + pendingBlocksMock.EXPECT().hasBlock(block2AnnounceHeader.Hash()).Return(false) + pendingBlocksMock.EXPECT().addHeader(block2AnnounceHeader).Return(nil) + pendingBlocksMock.EXPECT().removeBlock(block2AnnounceHeader.Hash()) + pendingBlocksMock.EXPECT().size().Return(int(0)) + + blockStateMock := NewMockBlockState(ctrl) + blockStateMock.EXPECT(). + HasHeader(block2AnnounceHeader.Hash()). + Return(false, nil) + + blockStateMock.EXPECT(). + BestBlockHeader(). + Return(block1AnnounceHeader, nil) + + blockStateMock.EXPECT(). + GetHighestFinalisedHeader(). + Return(block2AnnounceHeader, nil) + + expectedRequest := network.NewBlockRequest(*variadic.MustNewUint32OrHash(block2AnnounceHeader.Hash()), + 1, network.BootstrapRequestData, network.Descending) + + fakeBlockBody := types.Body([]types.Extrinsic{}) + mockedBlockResponse := &network.BlockResponseMessage{ + BlockData: []*types.BlockData{ + { + Hash: block2AnnounceHeader.Hash(), + Header: block2AnnounceHeader, + Body: &fakeBlockBody, + }, + }, } - }, - peerID: somePeer, - hash: someHash, - number: 1, - errWrapped: errPeerOnInvalidFork, - errMessage: "peer is on an invalid fork: for peer ZiCa and block number 1", - expectedPeerIDToPeerState: map[peer.ID]*peerState{ - somePeer: { - who: somePeer, - hash: someHash, - number: 1, - }, - }, - }, - "number smaller than best block number and " + - "finalised number smaller than number and " + - "has_header_error": { - chainSyncBuilder: func(ctrl *gomock.Controller) *chainSync { - blockState := NewMockBlockState(ctrl) - bestBlockHeader := &types.Header{Number: 3} - blockState.EXPECT().BestBlockHeader().Return(bestBlockHeader, nil) - blockState.EXPECT().GetHashByNumber(uint(2)). - Return(common.Hash{2}, nil) // other hash than someHash - finalisedBlockHeader := &types.Header{Number: 1} - blockState.EXPECT().GetHighestFinalisedHeader().Return(finalisedBlockHeader, nil) - blockState.EXPECT().HasHeader(someHash).Return(false, errTest) + + networkMock := NewMockNetwork(ctrl) + requestMaker := NewMockRequestMaker(ctrl) + requestMaker.EXPECT(). + Do(somePeer, expectedRequest, &network.BlockResponseMessage{}). + DoAndReturn(func(_, _, response any) any { + responsePtr := response.(*network.BlockResponseMessage) + *responsePtr = *mockedBlockResponse + return nil + }) + + babeVerifierMock := NewMockBabeVerifier(ctrl) + storageStateMock := NewMockStorageState(ctrl) + importHandlerMock := NewMockBlockImportHandler(ctrl) + telemetryMock := NewMockTelemetry(ctrl) + + const announceBlock = true + ensureSuccessfulBlockImportFlow(t, block1AnnounceHeader, mockedBlockResponse.BlockData, + blockStateMock, babeVerifierMock, storageStateMock, importHandlerMock, telemetryMock, + announceBlock) + + workerPool := newSyncWorkerPool(networkMock, requestMaker) + // include the peer who announced the block in the pool + workerPool.newPeer(somePeer) + + state := atomic.Value{} + state.Store(tip) + return &chainSync{ - peerState: map[peer.ID]*peerState{}, - blockState: blockState, + stopCh: make(chan struct{}), + pendingBlocks: pendingBlocksMock, + syncMode: state, + workerPool: workerPool, + network: networkMock, + blockState: blockStateMock, + babeVerifier: babeVerifierMock, + telemetry: telemetryMock, + storageState: storageStateMock, + blockImportHandler: importHandlerMock, } }, - peerID: somePeer, - hash: someHash, - number: 2, - errWrapped: errTest, - errMessage: "has header: test error", - expectedPeerIDToPeerState: map[peer.ID]*peerState{ - somePeer: { - who: somePeer, - hash: someHash, - number: 2, - }, - }, + peerID: somePeer, + blockAnnounceHeader: block2AnnounceHeader, }, - "number smaller than best block number and " + - "finalised number smaller than number and " + - "has_the_hash": { - chainSyncBuilder: func(ctrl *gomock.Controller) *chainSync { - blockState := NewMockBlockState(ctrl) - bestBlockHeader := &types.Header{Number: 3} - blockState.EXPECT().BestBlockHeader().Return(bestBlockHeader, nil) - blockState.EXPECT().GetHashByNumber(uint(2)). - Return(common.Hash{2}, nil) // other hash than someHash - finalisedBlockHeader := &types.Header{Number: 1} - blockState.EXPECT().GetHighestFinalisedHeader().Return(finalisedBlockHeader, nil) - blockState.EXPECT().HasHeader(someHash).Return(true, nil) - return &chainSync{ - peerState: map[peer.ID]*peerState{}, - blockState: blockState, - } - }, - peerID: somePeer, - hash: someHash, - number: 2, - expectedPeerIDToPeerState: map[peer.ID]*peerState{ - somePeer: { - who: somePeer, - hash: someHash, - number: 2, - }, - }, + } + + for name, tt := range testCases { + tt := tt + t.Run(name, func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + + chainSync := tt.chainSyncBuilder(ctrl) + err := chainSync.onBlockAnnounce(announcedBlock{ + who: tt.peerID, + header: tt.blockAnnounceHeader, + }) + + assert.ErrorIs(t, err, tt.errWrapped) + if tt.errWrapped != nil { + assert.EqualError(t, err, tt.errMessage) + } + + if tt.waitBootstrapSync { + chainSync.wg.Wait() + err = chainSync.workerPool.stop() + require.NoError(t, err) + } + }) + } +} + +func Test_chainSync_onBlockAnnounceHandshake_tipModeNeedToCatchup(t *testing.T) { + ctrl := gomock.NewController(t) + const somePeer = peer.ID("abc") + + emptyTrieState := storage.NewTrieState(nil) + block1AnnounceHeader := types.NewHeader(common.Hash{}, emptyTrieState.MustRoot(), + common.Hash{}, 1, scale.VaryingDataTypeSlice{}) + block2AnnounceHeader := types.NewHeader(block1AnnounceHeader.Hash(), emptyTrieState.MustRoot(), + common.Hash{}, 130, scale.VaryingDataTypeSlice{}) + + blockStateMock := NewMockBlockState(ctrl) + blockStateMock.EXPECT(). + BestBlockHeader(). + Return(block1AnnounceHeader, nil). + Times(2) + + blockStateMock.EXPECT(). + BestBlockHeader(). + Return(block2AnnounceHeader, nil). + Times(1) + + blockStateMock.EXPECT(). + GetHighestFinalisedHeader(). + Return(block1AnnounceHeader, nil). + Times(2) + + expectedRequest := network.NewAscendingBlockRequests( + block1AnnounceHeader.Number+1, + block2AnnounceHeader.Number, network.BootstrapRequestData) + + networkMock := NewMockNetwork(ctrl) + networkMock.EXPECT().Peers().Return([]common.PeerInfo{}). + Times(2) + networkMock.EXPECT().AllConnectedPeersIDs().Return([]peer.ID{}) + + firstMockedResponse := createSuccesfullBlockResponse(t, block1AnnounceHeader.Hash(), 2, 128) + latestItemFromMockedResponse := firstMockedResponse.BlockData[len(firstMockedResponse.BlockData)-1] + + secondMockedResponse := createSuccesfullBlockResponse(t, latestItemFromMockedResponse.Hash, + int(latestItemFromMockedResponse.Header.Number+1), 1) + + requestMaker := NewMockRequestMaker(ctrl) + requestMaker.EXPECT(). + Do(somePeer, expectedRequest[0], &network.BlockResponseMessage{}). + DoAndReturn(func(_, _, response any) any { + responsePtr := response.(*network.BlockResponseMessage) + *responsePtr = *firstMockedResponse + return nil + }) + + requestMaker.EXPECT(). + Do(somePeer, expectedRequest[1], &network.BlockResponseMessage{}). + DoAndReturn(func(_, _, response any) any { + responsePtr := response.(*network.BlockResponseMessage) + *responsePtr = *secondMockedResponse + return nil + }) + + babeVerifierMock := NewMockBabeVerifier(ctrl) + storageStateMock := NewMockStorageState(ctrl) + importHandlerMock := NewMockBlockImportHandler(ctrl) + telemetryMock := NewMockTelemetry(ctrl) + + const announceBlock = false + + ensureSuccessfulBlockImportFlow(t, block1AnnounceHeader, firstMockedResponse.BlockData, + blockStateMock, babeVerifierMock, storageStateMock, importHandlerMock, telemetryMock, + announceBlock) + ensureSuccessfulBlockImportFlow(t, latestItemFromMockedResponse.Header, secondMockedResponse.BlockData, + blockStateMock, babeVerifierMock, storageStateMock, importHandlerMock, telemetryMock, + announceBlock) + + state := atomic.Value{} + state.Store(tip) + + stopCh := make(chan struct{}) + defer close(stopCh) + + chainSync := &chainSync{ + stopCh: stopCh, + peerView: make(map[peer.ID]peerView), + syncMode: state, + pendingBlocks: newDisjointBlockSet(0), + workerPool: newSyncWorkerPool(networkMock, requestMaker), + network: networkMock, + blockState: blockStateMock, + babeVerifier: babeVerifierMock, + telemetry: telemetryMock, + storageState: storageStateMock, + blockImportHandler: importHandlerMock, + } + + err := chainSync.onBlockAnnounceHandshake(somePeer, block2AnnounceHeader.Hash(), block2AnnounceHeader.Number) + require.NoError(t, err) + + chainSync.wg.Wait() + err = chainSync.workerPool.stop() + require.NoError(t, err) + + require.Equal(t, chainSync.getSyncMode(), tip) +} + +func TestChainSync_onBlockAnnounceHandshake_onBootstrapMode(t *testing.T) { + const randomHashString = "0x580d77a9136035a0bc3c3cd86286172f7f81291164c5914266073a30466fba21" + randomHash := common.MustHexToHash(randomHashString) + + testcases := map[string]struct { + newChainSync func(t *testing.T, ctrl *gomock.Controller) *chainSync + peerID peer.ID + bestHash common.Hash + bestNumber uint + shouldBeAWorker bool + workerStatus byte + }{ + "new_peer": { + newChainSync: func(t *testing.T, ctrl *gomock.Controller) *chainSync { + networkMock := NewMockNetwork(ctrl) + workerPool := newSyncWorkerPool(networkMock, NewMockRequestMaker(nil)) + + cs := newChainSyncTest(t, ctrl) + cs.syncMode.Store(bootstrap) + cs.workerPool = workerPool + return cs + }, + peerID: peer.ID("peer-test"), + bestHash: randomHash, + bestNumber: uint(20), + shouldBeAWorker: true, + workerStatus: available, }, - "number_bigger_than_the_head_number_add_hash_and_number_error": { - chainSyncBuilder: func(ctrl *gomock.Controller) *chainSync { - blockState := NewMockBlockState(ctrl) - bestBlockHeader := &types.Header{Number: 1} - blockState.EXPECT().BestBlockHeader().Return(bestBlockHeader, nil) - pendingBlocks := NewMockDisjointBlockSet(ctrl) - pendingBlocks.EXPECT().addHashAndNumber(someHash, uint(2)). - Return(errTest) - return &chainSync{ - peerState: map[peer.ID]*peerState{}, - blockState: blockState, - pendingBlocks: pendingBlocks, + "ignore_peer_should_not_be_included_in_the_workerpoll": { + newChainSync: func(t *testing.T, ctrl *gomock.Controller) *chainSync { + networkMock := NewMockNetwork(ctrl) + workerPool := newSyncWorkerPool(networkMock, NewMockRequestMaker(nil)) + workerPool.ignorePeers = map[peer.ID]struct{}{ + peer.ID("peer-test"): {}, } + + cs := newChainSyncTest(t, ctrl) + cs.syncMode.Store(bootstrap) + cs.workerPool = workerPool + return cs }, - peerID: somePeer, - hash: someHash, - number: 2, - errWrapped: errTest, - errMessage: "add hash and number: test error", - expectedPeerIDToPeerState: map[peer.ID]*peerState{ - somePeer: { - who: somePeer, - hash: someHash, - number: 2, - }, - }, + peerID: peer.ID("peer-test"), + bestHash: randomHash, + bestNumber: uint(20), + shouldBeAWorker: false, }, - "number_bigger_than_the_head_number_success": { - chainSyncBuilder: func(ctrl *gomock.Controller) *chainSync { - blockState := NewMockBlockState(ctrl) - bestBlockHeader := &types.Header{Number: 1} - blockState.EXPECT().BestBlockHeader().Return(bestBlockHeader, nil) - pendingBlocks := NewMockDisjointBlockSet(ctrl) - pendingBlocks.EXPECT().addHashAndNumber(someHash, uint(2)). - Return(nil) - return &chainSync{ - peerState: map[peer.ID]*peerState{}, - blockState: blockState, - pendingBlocks: pendingBlocks, - // buffered of 1 so setPeerHead can write to it - // without a consumer of the channel on the other end. - workQueue: make(chan *peerState, 1), + "peer_already_exists_in_the_pool": { + newChainSync: func(t *testing.T, ctrl *gomock.Controller) *chainSync { + networkMock := NewMockNetwork(ctrl) + workerPool := newSyncWorkerPool(networkMock, NewMockRequestMaker(nil)) + workerPool.workers = map[peer.ID]*syncWorker{ + peer.ID("peer-test"): { + worker: &worker{status: available}, + }, } + + cs := newChainSyncTest(t, ctrl) + cs.syncMode.Store(bootstrap) + cs.workerPool = workerPool + return cs }, - peerID: somePeer, - hash: someHash, - number: 2, - expectedPeerIDToPeerState: map[peer.ID]*peerState{ - somePeer: { - who: somePeer, - hash: someHash, - number: 2, - }, - }, - expectedQueuedPeerStates: []*peerState{ - { - who: somePeer, - hash: someHash, - number: 2, - }, - }, + peerID: peer.ID("peer-test"), + bestHash: randomHash, + bestNumber: uint(20), + shouldBeAWorker: true, + workerStatus: available, }, } - for name, testCase := range testCases { - testCase := testCase - t.Run(name, func(t *testing.T) { - t.Parallel() + for tname, tt := range testcases { + tt := tt + t.Run(tname, func(t *testing.T) { ctrl := gomock.NewController(t) + cs := tt.newChainSync(t, ctrl) + cs.onBlockAnnounceHandshake(tt.peerID, tt.bestHash, tt.bestNumber) + + view, exists := cs.peerView[tt.peerID] + require.True(t, exists) + require.Equal(t, tt.peerID, view.who) + require.Equal(t, tt.bestHash, view.hash) + require.Equal(t, tt.bestNumber, view.number) + + if tt.shouldBeAWorker { + syncWorker, exists := cs.workerPool.workers[tt.peerID] + require.True(t, exists) + require.Equal(t, tt.workerStatus, syncWorker.worker.status) + } else { + _, exists := cs.workerPool.workers[tt.peerID] + require.False(t, exists) + } + }) + } +} - chainSync := testCase.chainSyncBuilder(ctrl) +func newChainSyncTest(t *testing.T, ctrl *gomock.Controller) *chainSync { + t.Helper() - err := chainSync.setPeerHead(testCase.peerID, testCase.hash, testCase.number) + mockBlockState := NewMockBlockState(ctrl) + mockBlockState.EXPECT().GetFinalisedNotifierChannel().Return(make(chan *types.FinalisationInfo)) - assert.ErrorIs(t, err, testCase.errWrapped) - if testCase.errWrapped != nil { - assert.EqualError(t, err, testCase.errMessage) - } - assert.Equal(t, testCase.expectedPeerIDToPeerState, chainSync.peerState) + cfg := chainSyncConfig{ + bs: mockBlockState, + pendingBlocks: newDisjointBlockSet(pendingBlocksLimit), + minPeers: 1, + maxPeers: 5, + slotDuration: 6 * time.Second, + } - require.Equal(t, len(testCase.expectedQueuedPeerStates), len(chainSync.workQueue)) - for _, expectedPeerState := range testCase.expectedQueuedPeerStates { - peerState := <-chainSync.workQueue - assert.Equal(t, expectedPeerState, peerState) - } - }) + return newChainSync(cfg) +} + +func setupChainSyncToBootstrapMode(t *testing.T, blocksAhead uint, + bs BlockState, net Network, reqMaker network.RequestMaker, babeVerifier BabeVerifier, + storageState StorageState, blockImportHandler BlockImportHandler, telemetry Telemetry) *chainSync { + t.Helper() + mockedPeerID := []peer.ID{ + peer.ID("some_peer_1"), + peer.ID("some_peer_2"), + peer.ID("some_peer_3"), + } + + peerViewMap := map[peer.ID]peerView{} + for _, p := range mockedPeerID { + peerViewMap[p] = peerView{ + who: p, + hash: common.Hash{1, 2, 3}, + number: blocksAhead, + } + } + + cfg := chainSyncConfig{ + pendingBlocks: newDisjointBlockSet(pendingBlocksLimit), + minPeers: 1, + maxPeers: 5, + slotDuration: 6 * time.Second, + bs: bs, + net: net, + requestMaker: reqMaker, + babeVerifier: babeVerifier, + storageState: storageState, + blockImportHandler: blockImportHandler, + telemetry: telemetry, } + + chainSync := newChainSync(cfg) + chainSync.peerView = peerViewMap + chainSync.syncMode.Store(bootstrap) + + return chainSync } -func TestChainSync_sync_bootstrap_withWorkerError(t *testing.T) { +func TestChainSync_BootstrapSync_SuccessfulSync_WithOneWorker(t *testing.T) { t.Parallel() - ctrl := gomock.NewController(t) - cs := newTestChainSync(ctrl) - mockBlockState := NewMockBlockState(ctrl) - mockHeader := types.NewHeader(common.NewHash([]byte{0}), trie.EmptyHash, trie.EmptyHash, 0, - types.NewDigest()) - mockBlockState.EXPECT().BestBlockHeader().Return(mockHeader, nil).Times(2) - cs.blockState = mockBlockState - cs.handler = newBootstrapSyncer(mockBlockState) - mockNetwork := NewMockNetwork(ctrl) + mockedGenesisHeader := types.NewHeader(common.NewHash([]byte{0}), trie.EmptyHash, + trie.EmptyHash, 0, types.NewDigest()) + + const blocksAhead = 128 + totalBlockResponse := createSuccesfullBlockResponse(t, mockedGenesisHeader.Hash(), 1, blocksAhead) + mockedNetwork := NewMockNetwork(ctrl) + + workerPeerID := peer.ID("noot") startingBlock := variadic.MustNewUint32OrHash(1) max := uint32(128) - mockReqRes := NewMockRequestMaker(ctrl) - mockReqRes.EXPECT().Do(peer.ID("noot"), &network.BlockRequestMessage{ - RequestedData: 19, + mockedRequestMaker := NewMockRequestMaker(ctrl) + + expectedBlockRequestMessage := &network.BlockRequestMessage{ + RequestedData: network.BootstrapRequestData, StartingBlock: *startingBlock, - Direction: 0, + Direction: network.Ascending, Max: &max, - }, &network.BlockResponseMessage{}) - cs.blockReqRes = mockReqRes - cs.network = mockNetwork + } - go cs.sync() - defer cs.cancel() + mockedRequestMaker.EXPECT(). + Do(workerPeerID, expectedBlockRequestMessage, &network.BlockResponseMessage{}). + DoAndReturn(func(_, _, response any) any { + responsePtr := response.(*network.BlockResponseMessage) + *responsePtr = *totalBlockResponse + return nil + }) - testPeer := peer.ID("noot") - cs.peerState[testPeer] = &peerState{ - number: 1000, - } + mockedBlockState := NewMockBlockState(ctrl) + mockedBlockState.EXPECT().GetFinalisedNotifierChannel().Return(make(chan *types.FinalisationInfo)) + + mockBabeVerifier := NewMockBabeVerifier(ctrl) + mockStorageState := NewMockStorageState(ctrl) + mockImportHandler := NewMockBlockImportHandler(ctrl) + mockTelemetry := NewMockTelemetry(ctrl) + + const announceBlock = false + // setup mocks for new synced blocks that doesn't exists in our local database + ensureSuccessfulBlockImportFlow(t, mockedGenesisHeader, totalBlockResponse.BlockData, mockedBlockState, + mockBabeVerifier, mockStorageState, mockImportHandler, mockTelemetry, announceBlock) + + // setup a chain sync which holds in its peer view map + // 3 peers, each one announce block X as its best block number. + // We start this test with genesis block being our best block, so + // we're far behind by X blocks, we should execute a bootstrap + // sync request those blocks + cs := setupChainSyncToBootstrapMode(t, blocksAhead, + mockedBlockState, mockedNetwork, mockedRequestMaker, mockBabeVerifier, + mockStorageState, mockImportHandler, mockTelemetry) + + target, err := cs.getTarget() + require.NoError(t, err) + require.Equal(t, uint(128), target) - cs.workQueue <- cs.peerState[testPeer] + // include a new worker in the worker pool set, this worker + // should be an available peer that will receive a block request + // the worker pool executes the workers management + cs.workerPool.fromBlockAnnounce(peer.ID("noot")) - select { - case res := <-cs.resultQueue: - expected := &workerError{ - err: errEmptyBlockData, // since MockNetwork returns a nil response - who: testPeer, - } - require.Equal(t, expected, res.err) - case <-time.After(5 * time.Second): - t.Fatal("did not get worker response") - } + err = cs.requestMaxBlocksFrom(mockedGenesisHeader) + require.NoError(t, err) - require.Equal(t, bootstrap, cs.state) + err = cs.workerPool.stop() + require.NoError(t, err) } -func TestChainSync_sync_tip(t *testing.T) { +func TestChainSync_BootstrapSync_SuccessfulSync_WithTwoWorkers(t *testing.T) { t.Parallel() - done := make(chan struct{}) - ctrl := gomock.NewController(t) - cs := newTestChainSync(ctrl) - header := types.NewHeader(common.NewHash([]byte{0}), trie.EmptyHash, trie.EmptyHash, 1000, - types.NewDigest()) - - bs := NewMockBlockState(ctrl) - bs.EXPECT().BestBlockHeader().Return(header, nil) - bs.EXPECT().GetHighestFinalisedHeader().DoAndReturn(func() (*types.Header, error) { - close(done) - return header, nil - }) - cs.blockState = bs - - go cs.sync() - defer cs.cancel() - - testPeer := peer.ID("noot") - cs.peerState[testPeer] = &peerState{ - number: 999, - } + mockBlockState := NewMockBlockState(ctrl) + mockBlockState.EXPECT().GetFinalisedNotifierChannel().Return(make(chan *types.FinalisationInfo)) + mockedGenesisHeader := types.NewHeader(common.NewHash([]byte{0}), trie.EmptyHash, + trie.EmptyHash, 0, types.NewDigest()) - cs.workQueue <- cs.peerState[testPeer] - <-done - require.Equal(t, tip, cs.state) -} + mockNetwork := NewMockNetwork(ctrl) + mockRequestMaker := NewMockRequestMaker(ctrl) -func TestChainSync_getTarget(t *testing.T) { - ctrl := gomock.NewController(t) - cs := newTestChainSync(ctrl) - require.Equal(t, uint(1<<32-1), cs.getTarget()) - cs.peerState = map[peer.ID]*peerState{ - "a": { - number: 0, // outlier - }, - "b": { - number: 110, - }, - "c": { - number: 120, - }, - "d": { - number: 130, - }, - "e": { - number: 140, - }, - "f": { - number: 150, - }, - "g": { - number: 1000, // outlier - }, - } + mockBabeVerifier := NewMockBabeVerifier(ctrl) + mockStorageState := NewMockStorageState(ctrl) + mockImportHandler := NewMockBlockImportHandler(ctrl) + mockTelemetry := NewMockTelemetry(ctrl) - require.Equal(t, uint(130), cs.getTarget()) // sum:650/count:5= avg:130 + // this test expects two workers responding each request with 128 blocks which means + // we should import 256 blocks in total + blockResponse := createSuccesfullBlockResponse(t, mockedGenesisHeader.Hash(), 1, 256) - cs.peerState = map[peer.ID]*peerState{ - "testA": { - number: 1000, - }, - "testB": { - number: 2000, - }, + // here we split the whole set in two parts each one will be the "response" for each peer + worker1Response := &network.BlockResponseMessage{ + BlockData: blockResponse.BlockData[:128], } + const announceBlock = false + // the first peer will respond the from the block 1 to 128 so the ensureBlockImportFlow + // will setup the expectations starting from the genesis header until block 128 + ensureSuccessfulBlockImportFlow(t, mockedGenesisHeader, worker1Response.BlockData, mockBlockState, + mockBabeVerifier, mockStorageState, mockImportHandler, mockTelemetry, announceBlock) + + worker2Response := &network.BlockResponseMessage{ + BlockData: blockResponse.BlockData[128:], + } + // the worker 2 will respond from block 129 to 256 so the ensureBlockImportFlow + // will setup the expectations starting from block 128, from previous worker, until block 256 + parent := worker1Response.BlockData[len(worker1Response.BlockData)-1] + ensureSuccessfulBlockImportFlow(t, parent.Header, worker2Response.BlockData, mockBlockState, + mockBabeVerifier, mockStorageState, mockImportHandler, mockTelemetry, announceBlock) + + // we use gomock.Any since I cannot guarantee which peer picks which request + // but the first call to DoBlockRequest will return the first set and the second + // call will return the second set + mockRequestMaker.EXPECT(). + Do(gomock.Any(), gomock.Any(), &network.BlockResponseMessage{}). + DoAndReturn(func(_, _, response any) any { + responsePtr := response.(*network.BlockResponseMessage) + *responsePtr = *worker1Response + return nil + }) + + mockRequestMaker.EXPECT(). + Do(gomock.Any(), gomock.Any(), &network.BlockResponseMessage{}). + DoAndReturn(func(_, _, response any) any { + responsePtr := response.(*network.BlockResponseMessage) + *responsePtr = *worker2Response + return nil + }) + + // setup a chain sync which holds in its peer view map + // 3 peers, each one announce block 129 as its best block number. + // We start this test with genesis block being our best block, so + // we're far behind by 128 blocks, we should execute a bootstrap + // sync request those blocks + const blocksAhead = 256 + cs := setupChainSyncToBootstrapMode(t, blocksAhead, + mockBlockState, mockNetwork, mockRequestMaker, mockBabeVerifier, + mockStorageState, mockImportHandler, mockTelemetry) + + target, err := cs.getTarget() + require.NoError(t, err) + require.Equal(t, uint(blocksAhead), target) + + // include a new worker in the worker pool set, this worker + // should be an available peer that will receive a block request + // the worker pool executes the workers management + cs.workerPool.fromBlockAnnounce(peer.ID("noot")) + cs.workerPool.fromBlockAnnounce(peer.ID("noot2")) - require.Equal(t, uint(1500), cs.getTarget()) + err = cs.requestMaxBlocksFrom(mockedGenesisHeader) + require.NoError(t, err) + + err = cs.workerPool.stop() + require.NoError(t, err) } -func TestWorkerToRequests(t *testing.T) { +func TestChainSync_BootstrapSync_SuccessfulSync_WithOneWorkerFailing(t *testing.T) { t.Parallel() - w := &worker{ - startNumber: uintPtr(10), - targetNumber: uintPtr(1), - direction: network.Ascending, - } - _, err := workerToRequests(w) - require.Equal(t, errInvalidDirection, err) + ctrl := gomock.NewController(t) + mockBlockState := NewMockBlockState(ctrl) + mockBlockState.EXPECT().GetFinalisedNotifierChannel().Return(make(chan *types.FinalisationInfo)) + mockedGenesisHeader := types.NewHeader(common.NewHash([]byte{0}), trie.EmptyHash, + trie.EmptyHash, 0, types.NewDigest()) - type testCase struct { - w *worker - expected []*network.BlockRequestMessage - } + mockNetwork := NewMockNetwork(ctrl) + mockRequestMaker := NewMockRequestMaker(ctrl) - var ( - max128 = uint32(128) - max9 = uint32(9) - max64 = uint32(64) - ) - - testCases := map[string]testCase{ - "test_0": { - w: &worker{ - startNumber: uintPtr(1), - targetNumber: uintPtr(1 + maxResponseSize), - direction: network.Ascending, - requestData: bootstrapRequestData, - }, - expected: []*network.BlockRequestMessage{ - { - RequestedData: bootstrapRequestData, - StartingBlock: *variadic.MustNewUint32OrHash(1), - Direction: network.Ascending, - Max: &max128, - }, - }, - }, - "test_1": { - w: &worker{ - startNumber: uintPtr(1), - targetNumber: uintPtr(1 + (maxResponseSize * 2)), - direction: network.Ascending, - requestData: bootstrapRequestData, - }, - expected: []*network.BlockRequestMessage{ - { - RequestedData: bootstrapRequestData, - StartingBlock: *variadic.MustNewUint32OrHash(1), - Direction: network.Ascending, - Max: &max128, - }, - { - RequestedData: network.RequestedDataHeader + network.RequestedDataBody + network.RequestedDataJustification, - StartingBlock: *variadic.MustNewUint32OrHash(1 + maxResponseSize), - Direction: network.Ascending, - Max: &max128, - }, - }, - }, - "test_2": { - w: &worker{ - startNumber: uintPtr(1), - targetNumber: uintPtr(10), - direction: network.Ascending, - requestData: bootstrapRequestData, - }, - expected: []*network.BlockRequestMessage{ - { - RequestedData: bootstrapRequestData, - StartingBlock: *variadic.MustNewUint32OrHash(1), - Direction: network.Ascending, - Max: &max128, - }, - }, - }, - "test_3": { - w: &worker{ - startNumber: uintPtr(10), - targetNumber: uintPtr(1), - direction: network.Descending, - requestData: bootstrapRequestData, - }, - expected: []*network.BlockRequestMessage{ - { - RequestedData: bootstrapRequestData, - StartingBlock: *variadic.MustNewUint32OrHash(10), - Direction: network.Descending, - Max: &max9, - }, - }, - }, - "test_4": { - w: &worker{ - startNumber: uintPtr(1), - targetNumber: uintPtr(1 + maxResponseSize + (maxResponseSize / 2)), - direction: network.Ascending, - requestData: bootstrapRequestData, - }, - expected: []*network.BlockRequestMessage{ - { - RequestedData: bootstrapRequestData, - StartingBlock: *variadic.MustNewUint32OrHash(1), - Direction: network.Ascending, - Max: &max128, - }, - { - RequestedData: network.RequestedDataHeader + network.RequestedDataBody + network.RequestedDataJustification, - StartingBlock: *variadic.MustNewUint32OrHash(1 + maxResponseSize), - Direction: network.Ascending, - Max: &max128, - }, - }, - }, - "test_5": { - w: &worker{ - startNumber: uintPtr(1), - targetNumber: uintPtr(10), - targetHash: common.Hash{0xa}, - direction: network.Ascending, - requestData: bootstrapRequestData, - }, - expected: []*network.BlockRequestMessage{ - { - RequestedData: bootstrapRequestData, - StartingBlock: *variadic.MustNewUint32OrHash(1), - Direction: network.Ascending, - Max: &max128, - }, - }, - }, - "test_6": { - w: &worker{ - startNumber: uintPtr(1), - startHash: common.Hash{0xb}, - targetNumber: uintPtr(10), - targetHash: common.Hash{0xc}, - direction: network.Ascending, - requestData: bootstrapRequestData, - }, - expected: []*network.BlockRequestMessage{ - { - RequestedData: bootstrapRequestData, - StartingBlock: *variadic.MustNewUint32OrHash(common.Hash{0xb}), - Direction: network.Ascending, - Max: &max128, - }, - }, - }, - "test_7": { - w: &worker{ - startNumber: uintPtr(10), - targetNumber: uintPtr(10), - direction: network.Ascending, - requestData: bootstrapRequestData, - }, - expected: []*network.BlockRequestMessage{ - { - RequestedData: bootstrapRequestData, - StartingBlock: *variadic.MustNewUint32OrHash(10), - Direction: network.Ascending, - Max: &max128, - }, - }, - }, - "test_8": { - w: &worker{ - startNumber: uintPtr(1 + maxResponseSize + (maxResponseSize / 2)), - targetNumber: uintPtr(1), - direction: network.Descending, - requestData: bootstrapRequestData, - }, - expected: []*network.BlockRequestMessage{ - { - RequestedData: network.RequestedDataHeader + network.RequestedDataBody + network.RequestedDataJustification, - StartingBlock: *variadic.MustNewUint32OrHash(1 + (maxResponseSize / 2)), - Direction: network.Descending, - Max: &max64, - }, - { - RequestedData: bootstrapRequestData, - StartingBlock: *variadic.MustNewUint32OrHash(1 + maxResponseSize + (maxResponseSize / 2)), - Direction: network.Descending, - Max: &max128, - }, - }, - }, + mockBabeVerifier := NewMockBabeVerifier(ctrl) + mockStorageState := NewMockStorageState(ctrl) + mockImportHandler := NewMockBlockImportHandler(ctrl) + mockTelemetry := NewMockTelemetry(ctrl) + + // this test expects two workers responding each request with 128 blocks which means + // we should import 256 blocks in total + blockResponse := createSuccesfullBlockResponse(t, mockedGenesisHeader.Hash(), 1, 256) + const announceBlock = false + + // here we split the whole set in two parts each one will be the "response" for each peer + worker1Response := &network.BlockResponseMessage{ + BlockData: blockResponse.BlockData[:128], } - for name, tc := range testCases { - tc := tc - t.Run(name, func(t *testing.T) { - t.Parallel() - reqs, err := workerToRequests(tc.w) - require.NoError(t, err) - require.Equal(t, tc.expected, reqs) - }) + // the first peer will respond the from the block 1 to 128 so the ensureBlockImportFlow + // will setup the expectations starting from the genesis header until block 128 + ensureSuccessfulBlockImportFlow(t, mockedGenesisHeader, worker1Response.BlockData, mockBlockState, + mockBabeVerifier, mockStorageState, mockImportHandler, mockTelemetry, announceBlock) + + worker2Response := &network.BlockResponseMessage{ + BlockData: blockResponse.BlockData[128:], } + // the worker 2 will respond from block 129 to 256 so the ensureBlockImportFlow + // will setup the expectations starting from block 128, from previous worker, until block 256 + parent := worker1Response.BlockData[len(worker1Response.BlockData)-1] + ensureSuccessfulBlockImportFlow(t, parent.Header, worker2Response.BlockData, mockBlockState, + mockBabeVerifier, mockStorageState, mockImportHandler, mockTelemetry, announceBlock) + + // we use gomock.Any since I cannot guarantee which peer picks which request + // but the first call to DoBlockRequest will return the first set and the second + // call will return the second set + doBlockRequestCount := atomic.Int32{} + mockRequestMaker.EXPECT(). + Do(gomock.Any(), gomock.Any(), &network.BlockResponseMessage{}). + DoAndReturn(func(peerID, _, response any) any { + // lets ensure that the DoBlockRequest is called by + // peer.ID(alice) and peer.ID(bob). When bob calls, this method will fail + // then alice should pick the failed request and re-execute it which will + // be the third call + responsePtr := response.(*network.BlockResponseMessage) + defer func() { doBlockRequestCount.Add(1) }() + + switch doBlockRequestCount.Load() { + case 0: + *responsePtr = *worker1Response + case 1: + return errors.New("a bad error while getting a response") + default: + *responsePtr = *worker2Response + } + return nil + + }).Times(3) + + // setup a chain sync which holds in its peer view map + // 3 peers, each one announce block 129 as its best block number. + // We start this test with genesis block being our best block, so + // we're far behind by 128 blocks, we should execute a bootstrap + // sync request those blocks + const blocksAhead = 256 + cs := setupChainSyncToBootstrapMode(t, blocksAhead, + mockBlockState, mockNetwork, mockRequestMaker, mockBabeVerifier, + mockStorageState, mockImportHandler, mockTelemetry) + + target, err := cs.getTarget() + require.NoError(t, err) + require.Equal(t, uint(blocksAhead), target) + + // include a new worker in the worker pool set, this worker + // should be an available peer that will receive a block request + // the worker pool executes the workers management + cs.workerPool.fromBlockAnnounce(peer.ID("alice")) + cs.workerPool.fromBlockAnnounce(peer.ID("bob")) + + err = cs.requestMaxBlocksFrom(mockedGenesisHeader) + require.NoError(t, err) + + err = cs.workerPool.stop() + require.NoError(t, err) } -func TestChainSync_validateResponse(t *testing.T) { +func TestChainSync_BootstrapSync_SuccessfulSync_WithProtocolNotSupported(t *testing.T) { t.Parallel() - badBlockHash := common.NewHash([]byte("badblockhash")) - - tests := map[string]struct { - blockStateBuilder func(ctrl *gomock.Controller) BlockState - networkBuilder func(ctrl *gomock.Controller) Network - req *network.BlockRequestMessage - resp *network.BlockResponseMessage - expectedError error - }{ - "nil_req,_nil_resp": { - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().GetFinalisedNotifierChannel().Return(make(chan *types.FinalisationInfo)) - return mockBlockState - }, - networkBuilder: func(ctrl *gomock.Controller) Network { - return NewMockNetwork(ctrl) - }, - expectedError: errEmptyBlockData, - }, - "handle_error_response_is_not_chain,_has_header": { - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().GetFinalisedNotifierChannel().Return(make(chan *types.FinalisationInfo)) - mockBlockState.EXPECT().HasHeader(common.Hash{}).Return(true, nil) - return mockBlockState - }, - networkBuilder: func(ctrl *gomock.Controller) Network { - return NewMockNetwork(ctrl) - }, - req: &network.BlockRequestMessage{ - RequestedData: network.RequestedDataHeader, - }, - resp: &network.BlockResponseMessage{ - BlockData: []*types.BlockData{ - { - Header: &types.Header{ - Number: 1, - }, - Body: &types.Body{}, - }, - { - Header: &types.Header{ - Number: 2, - }, - Body: &types.Body{}, - }, - }, - }, - expectedError: errResponseIsNotChain, - }, - "handle_justification-only_request,_unknown_block": { - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().GetFinalisedNotifierChannel().Return(make(chan *types.FinalisationInfo)) - mockBlockState.EXPECT().HasHeader(common.Hash{}).Return(false, nil) - return mockBlockState - }, - networkBuilder: func(ctrl *gomock.Controller) Network { - mockNetwork := NewMockNetwork(ctrl) - mockNetwork.EXPECT().ReportPeer(peerset.ReputationChange{ - Value: peerset.BadJustificationValue, - Reason: peerset.BadJustificationReason, - }, peer.ID("")) - return mockNetwork - }, - req: &network.BlockRequestMessage{ - RequestedData: network.RequestedDataJustification, - }, - resp: &network.BlockResponseMessage{ - BlockData: []*types.BlockData{ - { - Justification: &[]byte{0}, - }, - }, - }, - expectedError: errUnknownBlockForJustification, - }, - "handle_error_unknown_parent": { - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().GetFinalisedNotifierChannel().Return(make(chan *types.FinalisationInfo)) - mockBlockState.EXPECT().HasHeader(common.Hash{}).Return(false, nil) - return mockBlockState - }, - networkBuilder: func(ctrl *gomock.Controller) Network { - return NewMockNetwork(ctrl) - }, - req: &network.BlockRequestMessage{ - RequestedData: network.RequestedDataHeader, - }, - resp: &network.BlockResponseMessage{ - BlockData: []*types.BlockData{ - { - Header: &types.Header{ - Number: 1, - }, - Body: &types.Body{}, - }, - { - Header: &types.Header{ - Number: 2, - }, - Body: &types.Body{}, - }, - }, - }, - expectedError: errUnknownParent, - }, - "handle_error_bad_block": { - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().GetFinalisedNotifierChannel().Return(make(chan *types.FinalisationInfo)) - return mockBlockState - }, - networkBuilder: func(ctrl *gomock.Controller) Network { - return NewMockNetwork(ctrl) - }, - req: &network.BlockRequestMessage{ - RequestedData: network.RequestedDataHeader, - }, - resp: &network.BlockResponseMessage{ - BlockData: []*types.BlockData{ - { - Hash: badBlockHash, - Header: &types.Header{ - Number: 2, - }, - Body: &types.Body{}, - }, - }, - }, - expectedError: errBadBlock, - }, - "no_error": { - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().GetFinalisedNotifierChannel().Return(make(chan *types.FinalisationInfo)) - mockBlockState.EXPECT().HasHeader(common.Hash{}).Return(true, nil) - return mockBlockState - }, - networkBuilder: func(ctrl *gomock.Controller) Network { - return NewMockNetwork(ctrl) - }, - req: &network.BlockRequestMessage{ - RequestedData: network.RequestedDataHeader, - }, - resp: &network.BlockResponseMessage{ - BlockData: []*types.BlockData{ - { - Header: &types.Header{ - Number: 2, - }, - Body: &types.Body{}, - }, - { - Header: &types.Header{ - ParentHash: (&types.Header{ - Number: 2, - }).Hash(), - Number: 3, - }, - Body: &types.Body{}, - }, - }, - }, - }, + + ctrl := gomock.NewController(t) + mockBlockState := NewMockBlockState(ctrl) + mockBlockState.EXPECT().GetFinalisedNotifierChannel().Return(make(chan *types.FinalisationInfo)) + mockedGenesisHeader := types.NewHeader(common.NewHash([]byte{0}), trie.EmptyHash, + trie.EmptyHash, 0, types.NewDigest()) + + mockNetwork := NewMockNetwork(ctrl) + mockRequestMaker := NewMockRequestMaker(ctrl) + + mockBabeVerifier := NewMockBabeVerifier(ctrl) + mockStorageState := NewMockStorageState(ctrl) + mockImportHandler := NewMockBlockImportHandler(ctrl) + mockTelemetry := NewMockTelemetry(ctrl) + + // this test expects two workers responding each request with 128 blocks which means + // we should import 256 blocks in total + blockResponse := createSuccesfullBlockResponse(t, mockedGenesisHeader.Hash(), 1, 256) + const announceBlock = false + + // here we split the whole set in two parts each one will be the "response" for each peer + worker1Response := &network.BlockResponseMessage{ + BlockData: blockResponse.BlockData[:128], } - for name, tt := range tests { - tt := tt - t.Run(name, func(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - cfg := chainSyncConfig{ - bs: tt.blockStateBuilder(ctrl), - pendingBlocks: newDisjointBlockSet(pendingBlocksLimit), - readyBlocks: newBlockQueue(maxResponseSize), - net: tt.networkBuilder(ctrl), - badBlocks: []string{ - badBlockHash.String(), - }, + // the first peer will respond the from the block 1 to 128 so the ensureBlockImportFlow + // will setup the expectations starting from the genesis header until block 128 + ensureSuccessfulBlockImportFlow(t, mockedGenesisHeader, worker1Response.BlockData, mockBlockState, + mockBabeVerifier, mockStorageState, mockImportHandler, mockTelemetry, announceBlock) + + worker2Response := &network.BlockResponseMessage{ + BlockData: blockResponse.BlockData[128:], + } + // the worker 2 will respond from block 129 to 256 so the ensureBlockImportFlow + // will setup the expectations starting from block 128, from previous worker, until block 256 + parent := worker1Response.BlockData[len(worker1Response.BlockData)-1] + ensureSuccessfulBlockImportFlow(t, parent.Header, worker2Response.BlockData, mockBlockState, + mockBabeVerifier, mockStorageState, mockImportHandler, mockTelemetry, announceBlock) + + // we use gomock.Any since I cannot guarantee which peer picks which request + // but the first call to DoBlockRequest will return the first set and the second + // call will return the second set + doBlockRequestCount := atomic.Int32{} + mockRequestMaker.EXPECT(). + Do(gomock.Any(), gomock.Any(), &network.BlockResponseMessage{}). + DoAndReturn(func(peerID, _, response any) any { + // lets ensure that the DoBlockRequest is called by + // peer.ID(alice) and peer.ID(bob). When bob calls, this method will fail + // then alice should pick the failed request and re-execute it which will + // be the third call + responsePtr := response.(*network.BlockResponseMessage) + defer func() { doBlockRequestCount.Add(1) }() + + switch doBlockRequestCount.Load() { + case 0: + *responsePtr = *worker1Response + case 1: + return errors.New("protocols not supported") + default: + *responsePtr = *worker2Response } - mockReqRes := NewMockRequestMaker(ctrl) - cs := newChainSync(cfg, mockReqRes) + return nil + }).Times(3) + + // since some peer will fail with protocols not supported his + // reputation will be affected and + mockNetwork.EXPECT().ReportPeer(peerset.ReputationChange{ + Value: peerset.BadProtocolValue, + Reason: peerset.BadProtocolReason, + }, gomock.AssignableToTypeOf(peer.ID(""))) + // setup a chain sync which holds in its peer view map + // 3 peers, each one announce block 129 as its best block number. + // We start this test with genesis block being our best block, so + // we're far behind by 128 blocks, we should execute a bootstrap + // sync request those blocks + const blocksAhead = 256 + cs := setupChainSyncToBootstrapMode(t, blocksAhead, + mockBlockState, mockNetwork, mockRequestMaker, mockBabeVerifier, + mockStorageState, mockImportHandler, mockTelemetry) + + target, err := cs.getTarget() + require.NoError(t, err) + require.Equal(t, uint(blocksAhead), target) - err := cs.validateResponse(tt.req, tt.resp, "") - if tt.expectedError != nil { - assert.EqualError(t, err, tt.expectedError.Error()) - } else { - assert.NoError(t, err) - } - }) - } + // include a new worker in the worker pool set, this worker + // should be an available peer that will receive a block request + // the worker pool executes the workers management + cs.workerPool.fromBlockAnnounce(peer.ID("alice")) + cs.workerPool.fromBlockAnnounce(peer.ID("bob")) + + err = cs.requestMaxBlocksFrom(mockedGenesisHeader) + require.NoError(t, err) + + err = cs.workerPool.stop() + require.NoError(t, err) } -func TestChainSync_doSync(t *testing.T) { +func TestChainSync_BootstrapSync_SuccessfulSync_WithNilHeaderInResponse(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) - readyBlocks := newBlockQueue(maxResponseSize) - cs := newTestChainSyncWithReadyBlocks(ctrl, readyBlocks) - - max := uint32(1) - req := &network.BlockRequestMessage{ - RequestedData: bootstrapRequestData, - StartingBlock: *variadic.MustNewUint32OrHash(1), - Direction: network.Ascending, - Max: &max, - } - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().HasHeader(common.Hash{}).Return(true, nil).Times(2) - cs.blockState = mockBlockState + mockBlockState.EXPECT().GetFinalisedNotifierChannel().Return(make(chan *types.FinalisationInfo)) + mockedGenesisHeader := types.NewHeader(common.NewHash([]byte{0}), trie.EmptyHash, + trie.EmptyHash, 0, types.NewDigest()) - workerErr := cs.doSync(req, make(map[peer.ID]struct{})) - require.NotNil(t, workerErr) - require.Equal(t, errNoPeers, workerErr.err) + mockNetwork := NewMockNetwork(ctrl) + mockRequestMaker := NewMockRequestMaker(ctrl) - cs.peerState["noot"] = &peerState{ - number: 100, - } + mockBabeVerifier := NewMockBabeVerifier(ctrl) + mockStorageState := NewMockStorageState(ctrl) + mockImportHandler := NewMockBlockImportHandler(ctrl) + mockTelemetry := NewMockTelemetry(ctrl) - mockNetwork := NewMockNetwork(ctrl) - startingBlock := variadic.MustNewUint32OrHash(1) - max1 := uint32(1) + // this test expects two workers responding each request with 128 blocks which means + // we should import 256 blocks in total + blockResponse := createSuccesfullBlockResponse(t, mockedGenesisHeader.Hash(), 1, 256) + const announceBlock = false - mockReqRes := NewMockRequestMaker(ctrl) - mockReqRes.EXPECT().Do(peer.ID("noot"), &network.BlockRequestMessage{ - RequestedData: 19, - StartingBlock: *startingBlock, - Direction: 0, - Max: &max1, - }, &network.BlockResponseMessage{}) - cs.blockReqRes = mockReqRes - - cs.network = mockNetwork - - workerErr = cs.doSync(req, make(map[peer.ID]struct{})) - require.NotNil(t, workerErr) - require.Equal(t, errEmptyBlockData, workerErr.err) - - expectedResp := &network.BlockResponseMessage{ - BlockData: []*types.BlockData{ - { - Hash: common.Hash{0x1}, - Header: &types.Header{ - Number: 1, - }, - Body: &types.Body{}, - }, - }, + // here we split the whole set in two parts each one will be the "response" for each peer + worker1Response := &network.BlockResponseMessage{ + BlockData: blockResponse.BlockData[:128], } - mockReqRes.EXPECT().Do(peer.ID("noot"), &network.BlockRequestMessage{ - RequestedData: 19, - StartingBlock: *startingBlock, - Direction: 0, - Max: &max1, - }, &network.BlockResponseMessage{}).Do( - func(_ peer.ID, _ *network.BlockRequestMessage, resp *network.BlockResponseMessage) { - *resp = *expectedResp - }, - ) + // the first peer will respond the from the block 1 to 128 so the ensureBlockImportFlow + // will setup the expectations starting from the genesis header until block 128 + ensureSuccessfulBlockImportFlow(t, mockedGenesisHeader, worker1Response.BlockData, mockBlockState, + mockBabeVerifier, mockStorageState, mockImportHandler, mockTelemetry, announceBlock) - workerErr = cs.doSync(req, make(map[peer.ID]struct{})) - require.Nil(t, workerErr) - bd, err := readyBlocks.pop(context.Background()) - require.NotNil(t, bd) - require.NoError(t, err) - require.Equal(t, expectedResp.BlockData[0], bd) - - parent := (&types.Header{ - Number: 2, - }).Hash() - expectedResp = &network.BlockResponseMessage{ - BlockData: []*types.BlockData{ - { - Hash: common.Hash{0x3}, - Header: &types.Header{ - ParentHash: parent, - Number: 3, - }, - Body: &types.Body{}, - }, - { - Hash: common.Hash{0x2}, - Header: &types.Header{ - Number: 2, - }, - Body: &types.Body{}, - }, - }, + worker2Response := &network.BlockResponseMessage{ + BlockData: blockResponse.BlockData[128:], } + // the worker 2 will respond from block 129 to 256 so the ensureBlockImportFlow + // will setup the expectations starting from block 128, from previous worker, until block 256 + parent := worker1Response.BlockData[127] + ensureSuccessfulBlockImportFlow(t, parent.Header, worker2Response.BlockData, mockBlockState, + mockBabeVerifier, mockStorageState, mockImportHandler, mockTelemetry, announceBlock) + + // we use gomock.Any since I cannot guarantee which peer picks which request + // but the first call to DoBlockRequest will return the first set and the second + // call will return the second set + doBlockRequestCount := atomic.Int32{} + mockRequestMaker.EXPECT(). + Do(gomock.Any(), gomock.Any(), &network.BlockResponseMessage{}). + DoAndReturn(func(peerID, _, response any) any { + // lets ensure that the DoBlockRequest is called by + // peer.ID(alice) and peer.ID(bob). When bob calls, this method return an + // response item but without header as was requested + responsePtr := response.(*network.BlockResponseMessage) + defer func() { doBlockRequestCount.Add(1) }() + + switch doBlockRequestCount.Load() { + case 0: + *responsePtr = *worker1Response + case 1: + incompleteBlockData := createSuccesfullBlockResponse(t, mockedGenesisHeader.Hash(), 128, 256) + incompleteBlockData.BlockData[0].Header = nil + + *responsePtr = *incompleteBlockData + default: + *responsePtr = *worker2Response + } - // test to see if descending blocks get reversed - req.Direction = network.Descending - - mockReqRes.EXPECT().Do(peer.ID("noot"), &network.BlockRequestMessage{ - RequestedData: 19, - StartingBlock: *startingBlock, - Direction: 1, - Max: &max1, - }, &network.BlockResponseMessage{}).Do( - func(_ peer.ID, _ *network.BlockRequestMessage, resp *network.BlockResponseMessage) { - *resp = *expectedResp - }, - ) + return nil + }).Times(3) + + // since some peer will fail with protocols not supported his + // reputation will be affected and + mockNetwork.EXPECT().ReportPeer(peerset.ReputationChange{ + Value: peerset.IncompleteHeaderValue, + Reason: peerset.IncompleteHeaderReason, + }, gomock.AssignableToTypeOf(peer.ID(""))) + // setup a chain sync which holds in its peer view map + // 3 peers, each one announce block 129 as its best block number. + // We start this test with genesis block being our best block, so + // we're far behind by 128 blocks, we should execute a bootstrap + // sync request those blocks + const blocksAhead = 256 + cs := setupChainSyncToBootstrapMode(t, blocksAhead, + mockBlockState, mockNetwork, mockRequestMaker, mockBabeVerifier, + mockStorageState, mockImportHandler, mockTelemetry) + + target, err := cs.getTarget() + require.NoError(t, err) + require.Equal(t, uint(blocksAhead), target) - cs.network = mockNetwork - workerErr = cs.doSync(req, make(map[peer.ID]struct{})) - require.Nil(t, workerErr) + // include a new worker in the worker pool set, this worker + // should be an available peer that will receive a block request + // the worker pool executes the workers management + cs.workerPool.fromBlockAnnounce(peer.ID("alice")) + cs.workerPool.fromBlockAnnounce(peer.ID("bob")) - bd, err = readyBlocks.pop(context.Background()) - require.NotNil(t, bd) - require.Equal(t, expectedResp.BlockData[0], bd) + err = cs.requestMaxBlocksFrom(mockedGenesisHeader) require.NoError(t, err) - bd, err = readyBlocks.pop(context.Background()) - require.NotNil(t, bd) - require.Equal(t, expectedResp.BlockData[1], bd) + err = cs.workerPool.stop() require.NoError(t, err) } -func TestHandleReadyBlock(t *testing.T) { +func TestChainSync_BootstrapSync_SuccessfulSync_WithResponseIsNotAChain(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) - readyBlocks := newBlockQueue(maxResponseSize) - cs := newTestChainSyncWithReadyBlocks(ctrl, readyBlocks) + mockBlockState := NewMockBlockState(ctrl) + mockBlockState.EXPECT().GetFinalisedNotifierChannel().Return(make(chan *types.FinalisationInfo)) + mockedGenesisHeader := types.NewHeader(common.NewHash([]byte{0}), trie.EmptyHash, + trie.EmptyHash, 0, types.NewDigest()) - // test that descendant chain gets returned by getReadyDescendants on block 1 being ready - header1 := &types.Header{ - Number: 1, - } - block1 := &types.Block{ - Header: *header1, - Body: types.Body{}, - } + mockNetwork := NewMockNetwork(ctrl) + mockRequestMaker := NewMockRequestMaker(ctrl) - header2 := &types.Header{ - ParentHash: header1.Hash(), - Number: 2, - } - block2 := &types.Block{ - Header: *header2, - Body: types.Body{}, - } - cs.pendingBlocks.addBlock(block2) + mockBabeVerifier := NewMockBabeVerifier(ctrl) + mockStorageState := NewMockStorageState(ctrl) + mockImportHandler := NewMockBlockImportHandler(ctrl) + mockTelemetry := NewMockTelemetry(ctrl) - header3 := &types.Header{ - ParentHash: header2.Hash(), - Number: 3, - } - block3 := &types.Block{ - Header: *header3, - Body: types.Body{}, - } - cs.pendingBlocks.addBlock(block3) + // this test expects two workers responding each request with 128 blocks which means + // we should import 256 blocks in total + blockResponse := createSuccesfullBlockResponse(t, mockedGenesisHeader.Hash(), 1, 256) + const announceBlock = false - header2NotDescendant := &types.Header{ - ParentHash: common.Hash{0xff}, - Number: 2, + // here we split the whole set in two parts each one will be the "response" for each peer + worker1Response := &network.BlockResponseMessage{ + BlockData: blockResponse.BlockData[:128], } - block2NotDescendant := &types.Block{ - Header: *header2NotDescendant, - Body: types.Body{}, + + // the first peer will respond the from the block 1 to 128 so the ensureBlockImportFlow + // will setup the expectations starting from the genesis header until block 128 + ensureSuccessfulBlockImportFlow(t, mockedGenesisHeader, worker1Response.BlockData, mockBlockState, + mockBabeVerifier, mockStorageState, mockImportHandler, mockTelemetry, announceBlock) + + worker2Response := &network.BlockResponseMessage{ + BlockData: blockResponse.BlockData[128:], } - cs.pendingBlocks.addBlock(block2NotDescendant) + // the worker 2 will respond from block 129 to 256 so the ensureBlockImportFlow + // will setup the expectations starting from block 128, from previous worker, until block 256 + parent := worker1Response.BlockData[127] + ensureSuccessfulBlockImportFlow(t, parent.Header, worker2Response.BlockData, mockBlockState, + mockBabeVerifier, mockStorageState, mockImportHandler, mockTelemetry, announceBlock) + + // we use gomock.Any since I cannot guarantee which peer picks which request + // but the first call to DoBlockRequest will return the first set and the second + // call will return the second set + doBlockRequestCount := atomic.Int32{} + mockRequestMaker.EXPECT(). + Do(gomock.Any(), gomock.Any(), &network.BlockResponseMessage{}). + DoAndReturn(func(peerID, _, response any) any { + // lets ensure that the DoBlockRequest is called by + // peer.ID(alice) and peer.ID(bob). When bob calls, this method return an + // response that does not form an chain + responsePtr := response.(*network.BlockResponseMessage) + defer func() { doBlockRequestCount.Add(1) }() + + switch doBlockRequestCount.Load() { + case 0: + *responsePtr = *worker1Response + case 1: + notAChainBlockData := createSuccesfullBlockResponse(t, mockedGenesisHeader.Hash(), 128, 256) + // swap positions to force the problem + notAChainBlockData.BlockData[0], notAChainBlockData.BlockData[130] = + notAChainBlockData.BlockData[130], notAChainBlockData.BlockData[0] + + *responsePtr = *notAChainBlockData + default: + *responsePtr = *worker2Response + } - cs.handleReadyBlock(block1.ToBlockData()) + return nil + }).Times(3) - require.False(t, cs.pendingBlocks.(*disjointBlockSet).hasBlock(header1.Hash())) - require.False(t, cs.pendingBlocks.(*disjointBlockSet).hasBlock(header2.Hash())) - require.False(t, cs.pendingBlocks.(*disjointBlockSet).hasBlock(header3.Hash())) - require.True(t, cs.pendingBlocks.(*disjointBlockSet).hasBlock(header2NotDescendant.Hash())) + // setup a chain sync which holds in its peer view map + // 3 peers, each one announce block 129 as its best block number. + // We start this test with genesis block being our best block, so + // we're far behind by 128 blocks, we should execute a bootstrap + // sync request those blocks + const blocksAhead = 256 + cs := setupChainSyncToBootstrapMode(t, blocksAhead, + mockBlockState, mockNetwork, mockRequestMaker, mockBabeVerifier, + mockStorageState, mockImportHandler, mockTelemetry) - blockData1, err := readyBlocks.pop(context.Background()) + target, err := cs.getTarget() require.NoError(t, err) - require.Equal(t, block1.ToBlockData(), blockData1) + require.Equal(t, uint(blocksAhead), target) - blockData2, err := readyBlocks.pop(context.Background()) + // include a new worker in the worker pool set, this worker + // should be an available peer that will receive a block request + // the worker pool executes the workers management + cs.workerPool.fromBlockAnnounce(peer.ID("alice")) + cs.workerPool.fromBlockAnnounce(peer.ID("bob")) + + err = cs.requestMaxBlocksFrom(mockedGenesisHeader) require.NoError(t, err) - require.Equal(t, block2.ToBlockData(), blockData2) - blockData3, err := readyBlocks.pop(context.Background()) + err = cs.workerPool.stop() require.NoError(t, err) - require.Equal(t, block3.ToBlockData(), blockData3) } -func TestChainSync_determineSyncPeers(t *testing.T) { +func TestChainSync_BootstrapSync_SuccessfulSync_WithReceivedBadBlock(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) - cs := newTestChainSync(ctrl) + mockBlockState := NewMockBlockState(ctrl) + mockBlockState.EXPECT().GetFinalisedNotifierChannel().Return(make(chan *types.FinalisationInfo)) + mockedGenesisHeader := types.NewHeader(common.NewHash([]byte{0}), trie.EmptyHash, + trie.EmptyHash, 0, types.NewDigest()) + + mockNetwork := NewMockNetwork(ctrl) + mockRequestMaker := NewMockRequestMaker(ctrl) + + mockBabeVerifier := NewMockBabeVerifier(ctrl) + mockStorageState := NewMockStorageState(ctrl) + mockImportHandler := NewMockBlockImportHandler(ctrl) + mockTelemetry := NewMockTelemetry(ctrl) - req := &network.BlockRequestMessage{} - testPeerA := peer.ID("a") - testPeerB := peer.ID("b") - peersTried := make(map[peer.ID]struct{}) + // this test expects two workers responding each request with 128 blocks which means + // we should import 256 blocks in total + blockResponse := createSuccesfullBlockResponse(t, mockedGenesisHeader.Hash(), 1, 256) + const announceBlock = false - // test base case - cs.peerState[testPeerA] = &peerState{ - number: 129, + // here we split the whole set in two parts each one will be the "response" for each peer + worker1Response := &network.BlockResponseMessage{ + BlockData: blockResponse.BlockData[:128], } - cs.peerState[testPeerB] = &peerState{ - number: 257, + + // the first peer will respond the from the block 1 to 128 so the ensureBlockImportFlow + // will setup the expectations starting from the genesis header until block 128 + ensureSuccessfulBlockImportFlow(t, mockedGenesisHeader, worker1Response.BlockData, mockBlockState, + mockBabeVerifier, mockStorageState, mockImportHandler, mockTelemetry, announceBlock) + + worker2Response := &network.BlockResponseMessage{ + BlockData: blockResponse.BlockData[128:], } + // the worker 2 will respond from block 129 to 256 so the ensureBlockImportFlow + // will setup the expectations starting from block 128, from previous worker, until block 256 + parent := worker1Response.BlockData[len(worker1Response.BlockData)-1] + ensureSuccessfulBlockImportFlow(t, parent.Header, worker2Response.BlockData, mockBlockState, + mockBabeVerifier, mockStorageState, mockImportHandler, mockTelemetry, announceBlock) + + fakeBadBlockHash := common.MustHexToHash("0x18767cb4bb4cc13bf119f6613aec5487d4c06a2e453de53d34aea6f3f1ee9855") + + // we use gomock.Any since I cannot guarantee which peer picks which request + // but the first call to DoBlockRequest will return the first set and the second + // call will return the second set + doBlockRequestCount := atomic.Int32{} + mockRequestMaker.EXPECT(). + Do(gomock.Any(), gomock.Any(), &network.BlockResponseMessage{}). + DoAndReturn(func(peerID, _, response any) any { + // lets ensure that the DoBlockRequest is called by + // peer.ID(alice) and peer.ID(bob). When bob calls, this method return an + // response that contains a know bad block + responsePtr := response.(*network.BlockResponseMessage) + defer func() { doBlockRequestCount.Add(1) }() + + switch doBlockRequestCount.Load() { + case 0: + *responsePtr = *worker1Response + case 1: + // use the fisrt response last item hash to produce the second response block data + // so we can guarantee that the second response continues the first response blocks + firstResponseLastItem := worker1Response.BlockData[len(worker1Response.BlockData)-1] + blockDataWithBadBlock := createSuccesfullBlockResponse(t, + firstResponseLastItem.Header.Hash(), + 129, + 128) + + // changes the last item from the second response to be a bad block, so we guarantee that + // this second response is a chain, (changing the hash from a block in the middle of the block + // response brokes the `isAChain` verification) + lastItem := len(blockDataWithBadBlock.BlockData) - 1 + blockDataWithBadBlock.BlockData[lastItem].Hash = fakeBadBlockHash + *responsePtr = *blockDataWithBadBlock + default: + *responsePtr = *worker2Response + } + + return nil + }).Times(3) + + mockNetwork.EXPECT().ReportPeer(peerset.ReputationChange{ + Value: peerset.BadBlockAnnouncementValue, + Reason: peerset.BadBlockAnnouncementReason, + }, gomock.AssignableToTypeOf(peer.ID(""))) + // setup a chain sync which holds in its peer view map + // 3 peers, each one announce block 129 as its best block number. + // We start this test with genesis block being our best block, so + // we're far behind by 128 blocks, we should execute a bootstrap + // sync request those blocks + const blocksAhead = 256 + cs := setupChainSyncToBootstrapMode(t, blocksAhead, + mockBlockState, mockNetwork, mockRequestMaker, mockBabeVerifier, + mockStorageState, mockImportHandler, mockTelemetry) + + cs.badBlocks = []string{fakeBadBlockHash.String()} + + target, err := cs.getTarget() + require.NoError(t, err) + require.Equal(t, uint(blocksAhead), target) + + // include a new worker in the worker pool set, this worker + // should be an available peer that will receive a block request + // the worker pool executes the workers management + cs.workerPool.fromBlockAnnounce(peer.ID("alice")) + cs.workerPool.fromBlockAnnounce(peer.ID("bob")) + + err = cs.requestMaxBlocksFrom(mockedGenesisHeader) + require.NoError(t, err) - peers := cs.determineSyncPeers(req, peersTried) - require.Equal(t, 2, len(peers)) - require.Contains(t, peers, testPeerA) - require.Contains(t, peers, testPeerB) - - // test peer ignored case - cs.ignorePeers[testPeerA] = struct{}{} - peers = cs.determineSyncPeers(req, peersTried) - require.Equal(t, 1, len(peers)) - require.Equal(t, []peer.ID{testPeerB}, peers) - - // test all peers ignored case - cs.ignorePeers[testPeerB] = struct{}{} - peers = cs.determineSyncPeers(req, peersTried) - require.Equal(t, 2, len(peers)) - require.Contains(t, peers, testPeerA) - require.Contains(t, peers, testPeerB) - require.Equal(t, 0, len(cs.ignorePeers)) - - // test peer's best block below number case, shouldn't include that peer - start, err := variadic.NewUint32OrHash(130) + err = cs.workerPool.stop() require.NoError(t, err) - req.StartingBlock = *start - peers = cs.determineSyncPeers(req, peersTried) - require.Equal(t, 1, len(peers)) - require.Equal(t, []peer.ID{testPeerB}, peers) - - // test peer tried case, should ignore peer already tried - peersTried[testPeerA] = struct{}{} - req.StartingBlock = variadic.Uint32OrHash{} - peers = cs.determineSyncPeers(req, peersTried) - require.Equal(t, 1, len(peers)) - require.Equal(t, []peer.ID{testPeerB}, peers) + + // peer should be not in the worker pool + // peer should be in the ignore list + require.Len(t, cs.workerPool.workers, 1) + require.Len(t, cs.workerPool.ignorePeers, 1) } -func Test_chainSync_logSyncSpeed(t *testing.T) { +func TestChainSync_BootstrapSync_SucessfulSync_ReceivedPartialBlockData(t *testing.T) { t.Parallel() - type fields struct { - blockStateBuilder func(ctrl *gomock.Controller) BlockState - networkBuilder func(ctrl *gomock.Controller) Network - state chainSyncState - benchmarker *syncBenchmarker + ctrl := gomock.NewController(t) + mockBlockState := NewMockBlockState(ctrl) + mockBlockState.EXPECT().GetFinalisedNotifierChannel().Return(make(chan *types.FinalisationInfo)) + mockedGenesisHeader := types.NewHeader(common.NewHash([]byte{0}), trie.EmptyHash, + trie.EmptyHash, 0, types.NewDigest()) + + mockNetwork := NewMockNetwork(ctrl) + mockRequestMaker := NewMockRequestMaker(ctrl) + + mockBabeVerifier := NewMockBabeVerifier(ctrl) + mockStorageState := NewMockStorageState(ctrl) + mockImportHandler := NewMockBlockImportHandler(ctrl) + mockTelemetry := NewMockTelemetry(ctrl) + + // create a set of 128 blocks + blockResponse := createSuccesfullBlockResponse(t, mockedGenesisHeader.Hash(), 1, 128) + const announceBlock = false + + // the worker will return a partial size of the set + worker1Response := &network.BlockResponseMessage{ + BlockData: blockResponse.BlockData[:97], } - tests := []struct { - name string - fields fields - }{ - { - name: "state_bootstrap", - fields: fields{ - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().BestBlockHeader().Return(&types.Header{}, nil).Times(3) - mockBlockState.EXPECT().GetHighestFinalisedHeader().Return(&types.Header{}, nil) - return mockBlockState - }, - networkBuilder: func(ctrl *gomock.Controller) Network { - mockNetwork := NewMockNetwork(ctrl) - mockNetwork.EXPECT().Peers().Return(nil) - return mockNetwork - }, - benchmarker: newSyncBenchmarker(10), - state: bootstrap, - }, - }, - { - name: "case_tip", - fields: fields{ - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().BestBlockHeader().Return(&types.Header{}, nil).Times(3) - mockBlockState.EXPECT().GetHighestFinalisedHeader().Return(&types.Header{}, nil) - return mockBlockState - }, - networkBuilder: func(ctrl *gomock.Controller) Network { - mockNetwork := NewMockNetwork(ctrl) - mockNetwork.EXPECT().Peers().Return(nil) - return mockNetwork - }, - benchmarker: newSyncBenchmarker(10), - state: tip, - }, - }, + + // the first peer will respond the from the block 1 to 96 so the ensureBlockImportFlow + // will setup the expectations starting from the genesis header until block 96 + ensureSuccessfulBlockImportFlow(t, mockedGenesisHeader, worker1Response.BlockData, mockBlockState, + mockBabeVerifier, mockStorageState, mockImportHandler, mockTelemetry, announceBlock) + + worker1MissingBlocksResponse := &network.BlockResponseMessage{ + BlockData: blockResponse.BlockData[97:], } - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - ctx, cancel := context.WithCancel(context.Background()) - tickerChannel := make(chan time.Time) - cs := &chainSync{ - ctx: ctx, - cancel: cancel, - blockState: tt.fields.blockStateBuilder(ctrl), - network: tt.fields.networkBuilder(ctrl), - state: tt.fields.state, - benchmarker: tt.fields.benchmarker, - logSyncTickerC: tickerChannel, - logSyncTicker: time.NewTicker(time.Hour), // just here to be stopped - logSyncDone: make(chan struct{}), + + // last item from the previous response + parent := worker1Response.BlockData[96] + ensureSuccessfulBlockImportFlow(t, parent.Header, worker1MissingBlocksResponse.BlockData, mockBlockState, + mockBabeVerifier, mockStorageState, mockImportHandler, mockTelemetry, announceBlock) + + doBlockRequestCount := 0 + mockRequestMaker.EXPECT(). + Do(gomock.Any(), gomock.Any(), &network.BlockResponseMessage{}). + DoAndReturn(func(peerID, _, response any) any { + // lets ensure that the DoBlockRequest is called by + // peer.ID(alice). The first call will return only 97 blocks + // the handler should issue another call to retrieve the missing blocks + responsePtr := response.(*network.BlockResponseMessage) + defer func() { doBlockRequestCount++ }() + + if doBlockRequestCount == 0 { + *responsePtr = *worker1Response + } else { + *responsePtr = *worker1MissingBlocksResponse } - go cs.logSyncSpeed() + return nil + }).Times(2) - tickerChannel <- time.Time{} - cs.cancel() - <-cs.logSyncDone - }) - } + const blocksAhead = 128 + cs := setupChainSyncToBootstrapMode(t, blocksAhead, + mockBlockState, mockNetwork, mockRequestMaker, mockBabeVerifier, + mockStorageState, mockImportHandler, mockTelemetry) + + target, err := cs.getTarget() + require.NoError(t, err) + require.Equal(t, uint(blocksAhead), target) + + cs.workerPool.fromBlockAnnounce(peer.ID("alice")) + + err = cs.requestMaxBlocksFrom(mockedGenesisHeader) + require.NoError(t, err) + + err = cs.workerPool.stop() + require.NoError(t, err) + + require.Len(t, cs.workerPool.workers, 1) + + _, ok := cs.workerPool.workers[peer.ID("alice")] + require.True(t, ok) } -func Test_chainSync_start(t *testing.T) { - t.Parallel() +func createSuccesfullBlockResponse(t *testing.T, parentHeader common.Hash, + startingAt, numBlocks int) *network.BlockResponseMessage { + t.Helper() + + response := new(network.BlockResponseMessage) + response.BlockData = make([]*types.BlockData, numBlocks) - type fields struct { - blockStateBuilder func(ctrl *gomock.Controller) BlockState - disjointBlockSetBuilder func(ctrl *gomock.Controller, called chan<- struct{}) DisjointBlockSet - benchmarker *syncBenchmarker + emptyTrieState := storage.NewTrieState(nil) + tsRoot := emptyTrieState.MustRoot() + + firstHeader := types.NewHeader(parentHeader, tsRoot, common.Hash{}, + uint(startingAt), scale.VaryingDataTypeSlice{}) + response.BlockData[0] = &types.BlockData{ + Hash: firstHeader.Hash(), + Header: firstHeader, + Body: types.NewBody([]types.Extrinsic{}), + Justification: nil, } - tests := []struct { - name string - fields fields - }{ - { - name: "base_case", - fields: fields{ - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().BestBlockHeader().Return(&types.Header{}, nil) - return mockBlockState - }, - disjointBlockSetBuilder: func(ctrl *gomock.Controller, called chan<- struct{}) DisjointBlockSet { - mockDisjointBlockSet := NewMockDisjointBlockSet(ctrl) - mockDisjointBlockSet.EXPECT().run(gomock.AssignableToTypeOf(make(<-chan struct{}))). - DoAndReturn(func(stop <-chan struct{}) { - close(called) // test glue, ideally we would use a ready chan struct passed to run(). - }) - return mockDisjointBlockSet - }, - benchmarker: newSyncBenchmarker(1), - }, - }, + + parentHash := firstHeader.Hash() + for idx := 1; idx < numBlocks; idx++ { + blockNumber := idx + startingAt + header := types.NewHeader(parentHash, tsRoot, common.Hash{}, + uint(blockNumber), scale.VaryingDataTypeSlice{}) + response.BlockData[idx] = &types.BlockData{ + Hash: header.Hash(), + Header: header, + Body: types.NewBody([]types.Extrinsic{}), + Justification: nil, + } + parentHash = header.Hash() } - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - ctx, cancel := context.WithCancel(context.Background()) - disjointBlockSetCalled := make(chan struct{}) - cs := &chainSync{ - ctx: ctx, - cancel: cancel, - blockState: tt.fields.blockStateBuilder(ctrl), - pendingBlocks: tt.fields.disjointBlockSetBuilder(ctrl, disjointBlockSetCalled), - benchmarker: tt.fields.benchmarker, - slotDuration: time.Hour, - logSyncTicker: time.NewTicker(time.Hour), // just here to be closed - logSyncDone: make(chan struct{}), - } - cs.start() - <-disjointBlockSetCalled - cs.stop() - }) + + return response +} + +// ensureSuccessfulBlockImportFlow will setup the expectations for method calls +// that happens while chain sync imports a block +func ensureSuccessfulBlockImportFlow(t *testing.T, parentHeader *types.Header, + blocksReceived []*types.BlockData, mockBlockState *MockBlockState, + mockBabeVerifier *MockBabeVerifier, mockStorageState *MockStorageState, + mockImportHandler *MockBlockImportHandler, mockTelemetry *MockTelemetry, announceBlock bool) { + t.Helper() + + for idx, blockData := range blocksReceived { + mockBlockState.EXPECT().HasHeader(blockData.Header.Hash()).Return(false, nil) + mockBlockState.EXPECT().HasBlockBody(blockData.Header.Hash()).Return(false, nil) + mockBabeVerifier.EXPECT().VerifyBlock(blockData.Header).Return(nil) + + var previousHeader *types.Header + if idx == 0 { + previousHeader = parentHeader + } else { + previousHeader = blocksReceived[idx-1].Header + } + + mockBlockState.EXPECT().GetHeader(blockData.Header.ParentHash).Return(previousHeader, nil) + mockStorageState.EXPECT().Lock() + mockStorageState.EXPECT().Unlock() + + emptyTrieState := storage.NewTrieState(nil) + parentStateRoot := previousHeader.StateRoot + mockStorageState.EXPECT().TrieState(&parentStateRoot). + Return(emptyTrieState, nil) + + ctrl := gomock.NewController(t) + mockRuntimeInstance := NewMockInstance(ctrl) + mockBlockState.EXPECT().GetRuntime(previousHeader.Hash()). + Return(mockRuntimeInstance, nil) + + expectedBlock := &types.Block{ + Header: *blockData.Header, + Body: *blockData.Body, + } + + mockRuntimeInstance.EXPECT().SetContextStorage(emptyTrieState) + mockRuntimeInstance.EXPECT().ExecuteBlock(expectedBlock). + Return(nil, nil) + + mockImportHandler.EXPECT().HandleBlockImport(expectedBlock, emptyTrieState, announceBlock). + Return(nil) + + blockHash := blockData.Header.Hash() + expectedTelemetryMessage := telemetry.NewBlockImport( + &blockHash, + blockData.Header.Number, + "NetworkInitialSync") + mockTelemetry.EXPECT().SendMessage(expectedTelemetryMessage) + + mockBlockState.EXPECT().CompareAndSetBlockData(blockData).Return(nil) } } -func Test_chainSync_setBlockAnnounce(t *testing.T) { +func TestChainSync_validateResponseFields(t *testing.T) { t.Parallel() - type args struct { - from peer.ID - header *types.Header + block1Header := &types.Header{ + ParentHash: common.MustHexToHash("0x00597cb4bb4cc13bf119f6613aec7642d4c06a2e453de53d34aea6f3f1eeb504"), + Number: 2, + } + + block2Header := &types.Header{ + ParentHash: block1Header.Hash(), + Number: 3, } - tests := map[string]struct { - chainSyncBuilder func(*types.Header, *gomock.Controller) chainSync - args args - wantErr error + + cases := map[string]struct { + wantErr error + errString string + setupChainSync func(t *testing.T) *chainSync + requestedData byte + blockData *types.BlockData }{ - "base_case": { - wantErr: blocktree.ErrBlockExists, - args: args{ - header: &types.Header{Number: 2}, - }, - chainSyncBuilder: func(_ *types.Header, ctrl *gomock.Controller) chainSync { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().HasHeader(common.MustHexToHash( - "0x05bdcc454f60a08d427d05e7f19f240fdc391f570ab76fcb96ecca0b5823d3bf")).Return(true, nil) - mockDisjointBlockSet := NewMockDisjointBlockSet(ctrl) - return chainSync{ - blockState: mockBlockState, - pendingBlocks: mockDisjointBlockSet, + "requested_bootstrap_data_but_got_nil_header": { + wantErr: errNilHeaderInResponse, + errString: "expected header, received none: " + + block2Header.Hash().String(), + requestedData: network.BootstrapRequestData, + blockData: &types.BlockData{ + Hash: block2Header.Hash(), + Header: nil, + Body: &types.Body{}, + Justification: &[]byte{0}, + }, + setupChainSync: func(t *testing.T) *chainSync { + ctrl := gomock.NewController(t) + blockStateMock := NewMockBlockState(ctrl) + blockStateMock.EXPECT().HasHeader(block1Header.ParentHash).Return(true, nil) + + networkMock := NewMockNetwork(ctrl) + networkMock.EXPECT().ReportPeer(peerset.ReputationChange{ + Value: peerset.IncompleteHeaderValue, + Reason: peerset.IncompleteHeaderReason, + }, peer.ID("peer")) + + return &chainSync{ + blockState: blockStateMock, + network: networkMock, } }, }, - "err_when_calling_has_header": { - wantErr: errors.New("checking header exists"), - args: args{ - header: &types.Header{Number: 2}, - }, - chainSyncBuilder: func(_ *types.Header, ctrl *gomock.Controller) chainSync { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT(). - HasHeader(common.MustHexToHash( - "0x05bdcc454f60a08d427d05e7f19f240fdc391f570ab76fcb96ecca0b5823d3bf")). - Return(false, errors.New("checking header exists")) - mockDisjointBlockSet := NewMockDisjointBlockSet(ctrl) - return chainSync{ - blockState: mockBlockState, - pendingBlocks: mockDisjointBlockSet, + "requested_bootstrap_data_but_got_nil_body": { + wantErr: errNilBodyInResponse, + errString: "expected body, received none: " + + block2Header.Hash().String(), + requestedData: network.BootstrapRequestData, + blockData: &types.BlockData{ + Hash: block2Header.Hash(), + Header: block2Header, + Body: nil, + Justification: &[]byte{0}, + }, + setupChainSync: func(t *testing.T) *chainSync { + ctrl := gomock.NewController(t) + blockStateMock := NewMockBlockState(ctrl) + blockStateMock.EXPECT().HasHeader(block1Header.ParentHash).Return(true, nil) + networkMock := NewMockNetwork(ctrl) + + return &chainSync{ + blockState: blockStateMock, + network: networkMock, } }, }, - "adding_block_header_to_pending_blocks": { - args: args{ - header: &types.Header{Number: 2}, - }, - chainSyncBuilder: func(expectedHeader *types.Header, ctrl *gomock.Controller) chainSync { - argumentHeaderHash := common.MustHexToHash( - "0x05bdcc454f60a08d427d05e7f19f240fdc391f570ab76fcb96ecca0b5823d3bf") - - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT(). - HasHeader(argumentHeaderHash). - Return(false, nil) + "requested_only_justification_but_got_nil": { + wantErr: errNilJustificationInResponse, + errString: "expected justification, received none: " + + block2Header.Hash().String(), + requestedData: network.RequestedDataJustification, + blockData: &types.BlockData{ + Hash: block2Header.Hash(), + Header: block2Header, + Body: nil, + Justification: nil, + }, + setupChainSync: func(t *testing.T) *chainSync { + ctrl := gomock.NewController(t) + blockStateMock := NewMockBlockState(ctrl) + blockStateMock.EXPECT().HasHeader(block1Header.ParentHash).Return(true, nil) + networkMock := NewMockNetwork(ctrl) - mockBlockState.EXPECT(). - BestBlockHeader(). - Return(&types.Header{Number: 1}, nil) - - mockDisjointBlockSet := NewMockDisjointBlockSet(ctrl) - mockDisjointBlockSet.EXPECT(). - addHeader(expectedHeader). - Return(nil) - - mockDisjointBlockSet.EXPECT(). - addHashAndNumber(argumentHeaderHash, uint(2)). - Return(nil) - - return chainSync{ - blockState: mockBlockState, - pendingBlocks: mockDisjointBlockSet, - peerState: make(map[peer.ID]*peerState), - // creating an buffered channel for this specific test - // since it will put a work on the queue and an unbufered channel - // will hang until we read on this channel and the goal is to - // put the work on the channel and don't block - workQueue: make(chan *peerState, 1), + return &chainSync{ + blockState: blockStateMock, + network: networkMock, } }, }, } - for name, tt := range tests { + + for tname, tt := range cases { tt := tt - t.Run(name, func(t *testing.T) { + t.Run(tname, func(t *testing.T) { t.Parallel() - ctrl := gomock.NewController(t) - sync := tt.chainSyncBuilder(tt.args.header, ctrl) - err := sync.setBlockAnnounce(tt.args.from, tt.args.header) - if tt.wantErr != nil { - assert.EqualError(t, err, tt.wantErr.Error()) - } else { - assert.NoError(t, err) - } - if sync.workQueue != nil { - assert.Equal(t, len(sync.workQueue), 1) + err := validateResponseFields(tt.requestedData, []*types.BlockData{tt.blockData}) + require.ErrorIs(t, err, tt.wantErr) + if tt.errString != "" { + require.EqualError(t, err, tt.errString) } }) } } -func Test_chainSync_getHighestBlock(t *testing.T) { +func TestChainSync_isResponseAChain(t *testing.T) { t.Parallel() - tests := []struct { - name string - peerState map[peer.ID]*peerState - wantHighestBlock uint - expectedError error - }{ - { - name: "error no peers", - expectedError: errors.New("no peers to sync with"), - }, - { - name: "base case", - peerState: map[peer.ID]*peerState{"1": {number: 2}}, - wantHighestBlock: 2, - }, + block1Header := &types.Header{ + ParentHash: common.MustHexToHash("0x00597cb4bb4cc13bf119f6613aec7642d4c06a2e453de53d34aea6f3f1eeb504"), + Number: 2, } - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - cs := &chainSync{ - peerState: tt.peerState, - } - gotHighestBlock, err := cs.getHighestBlock() - if tt.expectedError != nil { - assert.EqualError(t, err, tt.expectedError.Error()) - } else { - assert.NoError(t, err) - } - assert.Equal(t, tt.wantHighestBlock, gotHighestBlock) - }) + + block2Header := &types.Header{ + ParentHash: block1Header.Hash(), + Number: 3, } -} -func Test_chainSync_handleResult(t *testing.T) { - t.Parallel() - mockError := errors.New("test mock error") - tests := map[string]struct { - chainSyncBuilder func(ctrl *gomock.Controller, result *worker) chainSync - maxWorkerRetries uint16 - res *worker - err error + block4Header := &types.Header{ + ParentHash: common.MustHexToHash("0x198616547187613bf119f6613aec7642d4c06a2e453de53d34aea6f390788677"), + Number: 4, + } + + cases := map[string]struct { + expected bool + blockData []*types.BlockData }{ - "res.err_==_nil": { - chainSyncBuilder: func(ctrl *gomock.Controller, result *worker) chainSync { - return chainSync{ - workerState: newWorkerState(), - } - }, - res: &worker{}, - }, - "res.err.err.Error()_==_context.Canceled": { - chainSyncBuilder: func(ctrl *gomock.Controller, result *worker) chainSync { - return chainSync{ - workerState: newWorkerState(), - } - }, - res: &worker{ - ctx: context.Background(), - err: &workerError{ - err: context.Canceled, + "not_a_chain": { + expected: false, + blockData: []*types.BlockData{ + { + Hash: block1Header.Hash(), + Header: block1Header, + Body: &types.Body{}, + Justification: &[]byte{0}, }, - }, - }, - "res.err.err.Error()_==_context.DeadlineExceeded": { - chainSyncBuilder: func(ctrl *gomock.Controller, result *worker) chainSync { - mockNetwork := NewMockNetwork(ctrl) - mockNetwork.EXPECT().ReportPeer(peerset.ReputationChange{Value: -1024, Reason: "Request timeout"}, - peer.ID("")) - mockWorkHandler := NewMockworkHandler(ctrl) - mockWorkHandler.EXPECT().handleWorkerResult(result).Return(result, nil) - return chainSync{ - workerState: newWorkerState(), - network: mockNetwork, - handler: mockWorkHandler, - } - }, - res: &worker{ - ctx: context.Background(), - err: &workerError{ - err: context.DeadlineExceeded, + { + Hash: block2Header.Hash(), + Header: block2Header, + Body: &types.Body{}, + Justification: &[]byte{0}, }, - }, - }, - "res.err.err.Error()_dial_backoff": { - chainSyncBuilder: func(ctrl *gomock.Controller, result *worker) chainSync { - return chainSync{ - workerState: newWorkerState(), - } - }, - res: &worker{ - ctx: context.Background(), - err: &workerError{ - err: errors.New("dial backoff"), + { + Hash: block4Header.Hash(), + Header: block4Header, + Body: &types.Body{}, + Justification: &[]byte{0}, }, }, }, - "res.err.err.Error()_==_errNoPeers": { - chainSyncBuilder: func(ctrl *gomock.Controller, result *worker) chainSync { - return chainSync{ - workerState: newWorkerState(), - } - }, - res: &worker{ - ctx: context.Background(), - err: &workerError{ - err: errNoPeers, + "is_a_chain": { + expected: true, + blockData: []*types.BlockData{ + { + Hash: block1Header.Hash(), + Header: block1Header, + Body: &types.Body{}, + Justification: &[]byte{0}, }, - }, - }, - "res.err.err.Error()_==_protocol_not_supported": { - chainSyncBuilder: func(ctrl *gomock.Controller, result *worker) chainSync { - mockNetwork := NewMockNetwork(ctrl) - mockNetwork.EXPECT().ReportPeer(peerset.ReputationChange{Value: -2147483648, - Reason: "Unsupported protocol"}, - peer.ID("")) - return chainSync{ - workerState: newWorkerState(), - network: mockNetwork, - } - }, - res: &worker{ - ctx: context.Background(), - err: &workerError{ - err: errors.New("protocol not supported"), + { + Hash: block2Header.Hash(), + Header: block2Header, + Body: &types.Body{}, + Justification: &[]byte{0}, }, }, }, - "no_error,_no_retries": { - chainSyncBuilder: func(ctrl *gomock.Controller, result *worker) chainSync { - mockWorkHandler := NewMockworkHandler(ctrl) - mockWorkHandler.EXPECT().handleWorkerResult(result).Return(result, nil) - return chainSync{ - workerState: newWorkerState(), - handler: mockWorkHandler, - } + } + + for tname, tt := range cases { + tt := tt + t.Run(tname, func(t *testing.T) { + t.Parallel() + output := isResponseAChain(tt.blockData) + require.Equal(t, tt.expected, output) + }) + } +} + +func TestChainSync_doResponseGrowsTheChain(t *testing.T) { + block1Header := types.NewHeader(common.Hash{}, common.Hash{}, common.Hash{}, 1, types.NewDigest()) + block2Header := types.NewHeader(block1Header.Hash(), common.Hash{}, common.Hash{}, 2, types.NewDigest()) + block3Header := types.NewHeader(block2Header.Hash(), common.Hash{}, common.Hash{}, 3, types.NewDigest()) + block4Header := types.NewHeader(block3Header.Hash(), common.Hash{}, common.Hash{}, 4, types.NewDigest()) + + testcases := map[string]struct { + response []*types.BlockData + ongoingChain []*types.BlockData + startAt uint + exepectedTotal uint32 + expectedOut bool + }{ + // the ongoing chain does not have any data so the response + // can be inserted in the ongoing chain without any problems + "empty_ongoing_chain": { + ongoingChain: []*types.BlockData{}, + expectedOut: true, + }, + + "one_in_response_growing_ongoing_chain_without_check": { + startAt: 1, + exepectedTotal: 3, + // the ongoing chain contains 3 positions, the block number 1 is at position 0 + ongoingChain: []*types.BlockData{ + {Header: types.NewHeader(common.Hash{}, common.Hash{}, common.Hash{}, 1, types.NewDigest())}, + nil, + nil, }, - res: &worker{ - ctx: context.Background(), - err: &workerError{ - err: errors.New(""), - }, + + // the response contains the block number 3 which should be placed in position 2 + // in the ongoing chain, which means that no comparison should be done to place + // block number 3 in the ongoing chain + response: []*types.BlockData{ + {Header: types.NewHeader(common.Hash{}, common.Hash{}, common.Hash{}, 3, types.NewDigest())}, }, + expectedOut: true, }, - "handle_work_result_error,_no_retries": { - chainSyncBuilder: func(ctrl *gomock.Controller, result *worker) chainSync { - mockWorkHandler := NewMockworkHandler(ctrl) - mockWorkHandler.EXPECT().handleWorkerResult(result).Return(nil, mockError) - return chainSync{ - workerState: newWorkerState(), - handler: mockWorkHandler, - } + + "one_in_response_growing_ongoing_chain_by_checking_neighbours": { + startAt: 1, + exepectedTotal: 3, + // the ongoing chain contains 3 positions, the block number 1 is at position 0 + ongoingChain: []*types.BlockData{ + {Header: block1Header}, + nil, + {Header: block3Header}, }, - res: &worker{ - ctx: context.Background(), - err: &workerError{ - err: errors.New(""), - }, + + // the response contains the block number 2 which should be placed in position 1 + // in the ongoing chain, which means that a comparison should be made to check + // if the parent hash of block 2 is the same hash of block 1 + response: []*types.BlockData{ + {Header: block2Header}, }, - err: mockError, + expectedOut: true, }, - "handle_work_result_nil,_no_retries": { - chainSyncBuilder: func(ctrl *gomock.Controller, result *worker) chainSync { - mockWorkHandler := NewMockworkHandler(ctrl) - mockWorkHandler.EXPECT().handleWorkerResult(result).Return(nil, nil) - return chainSync{ - workerState: newWorkerState(), - handler: mockWorkHandler, - } + + "one_in_response_failed_to_grow_ongoing_chain": { + startAt: 1, + exepectedTotal: 3, + ongoingChain: []*types.BlockData{ + {Header: block1Header}, + nil, + nil, }, - res: &worker{ - ctx: context.Background(), - err: &workerError{ - err: errors.New(""), - }, + response: []*types.BlockData{ + {Header: types.NewHeader(common.Hash{}, common.Hash{}, common.Hash{}, 2, types.NewDigest())}, }, + expectedOut: false, }, - "no_error,_maxWorkerRetries_2": { - chainSyncBuilder: func(ctrl *gomock.Controller, result *worker) chainSync { - mockWorkHandler := NewMockworkHandler(ctrl) - mockWorkHandler.EXPECT().handleWorkerResult(result).Return(result, nil) - mockDisjointBlockSet := NewMockDisjointBlockSet(ctrl) - mockDisjointBlockSet.EXPECT().removeBlock(common.Hash{}) - return chainSync{ - workerState: newWorkerState(), - handler: mockWorkHandler, - pendingBlocks: mockDisjointBlockSet, - } + + "many_in_response_grow_ongoing_chain_only_left_check": { + startAt: 1, + exepectedTotal: 3, + ongoingChain: []*types.BlockData{ + {Header: block1Header}, + nil, + nil, + nil, }, - maxWorkerRetries: 2, - res: &worker{ - ctx: context.Background(), - err: &workerError{ - err: errors.New(""), - }, - pendingBlock: newPendingBlock(common.Hash{}, 1, nil, nil, time.Now()), + response: []*types.BlockData{ + {Header: block2Header}, + {Header: block3Header}, }, + expectedOut: true, }, - "no_error": { - chainSyncBuilder: func(ctrl *gomock.Controller, result *worker) chainSync { - mockWorkHandler := NewMockworkHandler(ctrl) - mockWorkHandler.EXPECT().handleWorkerResult(result).Return(result, nil) - mockWorkHandler.EXPECT().hasCurrentWorker(&worker{ - ctx: context.Background(), - err: &workerError{ - err: mockError, - }, - retryCount: 1, - peersTried: map[peer.ID]struct{}{ - "": {}, - }, - }, newWorkerState().workers).Return(true) - return chainSync{ - workerState: newWorkerState(), - handler: mockWorkHandler, - maxWorkerRetries: 2, - } + + "many_in_response_grow_ongoing_chain_left_right_check": { + startAt: 1, + exepectedTotal: 3, + ongoingChain: []*types.BlockData{ + {Header: block1Header}, + nil, + nil, + {Header: block4Header}, }, - res: &worker{ - ctx: context.Background(), - err: &workerError{ - err: mockError, - }, + response: []*types.BlockData{ + {Header: block2Header}, + {Header: block3Header}, }, + expectedOut: true, }, } - for testName, tt := range tests { + + for tname, tt := range testcases { tt := tt - t.Run(testName, func(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - sync := tt.chainSyncBuilder(ctrl, tt.res) - err := sync.handleResult(tt.res) - if tt.err != nil { - assert.EqualError(t, err, tt.err.Error()) - } else { - assert.NoError(t, err) - } + + t.Run(tname, func(t *testing.T) { + out := doResponseGrowsTheChain(tt.response, tt.ongoingChain, tt.startAt, tt.exepectedTotal) + require.Equal(t, tt.expectedOut, out) }) } } -func newTestChainSyncWithReadyBlocks(ctrl *gomock.Controller, readyBlocks *blockQueue) *chainSync { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().GetFinalisedNotifierChannel().Return(make(chan *types.FinalisationInfo)) +func TestChainSync_getHighestBlock(t *testing.T) { + t.Parallel() - cfg := chainSyncConfig{ - bs: mockBlockState, - readyBlocks: readyBlocks, - pendingBlocks: newDisjointBlockSet(pendingBlocksLimit), - minPeers: 1, - maxPeers: 5, - slotDuration: defaultSlotDuration, + cases := map[string]struct { + expectedHighestBlock uint + wantErr error + chainSyncPeerView map[peer.ID]peerView + }{ + "no_peer_view": { + wantErr: errNoPeers, + expectedHighestBlock: 0, + chainSyncPeerView: make(map[peer.ID]peerView), + }, + "highest_block": { + expectedHighestBlock: 500, + chainSyncPeerView: map[peer.ID]peerView{ + peer.ID("peer-A"): { + number: 100, + }, + peer.ID("peer-B"): { + number: 500, + }, + }, + }, } - mockReqRes := NewMockRequestMaker(ctrl) - return newChainSync(cfg, mockReqRes) -} + for tname, tt := range cases { + tt := tt + t.Run(tname, func(t *testing.T) { + t.Parallel() -func newTestChainSync(ctrl *gomock.Controller) *chainSync { - readyBlocks := newBlockQueue(maxResponseSize) - return newTestChainSyncWithReadyBlocks(ctrl, readyBlocks) + chainSync := &chainSync{ + peerView: tt.chainSyncPeerView, + } + + highestBlock, err := chainSync.getHighestBlock() + require.ErrorIs(t, err, tt.wantErr) + require.Equal(t, tt.expectedHighestBlock, highestBlock) + }) + } } diff --git a/dot/sync/disjoint_block_set.go b/dot/sync/disjoint_block_set.go index 69c5462c16..95b9f7407b 100644 --- a/dot/sync/disjoint_block_set.go +++ b/dot/sync/disjoint_block_set.go @@ -10,6 +10,7 @@ import ( "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" + "golang.org/x/exp/maps" ) const ( @@ -26,7 +27,7 @@ var ( // DisjointBlockSet represents a set of incomplete blocks, or blocks // with an unknown parent. it is implemented by *disjointBlockSet type DisjointBlockSet interface { - run(done <-chan struct{}) + run(finalisedCh <-chan *types.FinalisationInfo, stop <-chan struct{}, wg *sync.WaitGroup) addHashAndNumber(hash common.Hash, number uint) error addHeader(*types.Header) error addBlock(*types.Block) error @@ -35,7 +36,7 @@ type DisjointBlockSet interface { removeLowerBlocks(num uint) getBlock(common.Hash) *pendingBlock getBlocks() []*pendingBlock - getReadyDescendants(curr common.Hash, ready []*types.BlockData) []*types.BlockData + hasBlock(common.Hash) bool size() int } @@ -113,15 +114,20 @@ func newDisjointBlockSet(limit int) *disjointBlockSet { } } -func (s *disjointBlockSet) run(done <-chan struct{}) { +func (s *disjointBlockSet) run(finalisedCh <-chan *types.FinalisationInfo, stop <-chan struct{}, wg *sync.WaitGroup) { ticker := time.NewTicker(clearBlocksInterval) - defer ticker.Stop() + defer func() { + ticker.Stop() + wg.Done() + }() for { select { case <-ticker.C: s.clearBlocks() - case <-done: + case finalisedInfo := <-finalisedCh: + s.removeLowerBlocks(finalisedInfo.Header.Number) + case <-stop: return } } @@ -270,12 +276,6 @@ func (s *disjointBlockSet) size() int { return len(s.blocks) } -func (s *disjointBlockSet) getChildren(hash common.Hash) map[common.Hash]struct{} { - s.RLock() - defer s.RUnlock() - return s.parentToChildren[hash] -} - func (s *disjointBlockSet) getBlock(hash common.Hash) *pendingBlock { s.RLock() defer s.RUnlock() @@ -286,32 +286,5 @@ func (s *disjointBlockSet) getBlocks() []*pendingBlock { s.RLock() defer s.RUnlock() - blocks := make([]*pendingBlock, len(s.blocks)) - i := 0 - for _, b := range s.blocks { - blocks[i] = b - i++ - } - return blocks -} - -// getReadyDescendants recursively checks for descendants that are ready to be processed -func (s *disjointBlockSet) getReadyDescendants(curr common.Hash, ready []*types.BlockData) []*types.BlockData { - children := s.getChildren(curr) - if len(children) == 0 { - return ready - } - - for c := range children { - b := s.getBlock(c) - if b == nil || b.header == nil || b.body == nil { - continue - } - - // if the entire block's data is known, it's ready! - ready = append(ready, b.toBlockData()) - ready = s.getReadyDescendants(c, ready) - } - - return ready + return maps.Values(s.blocks) } diff --git a/dot/sync/disjoint_block_set_integration_test.go b/dot/sync/disjoint_block_set_integration_test.go index 2497b8f290..ec6745ba56 100644 --- a/dot/sync/disjoint_block_set_integration_test.go +++ b/dot/sync/disjoint_block_set_integration_test.go @@ -113,102 +113,6 @@ func TestPendingBlock_toBlockData(t *testing.T) { require.Equal(t, expected, pb.toBlockData()) } -func TestDisjointBlockSet_getReadyDescendants(t *testing.T) { - s := newDisjointBlockSet(pendingBlocksLimit) - - // test that descendant chain gets returned by getReadyDescendants on block 1 being ready - header1 := &types.Header{ - Number: 1, - } - block1 := &types.Block{ - Header: *header1, - Body: types.Body{}, - } - - header2 := &types.Header{ - ParentHash: header1.Hash(), - Number: 2, - } - block2 := &types.Block{ - Header: *header2, - Body: types.Body{}, - } - s.addBlock(block2) - - header3 := &types.Header{ - ParentHash: header2.Hash(), - Number: 3, - } - block3 := &types.Block{ - Header: *header3, - Body: types.Body{}, - } - s.addBlock(block3) - - header2NotDescendant := &types.Header{ - ParentHash: common.Hash{0xff}, - Number: 2, - } - block2NotDescendant := &types.Block{ - Header: *header2NotDescendant, - Body: types.Body{}, - } - s.addBlock(block2NotDescendant) - - ready := []*types.BlockData{block1.ToBlockData()} - ready = s.getReadyDescendants(header1.Hash(), ready) - require.Equal(t, 3, len(ready)) - require.Equal(t, block1.ToBlockData(), ready[0]) - require.Equal(t, block2.ToBlockData(), ready[1]) - require.Equal(t, block3.ToBlockData(), ready[2]) -} - -func TestDisjointBlockSet_getReadyDescendants_blockNotComplete(t *testing.T) { - s := newDisjointBlockSet(pendingBlocksLimit) - - // test that descendant chain gets returned by getReadyDescendants on block 1 being ready - // the ready list should contain only block 1 and 2, as block 3 is incomplete (body is missing) - header1 := &types.Header{ - Number: 1, - } - block1 := &types.Block{ - Header: *header1, - Body: types.Body{}, - } - - header2 := &types.Header{ - ParentHash: header1.Hash(), - Number: 2, - } - block2 := &types.Block{ - Header: *header2, - Body: types.Body{}, - } - s.addBlock(block2) - - header3 := &types.Header{ - ParentHash: header2.Hash(), - Number: 3, - } - s.addHeader(header3) - - header2NotDescendant := &types.Header{ - ParentHash: common.Hash{0xff}, - Number: 2, - } - block2NotDescendant := &types.Block{ - Header: *header2NotDescendant, - Body: types.Body{}, - } - s.addBlock(block2NotDescendant) - - ready := []*types.BlockData{block1.ToBlockData()} - ready = s.getReadyDescendants(header1.Hash(), ready) - require.Equal(t, 2, len(ready)) - require.Equal(t, block1.ToBlockData(), ready[0]) - require.Equal(t, block2.ToBlockData(), ready[1]) -} - func TestDisjointBlockSet_ClearBlocks(t *testing.T) { s := newDisjointBlockSet(pendingBlocksLimit) diff --git a/dot/sync/errors.go b/dot/sync/errors.go index fa4b4ebcad..564c878422 100644 --- a/dot/sync/errors.go +++ b/dot/sync/errors.go @@ -20,18 +20,15 @@ var ( errRequestStartTooHigh = errors.New("request start number is higher than our best block") // chainSync errors - errEmptyBlockData = errors.New("empty block data") - errNilBlockData = errors.New("block data is nil") - errNilHeaderInResponse = errors.New("expected header, received none") - errNilBodyInResponse = errors.New("expected body, received none") - errNoPeers = errors.New("no peers to sync with") - errResponseIsNotChain = errors.New("block response does not form a chain") - errPeerOnInvalidFork = errors.New("peer is on an invalid fork") - errInvalidDirection = errors.New("direction of request does not match specified start and target") - errUnknownParent = errors.New("parent of first block in block response is unknown") - errUnknownBlockForJustification = errors.New("received justification for unknown block") - errFailedToGetParent = errors.New("failed to get parent header") - errStartAndEndMismatch = errors.New("request start and end hash are not on the same chain") - errFailedToGetDescendant = errors.New("failed to find descendant block") - errBadBlock = errors.New("known bad block") + errNoPeerViews = errors.New("unable to get target") + errNilBlockData = errors.New("block data is nil") + errNilHeaderInResponse = errors.New("expected header, received none") + errNilBodyInResponse = errors.New("expected body, received none") + errNilJustificationInResponse = errors.New("expected justification, received none") + errNoPeers = errors.New("no peers to sync with") + errPeerOnInvalidFork = errors.New("peer is on an invalid fork") + errFailedToGetParent = errors.New("failed to get parent header") + errStartAndEndMismatch = errors.New("request start and end hash are not on the same chain") + errFailedToGetDescendant = errors.New("failed to find descendant block") + errAlreadyInDisjointSet = errors.New("already in disjoint set") ) diff --git a/dot/sync/interfaces.go b/dot/sync/interfaces.go index db38d06e3e..806eedb659 100644 --- a/dot/sync/interfaces.go +++ b/dot/sync/interfaces.go @@ -75,6 +75,8 @@ type Network interface { // ReportPeer reports peer based on the peer behaviour. ReportPeer(change peerset.ReputationChange, p peer.ID) + + AllConnectedPeersIDs() []peer.ID } // Telemetry is the telemetry client to send telemetry messages. diff --git a/dot/sync/message.go b/dot/sync/message.go index 4c1ad50470..b5c5d49363 100644 --- a/dot/sync/message.go +++ b/dot/sync/message.go @@ -11,11 +11,6 @@ import ( "github.com/ChainSafe/gossamer/lib/common" ) -const ( - // maxResponseSize is maximum number of block data a BlockResponse message can contain - maxResponseSize = 128 -) - // CreateBlockResponse creates a block response message from a block request message func (s *Service) CreateBlockResponse(req *network.BlockRequestMessage) (*network.BlockResponseMessage, error) { switch req.Direction { @@ -30,13 +25,13 @@ func (s *Service) CreateBlockResponse(req *network.BlockRequestMessage) (*networ func (s *Service) handleAscendingRequest(req *network.BlockRequestMessage) (*network.BlockResponseMessage, error) { var ( - max uint = maxResponseSize + max uint = network.MaxBlocksInResponse startHash *common.Hash startNumber uint ) // determine maximum response size - if req.Max != nil && *req.Max < maxResponseSize { + if req.Max != nil && *req.Max < network.MaxBlocksInResponse { max = uint(*req.Max) } @@ -107,11 +102,11 @@ func (s *Service) handleDescendingRequest(req *network.BlockRequestMessage) (*ne var ( startHash *common.Hash startNumber uint - max uint = maxResponseSize + max uint = network.MaxBlocksInResponse ) // determine maximum response size - if req.Max != nil && *req.Max < maxResponseSize { + if req.Max != nil && *req.Max < network.MaxBlocksInResponse { max = uint(*req.Max) } diff --git a/dot/sync/message_integration_test.go b/dot/sync/message_integration_test.go index a030f1593a..7b4ff69529 100644 --- a/dot/sync/message_integration_test.go +++ b/dot/sync/message_integration_test.go @@ -48,7 +48,7 @@ func addTestBlocksToState(t *testing.T, depth uint, blockState BlockState) { func TestService_CreateBlockResponse_MaxSize(t *testing.T) { s := newTestSyncer(t) - addTestBlocksToState(t, maxResponseSize*2, s.blockState) + addTestBlocksToState(t, network.MaxBlocksInResponse*2, s.blockState) // test ascending start, err := variadic.NewUint32OrHash(1) @@ -63,11 +63,11 @@ func TestService_CreateBlockResponse_MaxSize(t *testing.T) { resp, err := s.CreateBlockResponse(req) require.NoError(t, err) - require.Equal(t, int(maxResponseSize), len(resp.BlockData)) + require.Equal(t, int(network.MaxBlocksInResponse), len(resp.BlockData)) require.Equal(t, uint(1), resp.BlockData[0].Number()) require.Equal(t, uint(128), resp.BlockData[127].Number()) - max := uint32(maxResponseSize + 100) + max := uint32(network.MaxBlocksInResponse + 100) req = &network.BlockRequestMessage{ RequestedData: 3, StartingBlock: *start, @@ -77,7 +77,7 @@ func TestService_CreateBlockResponse_MaxSize(t *testing.T) { resp, err = s.CreateBlockResponse(req) require.NoError(t, err) - require.Equal(t, int(maxResponseSize), len(resp.BlockData)) + require.Equal(t, int(network.MaxBlocksInResponse), len(resp.BlockData)) require.Equal(t, uint(1), resp.BlockData[0].Number()) require.Equal(t, uint(128), resp.BlockData[127].Number()) @@ -108,11 +108,11 @@ func TestService_CreateBlockResponse_MaxSize(t *testing.T) { resp, err = s.CreateBlockResponse(req) require.NoError(t, err) - require.Equal(t, int(maxResponseSize), len(resp.BlockData)) + require.Equal(t, int(network.MaxBlocksInResponse), len(resp.BlockData)) require.Equal(t, uint(128), resp.BlockData[0].Number()) require.Equal(t, uint(1), resp.BlockData[127].Number()) - max = uint32(maxResponseSize + 100) + max = uint32(network.MaxBlocksInResponse + 100) start, err = variadic.NewUint32OrHash(uint32(256)) require.NoError(t, err) @@ -125,7 +125,7 @@ func TestService_CreateBlockResponse_MaxSize(t *testing.T) { resp, err = s.CreateBlockResponse(req) require.NoError(t, err) - require.Equal(t, int(maxResponseSize), len(resp.BlockData)) + require.Equal(t, int(network.MaxBlocksInResponse), len(resp.BlockData)) require.Equal(t, uint(256), resp.BlockData[0].Number()) require.Equal(t, uint(129), resp.BlockData[127].Number()) @@ -146,7 +146,7 @@ func TestService_CreateBlockResponse_MaxSize(t *testing.T) { func TestService_CreateBlockResponse_StartHash(t *testing.T) { s := newTestSyncer(t) - addTestBlocksToState(t, uint(maxResponseSize*2), s.blockState) + addTestBlocksToState(t, uint(network.MaxBlocksInResponse*2), s.blockState) // test ascending with nil endBlockHash startHash, err := s.blockState.GetHashByNumber(1) @@ -164,7 +164,7 @@ func TestService_CreateBlockResponse_StartHash(t *testing.T) { resp, err := s.CreateBlockResponse(req) require.NoError(t, err) - require.Equal(t, int(maxResponseSize), len(resp.BlockData)) + require.Equal(t, int(network.MaxBlocksInResponse), len(resp.BlockData)) require.Equal(t, uint(1), resp.BlockData[0].Number()) require.Equal(t, uint(128), resp.BlockData[127].Number()) @@ -201,7 +201,7 @@ func TestService_CreateBlockResponse_StartHash(t *testing.T) { require.Equal(t, uint(16), resp.BlockData[0].Number()) require.Equal(t, uint(1), resp.BlockData[15].Number()) - // test descending with nil endBlockHash and start > maxResponseSize + // test descending with nil endBlockHash and start > network.MaxBlocksInResponse startHash, err = s.blockState.GetHashByNumber(256) require.NoError(t, err) @@ -217,7 +217,7 @@ func TestService_CreateBlockResponse_StartHash(t *testing.T) { resp, err = s.CreateBlockResponse(req) require.NoError(t, err) - require.Equal(t, int(maxResponseSize), len(resp.BlockData)) + require.Equal(t, int(network.MaxBlocksInResponse), len(resp.BlockData)) require.Equal(t, uint(256), resp.BlockData[0].Number()) require.Equal(t, uint(129), resp.BlockData[127].Number()) @@ -236,7 +236,7 @@ func TestService_CreateBlockResponse_StartHash(t *testing.T) { resp, err = s.CreateBlockResponse(req) require.NoError(t, err) - require.Equal(t, maxResponseSize, len(resp.BlockData)) + require.Equal(t, network.MaxBlocksInResponse, len(resp.BlockData)) require.Equal(t, uint(128), resp.BlockData[0].Number()) require.Equal(t, uint(1), resp.BlockData[127].Number()) } diff --git a/dot/sync/mock_chain_processor_test.go b/dot/sync/mock_chain_processor_test.go deleted file mode 100644 index fc6b9c1569..0000000000 --- a/dot/sync/mock_chain_processor_test.go +++ /dev/null @@ -1,58 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/ChainSafe/gossamer/dot/sync (interfaces: ChainProcessor) - -// Package sync is a generated GoMock package. -package sync - -import ( - reflect "reflect" - - gomock "github.com/golang/mock/gomock" -) - -// MockChainProcessor is a mock of ChainProcessor interface. -type MockChainProcessor struct { - ctrl *gomock.Controller - recorder *MockChainProcessorMockRecorder -} - -// MockChainProcessorMockRecorder is the mock recorder for MockChainProcessor. -type MockChainProcessorMockRecorder struct { - mock *MockChainProcessor -} - -// NewMockChainProcessor creates a new mock instance. -func NewMockChainProcessor(ctrl *gomock.Controller) *MockChainProcessor { - mock := &MockChainProcessor{ctrl: ctrl} - mock.recorder = &MockChainProcessorMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockChainProcessor) EXPECT() *MockChainProcessorMockRecorder { - return m.recorder -} - -// processReadyBlocks mocks base method. -func (m *MockChainProcessor) processReadyBlocks() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "processReadyBlocks") -} - -// processReadyBlocks indicates an expected call of processReadyBlocks. -func (mr *MockChainProcessorMockRecorder) processReadyBlocks() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "processReadyBlocks", reflect.TypeOf((*MockChainProcessor)(nil).processReadyBlocks)) -} - -// stop mocks base method. -func (m *MockChainProcessor) stop() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "stop") -} - -// stop indicates an expected call of stop. -func (mr *MockChainProcessorMockRecorder) stop() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "stop", reflect.TypeOf((*MockChainProcessor)(nil).stop)) -} diff --git a/dot/sync/mock_chain_sync_test.go b/dot/sync/mock_chain_sync_test.go index f89250c252..2d38b8e60e 100644 --- a/dot/sync/mock_chain_sync_test.go +++ b/dot/sync/mock_chain_sync_test.go @@ -7,94 +7,11 @@ package sync import ( reflect "reflect" - types "github.com/ChainSafe/gossamer/dot/types" common "github.com/ChainSafe/gossamer/lib/common" gomock "github.com/golang/mock/gomock" peer "github.com/libp2p/go-libp2p/core/peer" ) -// MockworkHandler is a mock of workHandler interface. -type MockworkHandler struct { - ctrl *gomock.Controller - recorder *MockworkHandlerMockRecorder -} - -// MockworkHandlerMockRecorder is the mock recorder for MockworkHandler. -type MockworkHandlerMockRecorder struct { - mock *MockworkHandler -} - -// NewMockworkHandler creates a new mock instance. -func NewMockworkHandler(ctrl *gomock.Controller) *MockworkHandler { - mock := &MockworkHandler{ctrl: ctrl} - mock.recorder = &MockworkHandlerMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockworkHandler) EXPECT() *MockworkHandlerMockRecorder { - return m.recorder -} - -// handleNewPeerState mocks base method. -func (m *MockworkHandler) handleNewPeerState(arg0 *peerState) (*worker, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "handleNewPeerState", arg0) - ret0, _ := ret[0].(*worker) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// handleNewPeerState indicates an expected call of handleNewPeerState. -func (mr *MockworkHandlerMockRecorder) handleNewPeerState(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "handleNewPeerState", reflect.TypeOf((*MockworkHandler)(nil).handleNewPeerState), arg0) -} - -// handleTick mocks base method. -func (m *MockworkHandler) handleTick() ([]*worker, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "handleTick") - ret0, _ := ret[0].([]*worker) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// handleTick indicates an expected call of handleTick. -func (mr *MockworkHandlerMockRecorder) handleTick() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "handleTick", reflect.TypeOf((*MockworkHandler)(nil).handleTick)) -} - -// handleWorkerResult mocks base method. -func (m *MockworkHandler) handleWorkerResult(w *worker) (*worker, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "handleWorkerResult", w) - ret0, _ := ret[0].(*worker) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// handleWorkerResult indicates an expected call of handleWorkerResult. -func (mr *MockworkHandlerMockRecorder) handleWorkerResult(w interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "handleWorkerResult", reflect.TypeOf((*MockworkHandler)(nil).handleWorkerResult), w) -} - -// hasCurrentWorker mocks base method. -func (m *MockworkHandler) hasCurrentWorker(arg0 *worker, arg1 map[uint64]*worker) bool { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "hasCurrentWorker", arg0, arg1) - ret0, _ := ret[0].(bool) - return ret0 -} - -// hasCurrentWorker indicates an expected call of hasCurrentWorker. -func (mr *MockworkHandlerMockRecorder) hasCurrentWorker(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "hasCurrentWorker", reflect.TypeOf((*MockworkHandler)(nil).hasCurrentWorker), arg0, arg1) -} - // MockChainSync is a mock of ChainSync interface. type MockChainSync struct { ctrl *gomock.Controller @@ -133,32 +50,46 @@ func (mr *MockChainSyncMockRecorder) getHighestBlock() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "getHighestBlock", reflect.TypeOf((*MockChainSync)(nil).getHighestBlock)) } -// setBlockAnnounce mocks base method. -func (m *MockChainSync) setBlockAnnounce(from peer.ID, header *types.Header) error { +// getSyncMode mocks base method. +func (m *MockChainSync) getSyncMode() chainSyncState { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "getSyncMode") + ret0, _ := ret[0].(chainSyncState) + return ret0 +} + +// getSyncMode indicates an expected call of getSyncMode. +func (mr *MockChainSyncMockRecorder) getSyncMode() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "getSyncMode", reflect.TypeOf((*MockChainSync)(nil).getSyncMode)) +} + +// onBlockAnnounce mocks base method. +func (m *MockChainSync) onBlockAnnounce(arg0 announcedBlock) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "setBlockAnnounce", from, header) + ret := m.ctrl.Call(m, "onBlockAnnounce", arg0) ret0, _ := ret[0].(error) return ret0 } -// setBlockAnnounce indicates an expected call of setBlockAnnounce. -func (mr *MockChainSyncMockRecorder) setBlockAnnounce(from, header interface{}) *gomock.Call { +// onBlockAnnounce indicates an expected call of onBlockAnnounce. +func (mr *MockChainSyncMockRecorder) onBlockAnnounce(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "setBlockAnnounce", reflect.TypeOf((*MockChainSync)(nil).setBlockAnnounce), from, header) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "onBlockAnnounce", reflect.TypeOf((*MockChainSync)(nil).onBlockAnnounce), arg0) } -// setPeerHead mocks base method. -func (m *MockChainSync) setPeerHead(p peer.ID, hash common.Hash, number uint) error { +// onBlockAnnounceHandshake mocks base method. +func (m *MockChainSync) onBlockAnnounceHandshake(p peer.ID, hash common.Hash, number uint) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "setPeerHead", p, hash, number) + ret := m.ctrl.Call(m, "onBlockAnnounceHandshake", p, hash, number) ret0, _ := ret[0].(error) return ret0 } -// setPeerHead indicates an expected call of setPeerHead. -func (mr *MockChainSyncMockRecorder) setPeerHead(p, hash, number interface{}) *gomock.Call { +// onBlockAnnounceHandshake indicates an expected call of onBlockAnnounceHandshake. +func (mr *MockChainSyncMockRecorder) onBlockAnnounceHandshake(p, hash, number interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "setPeerHead", reflect.TypeOf((*MockChainSync)(nil).setPeerHead), p, hash, number) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "onBlockAnnounceHandshake", reflect.TypeOf((*MockChainSync)(nil).onBlockAnnounceHandshake), p, hash, number) } // start mocks base method. @@ -174,9 +105,11 @@ func (mr *MockChainSyncMockRecorder) start() *gomock.Call { } // stop mocks base method. -func (m *MockChainSync) stop() { +func (m *MockChainSync) stop() error { m.ctrl.T.Helper() - m.ctrl.Call(m, "stop") + ret := m.ctrl.Call(m, "stop") + ret0, _ := ret[0].(error) + return ret0 } // stop indicates an expected call of stop. @@ -184,17 +117,3 @@ func (mr *MockChainSyncMockRecorder) stop() *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "stop", reflect.TypeOf((*MockChainSync)(nil).stop)) } - -// syncState mocks base method. -func (m *MockChainSync) syncState() chainSyncState { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "syncState") - ret0, _ := ret[0].(chainSyncState) - return ret0 -} - -// syncState indicates an expected call of syncState. -func (mr *MockChainSyncMockRecorder) syncState() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "syncState", reflect.TypeOf((*MockChainSync)(nil).syncState)) -} diff --git a/dot/sync/mock_disjoint_block_set_test.go b/dot/sync/mock_disjoint_block_set_test.go index 07b5578dd9..98c93f577a 100644 --- a/dot/sync/mock_disjoint_block_set_test.go +++ b/dot/sync/mock_disjoint_block_set_test.go @@ -6,6 +6,7 @@ package sync import ( reflect "reflect" + sync0 "sync" types "github.com/ChainSafe/gossamer/dot/types" common "github.com/ChainSafe/gossamer/lib/common" @@ -119,18 +120,18 @@ func (mr *MockDisjointBlockSetMockRecorder) getBlocks() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "getBlocks", reflect.TypeOf((*MockDisjointBlockSet)(nil).getBlocks)) } -// getReadyDescendants mocks base method. -func (m *MockDisjointBlockSet) getReadyDescendants(arg0 common.Hash, arg1 []*types.BlockData) []*types.BlockData { +// hasBlock mocks base method. +func (m *MockDisjointBlockSet) hasBlock(arg0 common.Hash) bool { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "getReadyDescendants", arg0, arg1) - ret0, _ := ret[0].([]*types.BlockData) + ret := m.ctrl.Call(m, "hasBlock", arg0) + ret0, _ := ret[0].(bool) return ret0 } -// getReadyDescendants indicates an expected call of getReadyDescendants. -func (mr *MockDisjointBlockSetMockRecorder) getReadyDescendants(arg0, arg1 interface{}) *gomock.Call { +// hasBlock indicates an expected call of hasBlock. +func (mr *MockDisjointBlockSetMockRecorder) hasBlock(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "getReadyDescendants", reflect.TypeOf((*MockDisjointBlockSet)(nil).getReadyDescendants), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "hasBlock", reflect.TypeOf((*MockDisjointBlockSet)(nil).hasBlock), arg0) } // removeBlock mocks base method. @@ -158,15 +159,15 @@ func (mr *MockDisjointBlockSetMockRecorder) removeLowerBlocks(arg0 interface{}) } // run mocks base method. -func (m *MockDisjointBlockSet) run(arg0 <-chan struct{}) { +func (m *MockDisjointBlockSet) run(arg0 <-chan *types.FinalisationInfo, arg1 <-chan struct{}, arg2 *sync0.WaitGroup) { m.ctrl.T.Helper() - m.ctrl.Call(m, "run", arg0) + m.ctrl.Call(m, "run", arg0, arg1, arg2) } // run indicates an expected call of run. -func (mr *MockDisjointBlockSetMockRecorder) run(arg0 interface{}) *gomock.Call { +func (mr *MockDisjointBlockSetMockRecorder) run(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "run", reflect.TypeOf((*MockDisjointBlockSet)(nil).run), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "run", reflect.TypeOf((*MockDisjointBlockSet)(nil).run), arg0, arg1, arg2) } // size mocks base method. diff --git a/dot/sync/mock_req_res.go b/dot/sync/mock_request.go similarity index 100% rename from dot/sync/mock_req_res.go rename to dot/sync/mock_request.go diff --git a/dot/sync/mocks_generate_test.go b/dot/sync/mocks_generate_test.go index 7d4e8cb064..e970742556 100644 --- a/dot/sync/mocks_generate_test.go +++ b/dot/sync/mocks_generate_test.go @@ -6,7 +6,6 @@ package sync //go:generate mockgen -destination=mocks_test.go -package=$GOPACKAGE . BlockState,StorageState,TransactionState,BabeVerifier,FinalityGadget,BlockImportHandler,Network //go:generate mockgen -destination=mock_telemetry_test.go -package $GOPACKAGE . Telemetry //go:generate mockgen -destination=mock_runtime_test.go -package $GOPACKAGE github.com/ChainSafe/gossamer/lib/runtime Instance -//go:generate mockgen -destination=mock_req_res.go -package $GOPACKAGE github.com/ChainSafe/gossamer/dot/network RequestMaker -//go:generate mockgen -destination=mock_chain_processor_test.go -package=$GOPACKAGE . ChainProcessor -//go:generate mockgen -destination=mock_chain_sync_test.go -package $GOPACKAGE -source chain_sync.go . ChainSync,workHandler +//go:generate mockgen -destination=mock_chain_sync_test.go -package $GOPACKAGE -source chain_sync.go . ChainSync //go:generate mockgen -destination=mock_disjoint_block_set_test.go -package=$GOPACKAGE . DisjointBlockSet +//go:generate mockgen -destination=mock_request.go -package $GOPACKAGE github.com/ChainSafe/gossamer/dot/network RequestMaker diff --git a/dot/sync/mocks_test.go b/dot/sync/mocks_test.go index 57a85eb954..53350f11bc 100644 --- a/dot/sync/mocks_test.go +++ b/dot/sync/mocks_test.go @@ -608,6 +608,20 @@ func (m *MockNetwork) EXPECT() *MockNetworkMockRecorder { return m.recorder } +// AllConnectedPeersIDs mocks base method. +func (m *MockNetwork) AllConnectedPeersIDs() []peer.ID { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AllConnectedPeersIDs") + ret0, _ := ret[0].([]peer.ID) + return ret0 +} + +// AllConnectedPeersIDs indicates an expected call of AllConnectedPeersIDs. +func (mr *MockNetworkMockRecorder) AllConnectedPeersIDs() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllConnectedPeersIDs", reflect.TypeOf((*MockNetwork)(nil).AllConnectedPeersIDs)) +} + // Peers mocks base method. func (m *MockNetwork) Peers() []common.PeerInfo { m.ctrl.T.Helper() diff --git a/dot/sync/syncer.go b/dot/sync/syncer.go index 8cb53ef044..5413b8a002 100644 --- a/dot/sync/syncer.go +++ b/dot/sync/syncer.go @@ -4,9 +4,13 @@ package sync import ( + "errors" + "fmt" "time" + "github.com/ChainSafe/chaindb" "github.com/ChainSafe/gossamer/dot/network" + "github.com/ChainSafe/gossamer/dot/peerset" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/internal/log" @@ -17,10 +21,9 @@ var logger = log.NewFromGlobal(log.AddContext("pkg", "sync")) // Service deals with chain syncing by sending block request messages and watching for responses. type Service struct { - blockState BlockState - chainSync ChainSync - chainProcessor ChainProcessor - network Network + blockState BlockState + chainSync ChainSync + network Network } // Config is the configuration for the sync Service. @@ -37,31 +40,22 @@ type Config struct { SlotDuration time.Duration Telemetry Telemetry BadBlocks []string + RequestMaker network.RequestMaker } // NewService returns a new *sync.Service -func NewService(cfg *Config, blockReqRes network.RequestMaker) (*Service, error) { +func NewService(cfg *Config) (*Service, error) { logger.Patch(log.SetLevel(cfg.LogLvl)) - readyBlocks := newBlockQueue(maxResponseSize * 30) pendingBlocks := newDisjointBlockSet(pendingBlocksLimit) csCfg := chainSyncConfig{ - bs: cfg.BlockState, - net: cfg.Network, - readyBlocks: readyBlocks, - pendingBlocks: pendingBlocks, - minPeers: cfg.MinPeers, - maxPeers: cfg.MaxPeers, - slotDuration: cfg.SlotDuration, - } - chainSync := newChainSync(csCfg, blockReqRes) - - cpCfg := chainProcessorConfig{ - readyBlocks: readyBlocks, + bs: cfg.BlockState, + net: cfg.Network, pendingBlocks: pendingBlocks, - syncer: chainSync, - blockState: cfg.BlockState, + minPeers: cfg.MinPeers, + maxPeers: cfg.MaxPeers, + slotDuration: cfg.SlotDuration, storageState: cfg.StorageState, transactionState: cfg.TransactionState, babeVerifier: cfg.BabeVerifier, @@ -69,47 +63,107 @@ func NewService(cfg *Config, blockReqRes network.RequestMaker) (*Service, error) blockImportHandler: cfg.BlockImportHandler, telemetry: cfg.Telemetry, badBlocks: cfg.BadBlocks, + requestMaker: cfg.RequestMaker, } - chainProcessor := newChainProcessor(cpCfg) + chainSync := newChainSync(csCfg) return &Service{ - blockState: cfg.BlockState, - chainSync: chainSync, - chainProcessor: chainProcessor, - network: cfg.Network, + blockState: cfg.BlockState, + chainSync: chainSync, + network: cfg.Network, }, nil } // Start begins the chainSync and chainProcessor modules. It begins syncing in bootstrap mode func (s *Service) Start() error { go s.chainSync.start() - go s.chainProcessor.processReadyBlocks() return nil } // Stop stops the chainSync and chainProcessor modules func (s *Service) Stop() error { - s.chainSync.stop() - s.chainProcessor.stop() - return nil + return s.chainSync.stop() } // HandleBlockAnnounceHandshake notifies the `chainSync` module that // we have received a BlockAnnounceHandshake from the given peer. func (s *Service) HandleBlockAnnounceHandshake(from peer.ID, msg *network.BlockAnnounceHandshake) error { - return s.chainSync.setPeerHead(from, msg.BestBlockHash, uint(msg.BestBlockNumber)) + return s.chainSync.onBlockAnnounceHandshake(from, msg.BestBlockHash, uint(msg.BestBlockNumber)) } // HandleBlockAnnounce notifies the `chainSync` module that we have received a block announcement from the given peer. func (s *Service) HandleBlockAnnounce(from peer.ID, msg *network.BlockAnnounceMessage) error { logger.Debug("received BlockAnnounceMessage") - header := types.NewHeader(msg.ParentHash, msg.StateRoot, msg.ExtrinsicsRoot, msg.Number, msg.Digest) - return s.chainSync.setBlockAnnounce(from, header) + blockAnnounceHeader := types.NewHeader(msg.ParentHash, msg.StateRoot, msg.ExtrinsicsRoot, msg.Number, msg.Digest) + blockAnnounceHeaderHash := blockAnnounceHeader.Hash() + + // if the peer reports a lower or equal best block number than us, + // check if they are on a fork or not + bestBlockHeader, err := s.blockState.BestBlockHeader() + if err != nil { + return fmt.Errorf("best block header: %w", err) + } + + if blockAnnounceHeader.Number <= bestBlockHeader.Number { + // check if our block hash for that number is the same, if so, do nothing + // as we already have that block + ourHash, err := s.blockState.GetHashByNumber(blockAnnounceHeader.Number) + if err != nil && !errors.Is(err, chaindb.ErrKeyNotFound) { + return fmt.Errorf("get block hash by number: %w", err) + } + + if ourHash == blockAnnounceHeaderHash { + return nil + } + + // check if their best block is on an invalid chain, if it is, + // potentially downscore them + // for now, we can remove them from the syncing peers set + fin, err := s.blockState.GetHighestFinalisedHeader() + if err != nil { + return fmt.Errorf("get highest finalised header: %w", err) + } + + // their block hash doesn't match ours for that number (ie. they are on a different + // chain), and also the highest finalised block is higher than that number. + // thus the peer is on an invalid chain + if fin.Number >= blockAnnounceHeader.Number && msg.BestBlock { + // TODO: downscore this peer, or temporarily don't sync from them? (#1399) + // perhaps we need another field in `peerState` to mark whether the state is valid or not + s.network.ReportPeer(peerset.ReputationChange{ + Value: peerset.BadBlockAnnouncementValue, + Reason: peerset.BadBlockAnnouncementReason, + }, from) + return fmt.Errorf("%w: for peer %s and block number %d", + errPeerOnInvalidFork, from, blockAnnounceHeader.Number) + } + + // peer is on a fork, check if we have processed the fork already or not + // ie. is their block written to our db? + has, err := s.blockState.HasHeader(blockAnnounceHeaderHash) + if err != nil { + return fmt.Errorf("while checking if header exists: %w", err) + } + + // if so, do nothing, as we already have their fork + if has { + return nil + } + } + + // we assume that if a peer sends us a block announce for a certain block, + // that is also has the chain up until and including that block. + // this may not be a valid assumption, but perhaps we can assume that + // it is likely they will receive this block and its ancestors before us. + return s.chainSync.onBlockAnnounce(announcedBlock{ + who: from, + header: blockAnnounceHeader, + }) } // IsSynced exposes the synced state func (s *Service) IsSynced() bool { - return s.chainSync.syncState() == tip + return s.chainSync.getSyncMode() == tip } // HighestBlock gets the highest known block number diff --git a/dot/sync/syncer_integration_test.go b/dot/sync/syncer_integration_test.go index 4ad929948d..92d4f12970 100644 --- a/dot/sync/syncer_integration_test.go +++ b/dot/sync/syncer_integration_test.go @@ -117,8 +117,8 @@ func newTestSyncer(t *testing.T) *Service { cfg.FinalityGadget = mockFinalityGadget cfg.Network = NewMockNetwork(ctrl) cfg.Telemetry = mockTelemetryClient - mockReqRes := NewMockRequestMaker(ctrl) - syncer, err := NewService(cfg, mockReqRes) + cfg.RequestMaker = NewMockRequestMaker(ctrl) + syncer, err := NewService(cfg) require.NoError(t, err) return syncer } diff --git a/dot/sync/syncer_test.go b/dot/sync/syncer_test.go index 750d8886e1..64221216f5 100644 --- a/dot/sync/syncer_test.go +++ b/dot/sync/syncer_test.go @@ -9,12 +9,14 @@ import ( "testing" "github.com/ChainSafe/gossamer/dot/network" + "github.com/ChainSafe/gossamer/dot/peerset" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/pkg/scale" "github.com/golang/mock/gomock" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestNewService(t *testing.T) { @@ -46,9 +48,8 @@ func TestNewService(t *testing.T) { ctrl := gomock.NewController(t) config := tt.cfgBuilder(ctrl) - mockReqRes := NewMockRequestMaker(ctrl) - got, err := NewService(config, mockReqRes) + got, err := NewService(config) if tt.err != nil { assert.EqualError(t, err, tt.err.Error()) } else { @@ -64,133 +65,238 @@ func TestNewService(t *testing.T) { func TestService_HandleBlockAnnounce(t *testing.T) { t.Parallel() - ctrl := gomock.NewController(t) + errTest := errors.New("test error") + const somePeer = peer.ID("abc") - type fields struct { - chainSync ChainSync - } - type args struct { - from peer.ID - msg *network.BlockAnnounceMessage - } - tests := []struct { - name string - fields fields - args args - wantErr bool + block1AnnounceHeader := types.NewHeader(common.Hash{}, common.Hash{}, + common.Hash{}, 1, scale.VaryingDataTypeSlice{}) + block2AnnounceHeader := types.NewHeader(common.Hash{}, common.Hash{}, + common.Hash{}, 2, scale.VaryingDataTypeSlice{}) + + testCases := map[string]struct { + serviceBuilder func(ctrl *gomock.Controller) *Service + peerID peer.ID + blockAnnounceHeader *types.Header + errWrapped error + errMessage string }{ - { - name: "working_example", - fields: fields{ - chainSync: newMockChainSync(ctrl), - }, - args: args{ - from: peer.ID("1"), - msg: &network.BlockAnnounceMessage{ - ParentHash: common.Hash{}, - Number: 1, - StateRoot: common.Hash{}, - ExtrinsicsRoot: common.Hash{}, - Digest: scale.VaryingDataTypeSlice{}, - BestBlock: false, - }, + "best_block_header_error": { + serviceBuilder: func(ctrl *gomock.Controller) *Service { + blockState := NewMockBlockState(ctrl) + blockState.EXPECT().BestBlockHeader().Return(nil, errTest) + return &Service{ + blockState: blockState, + } }, - wantErr: false, + peerID: somePeer, + blockAnnounceHeader: block1AnnounceHeader, + errWrapped: errTest, + errMessage: "best block header: test error", }, - } - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - s := &Service{ - chainSync: tt.fields.chainSync, - } - if err := s.HandleBlockAnnounce(tt.args.from, tt.args.msg); (err != nil) != tt.wantErr { - t.Errorf("HandleBlockAnnounce() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func newMockChainSync(ctrl *gomock.Controller) ChainSync { - mock := NewMockChainSync(ctrl) - header := types.NewHeader(common.Hash{}, common.Hash{}, common.Hash{}, 1, - scale.VaryingDataTypeSlice{}) - - mock.EXPECT().setBlockAnnounce(peer.ID("1"), header).Return(nil).AnyTimes() - mock.EXPECT().setPeerHead(peer.ID("1"), common.Hash{}, uint(0)).Return(nil).AnyTimes() - mock.EXPECT().syncState().Return(bootstrap).AnyTimes() - mock.EXPECT().start().AnyTimes() - mock.EXPECT().stop().AnyTimes() - mock.EXPECT().getHighestBlock().Return(uint(2), nil).AnyTimes() - - return mock -} - -func Test_Service_HandleBlockAnnounceHandshake(t *testing.T) { - t.Parallel() + "number_smaller_than_best_block_number_get_hash_by_number_error": { + serviceBuilder: func(ctrl *gomock.Controller) *Service { + blockState := NewMockBlockState(ctrl) + bestBlockHeader := &types.Header{Number: 2} + blockState.EXPECT().BestBlockHeader().Return(bestBlockHeader, nil) + blockState.EXPECT().GetHashByNumber(uint(1)).Return(common.Hash{}, errTest) - errTest := errors.New("test error") + return &Service{ + blockState: blockState, + } + }, + peerID: somePeer, + blockAnnounceHeader: block1AnnounceHeader, + errWrapped: errTest, + errMessage: "get block hash by number: test error", + }, + "number_smaller_than_best_block_number_and_same_hash": { + serviceBuilder: func(ctrl *gomock.Controller) *Service { + blockState := NewMockBlockState(ctrl) + bestBlockHeader := &types.Header{Number: 2} + blockState.EXPECT().BestBlockHeader().Return(bestBlockHeader, nil) + blockState.EXPECT().GetHashByNumber(uint(1)).Return(block1AnnounceHeader.Hash(), nil) + return &Service{ + blockState: blockState, + } + }, + peerID: somePeer, + blockAnnounceHeader: block1AnnounceHeader, + }, + "number_smaller_than_best_block_number_get_highest_finalised_header_error": { + serviceBuilder: func(ctrl *gomock.Controller) *Service { + blockState := NewMockBlockState(ctrl) + bestBlockHeader := &types.Header{Number: 2} + blockState.EXPECT().BestBlockHeader().Return(bestBlockHeader, nil) + blockState.EXPECT().GetHashByNumber(uint(1)).Return(common.Hash{2}, nil) + blockState.EXPECT().GetHighestFinalisedHeader().Return(nil, errTest) + return &Service{ + blockState: blockState, + } + }, + peerID: somePeer, + blockAnnounceHeader: block1AnnounceHeader, + errWrapped: errTest, + errMessage: "get highest finalised header: test error", + }, + "number_smaller_than_best_block_announced_number_equaks_finalised_number": { + serviceBuilder: func(ctrl *gomock.Controller) *Service { + blockState := NewMockBlockState(ctrl) - testCases := map[string]struct { - serviceBuilder func(ctrl *gomock.Controller) Service - from peer.ID - message *network.BlockAnnounceHandshake - errWrapped error - errMessage string - }{ - "success": { - serviceBuilder: func(ctrl *gomock.Controller) Service { - chainSync := NewMockChainSync(ctrl) - chainSync.EXPECT().setPeerHead(peer.ID("abc"), common.Hash{1}, uint(2)). - Return(nil) - return Service{ - chainSync: chainSync, + bestBlockHeader := &types.Header{Number: 2} + blockState.EXPECT().BestBlockHeader().Return(bestBlockHeader, nil) + blockState.EXPECT().GetHashByNumber(uint(1)). + Return(common.Hash{2}, nil) // other hash than someHash + finalisedBlockHeader := &types.Header{Number: 1} + blockState.EXPECT().GetHighestFinalisedHeader().Return(finalisedBlockHeader, nil) + network := NewMockNetwork(ctrl) + network.EXPECT().ReportPeer(peerset.ReputationChange{ + Value: peerset.BadBlockAnnouncementValue, + Reason: peerset.BadBlockAnnouncementReason, + }, somePeer) + return &Service{ + blockState: blockState, + network: network, } }, - from: peer.ID("abc"), - message: &network.BlockAnnounceHandshake{ - BestBlockHash: common.Hash{1}, - BestBlockNumber: 2, + peerID: somePeer, + blockAnnounceHeader: block1AnnounceHeader, + errWrapped: errPeerOnInvalidFork, + errMessage: "peer is on an invalid fork: for peer ZiCa and block number 1", + }, + "number_smaller_than_best_block_number_and_finalised_number_bigger_than_number": { + serviceBuilder: func(ctrl *gomock.Controller) *Service { + blockState := NewMockBlockState(ctrl) + bestBlockHeader := &types.Header{Number: 2} + blockState.EXPECT().BestBlockHeader().Return(bestBlockHeader, nil) + blockState.EXPECT().GetHashByNumber(uint(1)). + Return(common.Hash{2}, nil) // other hash than someHash + finalisedBlockHeader := &types.Header{Number: 2} + blockState.EXPECT().GetHighestFinalisedHeader().Return(finalisedBlockHeader, nil) + network := NewMockNetwork(ctrl) + network.EXPECT().ReportPeer(peerset.ReputationChange{ + Value: peerset.BadBlockAnnouncementValue, + Reason: peerset.BadBlockAnnouncementReason, + }, somePeer) + return &Service{ + blockState: blockState, + network: network, + } }, + peerID: somePeer, + blockAnnounceHeader: block1AnnounceHeader, + errWrapped: errPeerOnInvalidFork, + errMessage: "peer is on an invalid fork: for peer ZiCa and block number 1", }, - "failure": { - serviceBuilder: func(ctrl *gomock.Controller) Service { - chainSync := NewMockChainSync(ctrl) - chainSync.EXPECT().setPeerHead(peer.ID("abc"), common.Hash{1}, uint(2)). - Return(errTest) - return Service{ - chainSync: chainSync, + "number_smaller_than_best_block_number_and_" + + "finalised_number_smaller_than_number_and_" + + "has_header_error": { + serviceBuilder: func(ctrl *gomock.Controller) *Service { + blockState := NewMockBlockState(ctrl) + bestBlockHeader := &types.Header{Number: 3} + blockState.EXPECT().BestBlockHeader().Return(bestBlockHeader, nil) + blockState.EXPECT().GetHashByNumber(uint(2)). + Return(common.Hash{5, 1, 2}, nil) // other hash than block2AnnounceHeader hash + finalisedBlockHeader := &types.Header{Number: 1} + blockState.EXPECT().GetHighestFinalisedHeader().Return(finalisedBlockHeader, nil) + blockState.EXPECT().HasHeader(block2AnnounceHeader.Hash()).Return(false, errTest) + return &Service{ + blockState: blockState, } }, - from: peer.ID("abc"), - message: &network.BlockAnnounceHandshake{ - BestBlockHash: common.Hash{1}, - BestBlockNumber: 2, + peerID: somePeer, + blockAnnounceHeader: block2AnnounceHeader, + errWrapped: errTest, + errMessage: "while checking if header exists: test error", + }, + "number_smaller_than_best_block_number_and_" + + "finalised_number_smaller_than_number_and_" + + "has_the_hash": { + serviceBuilder: func(ctrl *gomock.Controller) *Service { + blockState := NewMockBlockState(ctrl) + bestBlockHeader := &types.Header{Number: 3} + blockState.EXPECT().BestBlockHeader().Return(bestBlockHeader, nil) + blockState.EXPECT().GetHashByNumber(uint(2)). + Return(common.Hash{2}, nil) // other hash than someHash + finalisedBlockHeader := &types.Header{Number: 1} + blockState.EXPECT().GetHighestFinalisedHeader().Return(finalisedBlockHeader, nil) + blockState.EXPECT().HasHeader(block2AnnounceHeader.Hash()).Return(true, nil) + return &Service{ + blockState: blockState, + } }, - errWrapped: errTest, - errMessage: "test error", + peerID: somePeer, + blockAnnounceHeader: block2AnnounceHeader, + }, + "number_bigger_than_best_block_number_added_in_disjoint_set_with_success": { + serviceBuilder: func(ctrl *gomock.Controller) *Service { + + blockState := NewMockBlockState(ctrl) + bestBlockHeader := &types.Header{Number: 1} + blockState.EXPECT().BestBlockHeader().Return(bestBlockHeader, nil) + chainSyncMock := NewMockChainSync(ctrl) + + expectedAnnouncedBlock := announcedBlock{ + who: somePeer, + header: block2AnnounceHeader, + } + + chainSyncMock.EXPECT().onBlockAnnounce(expectedAnnouncedBlock).Return(nil) + + return &Service{ + blockState: blockState, + chainSync: chainSyncMock, + } + }, + peerID: somePeer, + blockAnnounceHeader: block2AnnounceHeader, }, } - for name, testCase := range testCases { - testCase := testCase + for name, tt := range testCases { + tt := tt t.Run(name, func(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) - service := testCase.serviceBuilder(ctrl) - - err := service.HandleBlockAnnounceHandshake(testCase.from, testCase.message) + service := tt.serviceBuilder(ctrl) - assert.ErrorIs(t, err, testCase.errWrapped) - if testCase.errWrapped != nil { - assert.EqualError(t, err, testCase.errMessage) + blockAnnounceMessage := &network.BlockAnnounceMessage{ + ParentHash: tt.blockAnnounceHeader.ParentHash, + Number: tt.blockAnnounceHeader.Number, + StateRoot: tt.blockAnnounceHeader.StateRoot, + ExtrinsicsRoot: tt.blockAnnounceHeader.ExtrinsicsRoot, + Digest: tt.blockAnnounceHeader.Digest, + BestBlock: true, + } + err := service.HandleBlockAnnounce(tt.peerID, blockAnnounceMessage) + assert.ErrorIs(t, err, tt.errWrapped) + if tt.errWrapped != nil { + assert.EqualError(t, err, tt.errMessage) } }) } } +func Test_Service_HandleBlockAnnounceHandshake(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + chainSync := NewMockChainSync(ctrl) + chainSync.EXPECT().onBlockAnnounceHandshake(peer.ID("peer"), common.Hash{1}, uint(2)) + + service := Service{ + chainSync: chainSync, + } + + message := &network.BlockAnnounceHandshake{ + BestBlockHash: common.Hash{1}, + BestBlockNumber: 2, + } + + err := service.HandleBlockAnnounceHandshake(peer.ID("peer"), message) + require.Nil(t, err) +} + func TestService_IsSynced(t *testing.T) { t.Parallel() @@ -201,7 +307,7 @@ func TestService_IsSynced(t *testing.T) { "tip": { serviceBuilder: func(ctrl *gomock.Controller) Service { chainSync := NewMockChainSync(ctrl) - chainSync.EXPECT().syncState().Return(tip) + chainSync.EXPECT().getSyncMode().Return(tip) return Service{ chainSync: chainSync, } @@ -211,7 +317,7 @@ func TestService_IsSynced(t *testing.T) { "not_tip": { serviceBuilder: func(ctrl *gomock.Controller) Service { chainSync := NewMockChainSync(ctrl) - chainSync.EXPECT().syncState().Return(bootstrap) + chainSync.EXPECT().getSyncMode().Return(bootstrap) return Service{ chainSync: chainSync, } @@ -246,15 +352,8 @@ func TestService_Start(t *testing.T) { allCalled.Done() }) - chainProcessor := NewMockChainProcessor(ctrl) - allCalled.Add(1) - chainProcessor.EXPECT().processReadyBlocks().DoAndReturn(func() { - allCalled.Done() - }) - service := Service{ - chainSync: chainSync, - chainProcessor: chainProcessor, + chainSync: chainSync, } err := service.Start() @@ -268,12 +367,8 @@ func TestService_Stop(t *testing.T) { chainSync := NewMockChainSync(ctrl) chainSync.EXPECT().stop() - chainProcessor := NewMockChainProcessor(ctrl) - chainProcessor.EXPECT().stop() - service := &Service{ - chainSync: chainSync, - chainProcessor: chainProcessor, + chainSync: chainSync, } err := service.Stop() diff --git a/dot/sync/tip_syncer.go b/dot/sync/tip_syncer.go deleted file mode 100644 index 9eeb640c53..0000000000 --- a/dot/sync/tip_syncer.go +++ /dev/null @@ -1,220 +0,0 @@ -// Copyright 2021 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package sync - -import ( - "errors" - - "github.com/ChainSafe/gossamer/dot/network" - "github.com/ChainSafe/gossamer/dot/types" - "github.com/ChainSafe/gossamer/lib/common" -) - -var _ workHandler = &tipSyncer{} - -type handleReadyBlockFunc func(*types.BlockData) - -// tipSyncer handles workers when syncing at the tip of the chain -type tipSyncer struct { - blockState BlockState - pendingBlocks DisjointBlockSet - readyBlocks *blockQueue - handleReadyBlock handleReadyBlockFunc -} - -func newTipSyncer(blockState BlockState, pendingBlocks DisjointBlockSet, readyBlocks *blockQueue, - handleReadyBlock handleReadyBlockFunc) *tipSyncer { - return &tipSyncer{ - blockState: blockState, - pendingBlocks: pendingBlocks, - readyBlocks: readyBlocks, - handleReadyBlock: handleReadyBlock, - } -} - -func (s *tipSyncer) handleNewPeerState(ps *peerState) (*worker, error) { - fin, err := s.blockState.GetHighestFinalisedHeader() - if err != nil { - return nil, err - } - - if ps.number <= fin.Number { - return nil, nil - } - - return &worker{ - startHash: ps.hash, - startNumber: uintPtr(ps.number), - targetHash: ps.hash, - targetNumber: uintPtr(ps.number), - requestData: bootstrapRequestData, - }, nil -} - -func (s *tipSyncer) handleWorkerResult(res *worker) ( - workerToRetry *worker, err error) { - if res.err == nil { - return nil, nil - } - - if errors.Is(res.err.err, errUnknownParent) { - // handleTick will handle the errUnknownParent case - return nil, nil - } - - fin, err := s.blockState.GetHighestFinalisedHeader() - if err != nil { - return nil, err - } - - // don't retry if we're requesting blocks lower than finalised - switch res.direction { - case network.Ascending: - if *res.targetNumber <= fin.Number { - return nil, nil - } - - // if start is lower than finalised, increase it to finalised+1 - if *res.startNumber <= fin.Number { - *res.startNumber = fin.Number + 1 - res.startHash = common.Hash{} - } - case network.Descending: - if *res.startNumber <= fin.Number { - return nil, nil - } - - // if target is lower than finalised, increase it to finalised+1 - if *res.targetNumber <= fin.Number { - *res.targetNumber = fin.Number + 1 - res.targetHash = common.Hash{} - } - } - - return &worker{ - startHash: res.startHash, - startNumber: res.startNumber, - targetHash: res.targetHash, - targetNumber: res.targetNumber, - direction: res.direction, - requestData: res.requestData, - }, nil -} - -func (*tipSyncer) hasCurrentWorker(w *worker, workers map[uint64]*worker) bool { - if w == nil || w.startNumber == nil || w.targetNumber == nil { - return true - } - - for _, curr := range workers { - if w.direction != curr.direction || w.requestData != curr.requestData { - continue - } - - switch w.direction { - case network.Ascending: - if *w.targetNumber > *curr.targetNumber || - *w.startNumber < *curr.startNumber { - continue - } - case network.Descending: - if *w.targetNumber < *curr.targetNumber || - *w.startNumber > *curr.startNumber { - continue - } - } - - // worker (start, end) is within curr (start, end), if hashes are equal then the request is either - // for the same data or some subset of data that is covered by curr - if w.startHash == curr.startHash || w.targetHash == curr.targetHash { - return true - } - } - - return false -} - -// handleTick traverses the pending blocks set to find which forks still need to be requested -func (s *tipSyncer) handleTick() ([]*worker, error) { - logger.Debugf("handling tick, we have %d pending blocks", s.pendingBlocks.size()) - - if s.pendingBlocks.size() == 0 { - return nil, nil - } - - fin, err := s.blockState.GetHighestFinalisedHeader() - if err != nil { - return nil, err - } - - // cases for each block in pending set: - // 1. only hash and number are known; in this case, request the full block (and ancestor chain) - // 2. only header is known; in this case, request the block body - // 3. entire block is known; in this case, check if we have become aware of the parent - // if we have, move it to the ready blocks queue; otherwise, request the chain of ancestors - - var workers []*worker - - for _, block := range s.pendingBlocks.getBlocks() { - if block.number <= fin.Number { - // delete from pending set (this should not happen, it should have already been deleted) - s.pendingBlocks.removeBlock(block.hash) - continue - } - - logger.Tracef("handling pending block number %d with hash %s", block.number, block.hash) - - if block.header == nil { - // case 1 - workers = append(workers, &worker{ - startHash: block.hash, - startNumber: uintPtr(block.number), - targetHash: fin.Hash(), - targetNumber: uintPtr(fin.Number), - direction: network.Descending, - requestData: bootstrapRequestData, - pendingBlock: block, - }) - continue - } - - if block.body == nil { - // case 2 - workers = append(workers, &worker{ - startHash: block.hash, - startNumber: uintPtr(block.number), - targetHash: block.hash, - targetNumber: uintPtr(block.number), - requestData: network.RequestedDataBody + network.RequestedDataJustification, - pendingBlock: block, - }) - continue - } - - // case 3 - has, err := s.blockState.HasHeader(block.header.ParentHash) - if err != nil { - return nil, err - } - - if has || s.readyBlocks.has(block.header.ParentHash) { - // block is ready, as parent is known! - // also, move any pendingBlocks that are descendants of this block to the ready blocks queue - s.handleReadyBlock(block.toBlockData()) - continue - } - - // request descending chain from (parent of pending block) -> (last finalised block) - workers = append(workers, &worker{ - startHash: block.header.ParentHash, - startNumber: uintPtr(block.number - 1), - targetNumber: uintPtr(fin.Number), - direction: network.Descending, - requestData: bootstrapRequestData, - pendingBlock: block, - }) - } - - return workers, nil -} diff --git a/dot/sync/tip_syncer_integration_test.go b/dot/sync/tip_syncer_integration_test.go deleted file mode 100644 index 5e433d73d9..0000000000 --- a/dot/sync/tip_syncer_integration_test.go +++ /dev/null @@ -1,372 +0,0 @@ -//go:build integration - -// Copyright 2021 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package sync - -import ( - "context" - "testing" - "time" - - "github.com/ChainSafe/gossamer/dot/network" - "github.com/ChainSafe/gossamer/dot/types" - "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" - - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func newTestTipSyncer(t *testing.T) *tipSyncer { - finHeader := types.NewHeader(common.NewHash([]byte{0}), - trie.EmptyHash, trie.EmptyHash, 200, types.NewDigest()) - - ctrl := gomock.NewController(t) - bs := NewMockBlockState(ctrl) - bs.EXPECT().GetHighestFinalisedHeader().Return(finHeader, nil).AnyTimes() - bs.EXPECT().HasHeader(gomock.AssignableToTypeOf(common.Hash{})).Return(true, nil).AnyTimes() - - readyBlocks := newBlockQueue(maxResponseSize) - pendingBlocks := newDisjointBlockSet(pendingBlocksLimit) - return newTipSyncer(bs, pendingBlocks, readyBlocks, nil) -} - -func TestTipSyncer_handleNewPeerState(t *testing.T) { - s := newTestTipSyncer(t) - - // peer reports state lower than our highest finalised, we should ignore - ps := &peerState{ - number: 1, - } - - w, err := s.handleNewPeerState(ps) - require.NoError(t, err) - require.Nil(t, w) - - ps = &peerState{ - number: 201, - hash: common.Hash{0xa, 0xb}, - } - - // otherwise, return a worker - expected := &worker{ - startNumber: uintPtr(ps.number), - startHash: ps.hash, - targetNumber: uintPtr(ps.number), - targetHash: ps.hash, - requestData: bootstrapRequestData, - } - - w, err = s.handleNewPeerState(ps) - require.NoError(t, err) - require.Equal(t, expected, w) -} - -func TestTipSyncer_handleWorkerResult(t *testing.T) { - s := newTestTipSyncer(t) - - w, err := s.handleWorkerResult(&worker{}) - require.NoError(t, err) - require.Nil(t, w) - - w, err = s.handleWorkerResult(&worker{ - err: &workerError{ - err: errUnknownParent, - }, - }) - require.NoError(t, err) - require.Nil(t, w) - - // worker is for blocks lower than finalised - w, err = s.handleWorkerResult(&worker{ - targetNumber: uintPtr(199), - }) - require.NoError(t, err) - require.Nil(t, w) - - w, err = s.handleWorkerResult(&worker{ - direction: network.Descending, - startNumber: uintPtr(199), - }) - require.NoError(t, err) - require.Nil(t, w) - - // worker start is lower than finalised, start should be updated - expected := &worker{ - direction: network.Ascending, - startNumber: uintPtr(201), - targetNumber: uintPtr(300), - requestData: bootstrapRequestData, - } - - w, err = s.handleWorkerResult(&worker{ - direction: network.Ascending, - startNumber: uintPtr(199), - targetNumber: uintPtr(300), - requestData: bootstrapRequestData, - err: &workerError{}, - }) - require.NoError(t, err) - require.Equal(t, expected, w) - - expected = &worker{ - direction: network.Descending, - startNumber: uintPtr(300), - targetNumber: uintPtr(201), - requestData: bootstrapRequestData, - } - - w, err = s.handleWorkerResult(&worker{ - direction: network.Descending, - startNumber: uintPtr(300), - targetNumber: uintPtr(199), - requestData: bootstrapRequestData, - err: &workerError{}, - }) - require.NoError(t, err) - require.Equal(t, expected, w) - - // start and target are higher than finalised, don't modify - expected = &worker{ - direction: network.Descending, - startNumber: uintPtr(300), - startHash: common.Hash{0xa, 0xb}, - targetNumber: uintPtr(201), - targetHash: common.Hash{0xc, 0xd}, - requestData: bootstrapRequestData, - } - - w, err = s.handleWorkerResult(&worker{ - direction: network.Descending, - startNumber: uintPtr(300), - startHash: common.Hash{0xa, 0xb}, - targetNumber: uintPtr(201), - targetHash: common.Hash{0xc, 0xd}, - requestData: bootstrapRequestData, - err: &workerError{}, - }) - require.NoError(t, err) - require.Equal(t, expected, w) -} - -func TestTipSyncer_handleTick_case1(t *testing.T) { - s := newTestTipSyncer(t) - - w, err := s.handleTick() - require.NoError(t, err) - require.Nil(t, w) - - fin, _ := s.blockState.GetHighestFinalisedHeader() - - // add pending blocks w/ only hash and number, lower than finalised should be removed - s.pendingBlocks.addHashAndNumber(common.Hash{0xa}, fin.Number) - s.pendingBlocks.addHashAndNumber(common.Hash{0xb}, fin.Number+1) - - expected := []*worker{ - { - startHash: common.Hash{0xb}, - startNumber: uintPtr(fin.Number + 1), - targetHash: fin.Hash(), - targetNumber: uintPtr(fin.Number), - direction: network.Descending, - requestData: bootstrapRequestData, - pendingBlock: &pendingBlock{ - hash: common.Hash{0xb}, - number: 201, - clearAt: time.Unix(0, 0), - }, - }, - } - w, err = s.handleTick() - require.NoError(t, err) - require.NotEmpty(t, w) - assert.Greater(t, w[0].pendingBlock.clearAt, time.Now()) - w[0].pendingBlock.clearAt = time.Unix(0, 0) - require.Equal(t, expected, w) - require.False(t, s.pendingBlocks.(*disjointBlockSet).hasBlock(common.Hash{0xa})) - require.True(t, s.pendingBlocks.(*disjointBlockSet).hasBlock(common.Hash{0xb})) -} - -func TestTipSyncer_handleTick_case2(t *testing.T) { - s := newTestTipSyncer(t) - - fin, _ := s.blockState.GetHighestFinalisedHeader() - - // add pending blocks w/ only header - header := &types.Header{ - Number: fin.Number + 1, - } - s.pendingBlocks.addHeader(header) - - expected := []*worker{ - { - startHash: header.Hash(), - startNumber: uintPtr(header.Number), - targetHash: header.Hash(), - targetNumber: uintPtr(header.Number), - direction: network.Ascending, - requestData: network.RequestedDataBody + network.RequestedDataJustification, - pendingBlock: &pendingBlock{ - hash: header.Hash(), - number: 201, - header: header, - clearAt: time.Time{}, - }, - }, - } - w, err := s.handleTick() - require.NoError(t, err) - require.NotEmpty(t, w) - assert.Greater(t, w[0].pendingBlock.clearAt, time.Now()) - w[0].pendingBlock.clearAt = time.Time{} - require.Equal(t, expected, w) - require.True(t, s.pendingBlocks.(*disjointBlockSet).hasBlock(header.Hash())) -} -func TestTipSyncer_handleTick_case3(t *testing.T) { - s := newTestTipSyncer(t) - s.handleReadyBlock = func(data *types.BlockData) { - s.pendingBlocks.removeBlock(data.Hash) - s.readyBlocks.push(data) - } - fin, _ := s.blockState.GetHighestFinalisedHeader() - - // add pending block w/ full block, HasHeader will return true, so the block will be processed - header := &types.Header{ - Number: fin.Number + 1, - } - block := &types.Block{ - Header: *header, - Body: types.Body{}, - } - s.pendingBlocks.addBlock(block) - - w, err := s.handleTick() - require.NoError(t, err) - require.Equal(t, []*worker(nil), w) - require.False(t, s.pendingBlocks.(*disjointBlockSet).hasBlock(header.Hash())) - readyBlockData, err := s.readyBlocks.pop(context.Background()) - require.Equal(t, block.ToBlockData(), readyBlockData) - require.NoError(t, err) - - // add pending block w/ full block, but block is not ready as parent is unknown - ctrl := gomock.NewController(t) - bs := NewMockBlockState(ctrl) - bs.EXPECT().GetHighestFinalisedHeader().Return(fin, nil).Times(2) - bs.EXPECT().HasHeader(gomock.AssignableToTypeOf(common.Hash{})).Return(false, nil).Times(2) - s.blockState = bs - - header = &types.Header{ - Number: fin.Number + 100, - } - block = &types.Block{ - Header: *header, - Body: types.Body{}, - } - s.pendingBlocks.addBlock(block) - - expected := []*worker{ - { - startHash: header.ParentHash, - startNumber: uintPtr(header.Number - 1), - targetNumber: uintPtr(fin.Number), - direction: network.Descending, - requestData: bootstrapRequestData, - pendingBlock: &pendingBlock{ - hash: header.Hash(), - number: 300, - header: header, - body: &types.Body{}, - clearAt: time.Time{}, - }, - }, - } - - w, err = s.handleTick() - require.NoError(t, err) - require.NotEmpty(t, w) - assert.Greater(t, w[0].pendingBlock.clearAt, time.Now()) - w[0].pendingBlock.clearAt = time.Time{} - require.Equal(t, expected, w) - require.True(t, s.pendingBlocks.(*disjointBlockSet).hasBlock(header.Hash())) - - // add parent block to readyBlocks, should move block to readyBlocks - s.readyBlocks.push(&types.BlockData{ - Hash: header.ParentHash, - }) - w, err = s.handleTick() - require.NoError(t, err) - require.Equal(t, []*worker(nil), w) - require.False(t, s.pendingBlocks.(*disjointBlockSet).hasBlock(header.Hash())) - _, _ = s.readyBlocks.pop(context.Background()) // first pop will remove parent - readyBlockData, err = s.readyBlocks.pop(context.Background()) - require.NoError(t, err) - require.Equal(t, block.ToBlockData(), readyBlockData) -} - -func TestTipSyncer_hasCurrentWorker(t *testing.T) { - s := newTestTipSyncer(t) - require.False(t, s.hasCurrentWorker(&worker{ - startNumber: uintPtr(0), - targetNumber: uintPtr(0), - }, nil)) - - workers := make(map[uint64]*worker) - workers[0] = &worker{ - startNumber: uintPtr(1), - targetNumber: uintPtr(128), - } - require.False(t, s.hasCurrentWorker(&worker{ - startNumber: uintPtr(1), - targetNumber: uintPtr(129), - }, workers)) - require.True(t, s.hasCurrentWorker(&worker{ - startNumber: uintPtr(1), - targetNumber: uintPtr(128), - }, workers)) - require.True(t, s.hasCurrentWorker(&worker{ - startNumber: uintPtr(1), - targetNumber: uintPtr(127), - }, workers)) - - workers[0] = &worker{ - startNumber: uintPtr(128), - targetNumber: uintPtr(255), - } - require.False(t, s.hasCurrentWorker(&worker{ - startNumber: uintPtr(127), - targetNumber: uintPtr(255), - }, workers)) - require.True(t, s.hasCurrentWorker(&worker{ - startNumber: uintPtr(128), - targetNumber: uintPtr(255), - }, workers)) - - workers[0] = &worker{ - startNumber: uintPtr(128), - targetNumber: uintPtr(1), - direction: network.Descending, - } - require.False(t, s.hasCurrentWorker(&worker{ - startNumber: uintPtr(129), - targetNumber: uintPtr(1), - direction: network.Descending, - }, workers)) - require.True(t, s.hasCurrentWorker(&worker{ - startNumber: uintPtr(128), - targetNumber: uintPtr(1), - direction: network.Descending, - }, workers)) - require.True(t, s.hasCurrentWorker(&worker{ - startNumber: uintPtr(128), - targetNumber: uintPtr(2), - direction: network.Descending, - }, workers)) - require.True(t, s.hasCurrentWorker(&worker{ - startNumber: uintPtr(127), - targetNumber: uintPtr(1), - direction: network.Descending, - }, workers)) -} diff --git a/dot/sync/tip_syncer_test.go b/dot/sync/tip_syncer_test.go deleted file mode 100644 index 09ed2b9eb6..0000000000 --- a/dot/sync/tip_syncer_test.go +++ /dev/null @@ -1,401 +0,0 @@ -// Copyright 2021 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package sync - -import ( - "testing" - - "github.com/ChainSafe/gossamer/dot/network" - "github.com/ChainSafe/gossamer/dot/types" - "github.com/ChainSafe/gossamer/lib/common" - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" -) - -func Test_tipSyncer_handleNewPeerState(t *testing.T) { - t.Parallel() - - type fields struct { - blockStateBuilder func(ctrl *gomock.Controller) BlockState - pendingBlocks DisjointBlockSet - readyBlocks *blockQueue - } - tests := map[string]struct { - fields fields - peerState *peerState - want *worker - err error - }{ - "peer_state_number_<_final_block_number": { - fields: fields{ - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().GetHighestFinalisedHeader().Return(&types.Header{ - Number: 2, - }, nil) - return mockBlockState - }, - }, - peerState: &peerState{number: 1}, - want: nil, - }, - "base_state": { - fields: fields{ - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().GetHighestFinalisedHeader().Return(&types.Header{ - Number: 2, - }, nil) - return mockBlockState - }, - }, - peerState: &peerState{number: 3}, - want: &worker{ - startNumber: uintPtr(3), - targetNumber: uintPtr(3), - requestData: bootstrapRequestData, - }, - }, - } - for name, tt := range tests { - tt := tt - t.Run(name, func(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - s := &tipSyncer{ - blockState: tt.fields.blockStateBuilder(ctrl), - pendingBlocks: tt.fields.pendingBlocks, - readyBlocks: tt.fields.readyBlocks, - } - got, err := s.handleNewPeerState(tt.peerState) - if tt.err != nil { - assert.EqualError(t, err, tt.err.Error()) - } else { - assert.NoError(t, err) - } - assert.Equal(t, tt.want, got) - }) - } -} - -func Test_tipSyncer_handleTick(t *testing.T) { - t.Parallel() - - type fields struct { - blockStateBuilder func(ctrl *gomock.Controller) BlockState - pendingBlocksBuilder func(ctrl *gomock.Controller) DisjointBlockSet - readyBlocks *blockQueue - } - tests := map[string]struct { - fields fields - want []*worker - err error - }{ - "base_case": { - fields: fields{ - pendingBlocksBuilder: func(ctrl *gomock.Controller) DisjointBlockSet { - mockDisjointBlockSet := NewMockDisjointBlockSet(ctrl) - mockDisjointBlockSet.EXPECT().size().Return(1).Times(2) - mockDisjointBlockSet.EXPECT().getBlocks().Return([]*pendingBlock{ - {number: 2}, - {number: 3}, - {number: 4, - header: &types.Header{ - Number: 4, - }, - }, - {number: 5, - header: &types.Header{ - Number: 5, - }, - body: &types.Body{}, - }, - }) - mockDisjointBlockSet.EXPECT().removeBlock(common.Hash{}) - return mockDisjointBlockSet - }, - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().GetHighestFinalisedHeader().Return(&types.Header{ - Number: 2, - }, nil) - mockBlockState.EXPECT().HasHeader(common.Hash{}).Return(false, nil) - return mockBlockState - }, - readyBlocks: newBlockQueue(3), - }, - want: []*worker{ - { - startNumber: uintPtr(3), - targetNumber: uintPtr(2), - targetHash: common.Hash{5, 189, 204, 69, 79, 96, 160, 141, 66, 125, 5, 231, 241, - 159, 36, 15, 220, 57, 31, 87, 10, 183, 111, 203, 150, 236, 202, 11, 88, 35, 211, 191}, - pendingBlock: &pendingBlock{number: 3}, - requestData: bootstrapRequestData, - direction: network.Descending, - }, - { - startNumber: uintPtr(4), - targetNumber: uintPtr(4), - pendingBlock: &pendingBlock{ - number: 4, - header: &types.Header{ - Number: 4, - }, - }, - requestData: network.RequestedDataBody + network.RequestedDataJustification, - }, - { - startNumber: uintPtr(4), - targetNumber: uintPtr(2), - direction: network.Descending, - pendingBlock: &pendingBlock{ - number: 5, - header: &types.Header{ - Number: 5, - }, - body: &types.Body{}, - }, - requestData: bootstrapRequestData, - }, - }, - err: nil, - }, - } - for name, tt := range tests { - tt := tt - t.Run(name, func(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - s := &tipSyncer{ - blockState: tt.fields.blockStateBuilder(ctrl), - pendingBlocks: tt.fields.pendingBlocksBuilder(ctrl), - readyBlocks: tt.fields.readyBlocks, - } - got, err := s.handleTick() - if tt.err != nil { - assert.EqualError(t, err, tt.err.Error()) - } else { - assert.NoError(t, err) - } - assert.Equal(t, tt.want, got) - }) - } -} - -func Test_tipSyncer_handleWorkerResult(t *testing.T) { - t.Parallel() - - tests := map[string]struct { - blockStateBuilder func(ctrl *gomock.Controller) BlockState - res *worker - want *worker - err error - }{ - "worker_error_is_nil": { - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - return NewMockBlockState(ctrl) - }, - res: &worker{}, - want: nil, - err: nil, - }, - "worker_error_is_error_unknown_parent": { - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - return NewMockBlockState(ctrl) - }, - res: &worker{ - err: &workerError{ - err: errUnknownParent, - }, - }, - want: nil, - err: nil, - }, - "ascending,_target_number_<_finalised_number": { - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().GetHighestFinalisedHeader().Return(&types.Header{ - Number: 2, - }, nil) - return mockBlockState - }, - res: &worker{ - targetNumber: uintPtr(1), - direction: network.Ascending, - err: &workerError{}, - }, - want: nil, - err: nil, - }, - "ascending,_start_number_<_finalised_number": { - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().GetHighestFinalisedHeader().Return(&types.Header{ - Number: 2, - }, nil) - return mockBlockState - }, - res: &worker{ - startNumber: uintPtr(1), - targetNumber: uintPtr(3), - direction: network.Ascending, - err: &workerError{}, - }, - want: &worker{ - startNumber: uintPtr(3), - targetNumber: uintPtr(3), - }, - err: nil, - }, - "descending,_start_number_<_finalised_number": { - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().GetHighestFinalisedHeader().Return(&types.Header{ - Number: 2, - }, nil) - return mockBlockState - }, - res: &worker{ - startNumber: uintPtr(1), - direction: network.Descending, - err: &workerError{}, - }, - want: nil, - err: nil, - }, - "descending,_target_number_<_finalised_number": { - blockStateBuilder: func(ctrl *gomock.Controller) BlockState { - mockBlockState := NewMockBlockState(ctrl) - mockBlockState.EXPECT().GetHighestFinalisedHeader().Return(&types.Header{ - Number: 2, - }, nil) - return mockBlockState - }, - res: &worker{ - startNumber: uintPtr(3), - targetNumber: uintPtr(1), - direction: network.Descending, - err: &workerError{}, - }, - want: &worker{ - startNumber: uintPtr(3), - targetNumber: uintPtr(3), - direction: network.Descending, - }, - err: nil, - }, - } - for name, tt := range tests { - tt := tt - t.Run(name, func(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - s := &tipSyncer{ - blockState: tt.blockStateBuilder(ctrl), - } - got, err := s.handleWorkerResult(tt.res) - if tt.err != nil { - assert.EqualError(t, err, tt.err.Error()) - } else { - assert.NoError(t, err) - } - assert.Equal(t, tt.want, got) - }) - } -} - -func Test_tipSyncer_hasCurrentWorker(t *testing.T) { - t.Parallel() - - type args struct { - w *worker - workers map[uint64]*worker - } - tests := map[string]struct { - args args - want bool - }{ - "worker_nil": { - want: true, - }, - "ascending,_false": { - args: args{ - w: &worker{ - direction: network.Ascending, - startNumber: uintPtr(2), - targetNumber: uintPtr(2), - }, - workers: map[uint64]*worker{ - 1: { - direction: network.Ascending, - targetNumber: uintPtr(3), - startNumber: uintPtr(3), - }, - }, - }, - want: false, - }, - "ascending,_true": { - args: args{ - w: &worker{ - direction: network.Ascending, - startNumber: uintPtr(2), - targetNumber: uintPtr(2), - }, - workers: map[uint64]*worker{ - 1: { - direction: network.Ascending, - targetNumber: uintPtr(3), - startNumber: uintPtr(1), - }, - }, - }, - want: true, - }, - "descending,_false": { - args: args{ - w: &worker{ - direction: network.Descending, - startNumber: uintPtr(2), - targetNumber: uintPtr(2), - }, - workers: map[uint64]*worker{ - 1: { - startNumber: uintPtr(3), - targetNumber: uintPtr(3), - direction: network.Descending, - }, - }, - }, - want: false, - }, - "descending,_true": { - args: args{ - w: &worker{ - direction: network.Descending, - startNumber: uintPtr(2), - targetNumber: uintPtr(2), - }, - workers: map[uint64]*worker{ - 1: { - startNumber: uintPtr(3), - targetNumber: uintPtr(1), - direction: network.Descending, - }, - }, - }, - want: true, - }, - } - for name, tt := range tests { - tt := tt - t.Run(name, func(t *testing.T) { - t.Parallel() - s := &tipSyncer{} - got := s.hasCurrentWorker(tt.args.w, tt.args.workers) - assert.Equal(t, tt.want, got) - }) - } -} diff --git a/dot/sync/worker.go b/dot/sync/worker.go index c597623089..e4c6252619 100644 --- a/dot/sync/worker.go +++ b/dot/sync/worker.go @@ -1,95 +1,64 @@ -// Copyright 2021 ChainSafe Systems (ON) +// Copyright 2023 ChainSafe Systems (ON) // SPDX-License-Identifier: LGPL-3.0-only package sync import ( - "context" + "errors" "sync" - "time" - - "github.com/libp2p/go-libp2p/core/peer" "github.com/ChainSafe/gossamer/dot/network" - "github.com/ChainSafe/gossamer/lib/common" + "github.com/libp2p/go-libp2p/core/peer" ) -// workerState helps track the current worker set and set the upcoming worker ID -type workerState struct { - ctx context.Context - cancel context.CancelFunc +var ErrStopTimeout = errors.New("stop timeout") - sync.Mutex - nextWorker uint64 - workers map[uint64]*worker +type worker struct { + status byte + peerID peer.ID + sharedGuard chan struct{} + requestMaker network.RequestMaker } -func newWorkerState() *workerState { - ctx, cancel := context.WithCancel(context.Background()) - return &workerState{ - ctx: ctx, - cancel: cancel, - workers: make(map[uint64]*worker), +func newWorker(pID peer.ID, sharedGuard chan struct{}, network network.RequestMaker) *worker { + return &worker{ + peerID: pID, + sharedGuard: sharedGuard, + requestMaker: network, + status: available, } } -func (s *workerState) add(w *worker) { - s.Lock() - defer s.Unlock() - - w.id = s.nextWorker - w.ctx = s.ctx - s.nextWorker++ - s.workers[w.id] = w -} - -func (s *workerState) delete(id uint64) { - s.Lock() - defer s.Unlock() - delete(s.workers, id) -} +func (w *worker) run(queue chan *syncTask, wg *sync.WaitGroup) { + defer func() { + logger.Debugf("[STOPPED] worker %s", w.peerID) + wg.Done() + }() -func (s *workerState) reset() { - s.cancel() - s.ctx, s.cancel = context.WithCancel(context.Background()) - - s.Lock() - defer s.Unlock() - - for id := range s.workers { - delete(s.workers, id) + for task := range queue { + executeRequest(w.peerID, w.requestMaker, task, w.sharedGuard) } - s.nextWorker = 0 } -// worker respresents a process that is attempting to sync from the specified start block to target block -// if it fails for some reason, `err` is set. -// otherwise, we can assume all the blocks have been received and added to the `readyBlocks` queue -type worker struct { - ctx context.Context - id uint64 - retryCount uint16 - peersTried map[peer.ID]struct{} - - startHash common.Hash - startNumber *uint - targetHash common.Hash - targetNumber *uint +func executeRequest(who peer.ID, requestMaker network.RequestMaker, + task *syncTask, sharedGuard chan struct{}) { + defer func() { + <-sharedGuard + }() - // if this worker is tied to a specific pending block, this field is set - pendingBlock *pendingBlock + sharedGuard <- struct{}{} - // bitmap of fields to request - requestData byte - direction network.SyncDirection + request := task.request + logger.Debugf("[EXECUTING] worker %s, block request: %s", who, request) + response := new(network.BlockResponseMessage) + err := requestMaker.Do(who, request, response) - duration time.Duration - err *workerError -} + task.resultCh <- &syncTaskResult{ + who: who, + request: request, + response: response, + err: err, + } -type workerError struct { - err error - who peer.ID // whose response caused the error, if any + logger.Debugf("[FINISHED] worker %s, err: %s, block data amount: %d", who, err, len(response.BlockData)) } - -func uintPtr(n uint) *uint { return &n } diff --git a/dot/sync/worker_pool.go b/dot/sync/worker_pool.go new file mode 100644 index 0000000000..1eb4640bf8 --- /dev/null +++ b/dot/sync/worker_pool.go @@ -0,0 +1,227 @@ +// Copyright 2023 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package sync + +import ( + "fmt" + + "crypto/rand" + "math/big" + "sync" + "time" + + "github.com/ChainSafe/gossamer/dot/network" + "github.com/libp2p/go-libp2p/core/peer" + "golang.org/x/exp/maps" +) + +const ( + available byte = iota + busy + punished +) + +const ( + punishmentBaseTimeout = 5 * time.Minute + maxRequestsAllowed uint = 60 +) + +type syncTask struct { + request *network.BlockRequestMessage + resultCh chan<- *syncTaskResult +} + +type syncTaskResult struct { + who peer.ID + request *network.BlockRequestMessage + response *network.BlockResponseMessage + err error +} + +type syncWorker struct { + worker *worker + queue chan *syncTask +} + +type syncWorkerPool struct { + mtx sync.RWMutex + wg sync.WaitGroup + + network Network + requestMaker network.RequestMaker + workers map[peer.ID]*syncWorker + ignorePeers map[peer.ID]struct{} + + sharedGuard chan struct{} +} + +func newSyncWorkerPool(net Network, requestMaker network.RequestMaker) *syncWorkerPool { + swp := &syncWorkerPool{ + network: net, + requestMaker: requestMaker, + workers: make(map[peer.ID]*syncWorker), + ignorePeers: make(map[peer.ID]struct{}), + sharedGuard: make(chan struct{}, maxRequestsAllowed), + } + + return swp +} + +// stop will shutdown all the available workers goroutines +func (s *syncWorkerPool) stop() error { + s.mtx.RLock() + defer s.mtx.RUnlock() + + for _, sw := range s.workers { + close(sw.queue) + } + + allWorkersDoneCh := make(chan struct{}) + go func() { + defer close(allWorkersDoneCh) + s.wg.Wait() + }() + + timeoutTimer := time.NewTimer(30 * time.Second) + select { + case <-timeoutTimer.C: + return fmt.Errorf("timeout reached while finishing workers") + case <-allWorkersDoneCh: + if !timeoutTimer.Stop() { + <-timeoutTimer.C + } + + return nil + } +} + +// useConnectedPeers will retrieve all connected peers +// through the network layer and use them as sources of blocks +func (s *syncWorkerPool) useConnectedPeers() { + connectedPeers := s.network.AllConnectedPeersIDs() + if len(connectedPeers) < 1 { + return + } + + s.mtx.Lock() + defer s.mtx.Unlock() + for _, connectedPeer := range connectedPeers { + s.newPeer(connectedPeer) + } +} + +func (s *syncWorkerPool) fromBlockAnnounce(who peer.ID) { + s.mtx.Lock() + defer s.mtx.Unlock() + s.newPeer(who) +} + +// newPeer a new peer will be included in the worker +// pool if it is not a peer to ignore or is not punished +func (s *syncWorkerPool) newPeer(who peer.ID) { + if _, ok := s.ignorePeers[who]; ok { + return + } + + _, has := s.workers[who] + if has { + return + } + + worker := newWorker(who, s.sharedGuard, s.requestMaker) + workerQueue := make(chan *syncTask, maxRequestsAllowed) + + s.wg.Add(1) + go worker.run(workerQueue, &s.wg) + + s.workers[who] = &syncWorker{ + worker: worker, + queue: workerQueue, + } + logger.Tracef("potential worker added, total in the pool %d", len(s.workers)) +} + +// submitRequest given a request, the worker pool will get the peer given the peer.ID +// parameter or if nil the very first available worker or +// to perform the request, the response will be dispatch in the resultCh. +func (s *syncWorkerPool) submitRequest(request *network.BlockRequestMessage, + who *peer.ID, resultCh chan<- *syncTaskResult) { + + task := &syncTask{ + request: request, + resultCh: resultCh, + } + + // if the request is bounded to a specific peer then just + // request it and sent through its queue otherwise send + // the request in the general queue where all worker are + // listening on + s.mtx.RLock() + defer s.mtx.RUnlock() + + if who != nil { + syncWorker := s.workers[*who] + syncWorker.queue <- task + return + } + + // if the exact peer is not specified then + // randomly select a worker and assign the + // task to it, if the amount of workers is + var selectedWorkerIdx int + workers := maps.Values(s.workers) + nBig, err := rand.Int(rand.Reader, big.NewInt(int64(len(workers)))) + if err != nil { + panic(fmt.Errorf("fail to get a random number: %w", err)) + } + selectedWorkerIdx = int(nBig.Int64()) + selectedWorker := workers[selectedWorkerIdx] + selectedWorker.queue <- task +} + +// submitRequests takes an set of requests and will submit to the pool through submitRequest +// the response will be dispatch in the resultCh +func (s *syncWorkerPool) submitRequests(requests []*network.BlockRequestMessage) (resultCh chan *syncTaskResult) { + resultCh = make(chan *syncTaskResult, maxRequestsAllowed+1) + + s.mtx.RLock() + defer s.mtx.RUnlock() + + allWorkers := maps.Values(s.workers) + for idx, request := range requests { + workerID := idx % len(allWorkers) + syncWorker := allWorkers[workerID] + + syncWorker.queue <- &syncTask{ + request: request, + resultCh: resultCh, + } + } + + return resultCh +} + +func (s *syncWorkerPool) ignorePeerAsWorker(who peer.ID) { + s.mtx.Lock() + defer s.mtx.Unlock() + + worker, has := s.workers[who] + if has { + close(worker.queue) + delete(s.workers, who) + s.ignorePeers[who] = struct{}{} + } +} + +// totalWorkers only returns available or busy workers +func (s *syncWorkerPool) totalWorkers() (total uint) { + s.mtx.RLock() + defer s.mtx.RUnlock() + + for range s.workers { + total++ + } + + return total +} diff --git a/dot/sync/worker_pool_test.go b/dot/sync/worker_pool_test.go new file mode 100644 index 0000000000..5ea0d4e9d7 --- /dev/null +++ b/dot/sync/worker_pool_test.go @@ -0,0 +1,247 @@ +// Copyright 2023 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package sync + +import ( + "testing" + "time" + + "github.com/ChainSafe/gossamer/dot/network" + "github.com/ChainSafe/gossamer/dot/types" + "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/common/variadic" + "github.com/golang/mock/gomock" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/stretchr/testify/require" + "golang.org/x/exp/maps" +) + +func TestSyncWorkerPool_useConnectedPeers(t *testing.T) { + t.Parallel() + cases := map[string]struct { + setupWorkerPool func(t *testing.T) *syncWorkerPool + exepectedWorkers []peer.ID + }{ + "no_connected_peers": { + setupWorkerPool: func(t *testing.T) *syncWorkerPool { + ctrl := gomock.NewController(t) + networkMock := NewMockNetwork(ctrl) + networkMock.EXPECT(). + AllConnectedPeersIDs(). + Return([]peer.ID{}) + + return newSyncWorkerPool(networkMock, nil) + }, + exepectedWorkers: []peer.ID{}, + }, + "3_available_peers": { + setupWorkerPool: func(t *testing.T) *syncWorkerPool { + ctrl := gomock.NewController(t) + networkMock := NewMockNetwork(ctrl) + networkMock.EXPECT(). + AllConnectedPeersIDs(). + Return([]peer.ID{ + peer.ID("available-1"), + peer.ID("available-2"), + peer.ID("available-3"), + }) + return newSyncWorkerPool(networkMock, nil) + }, + exepectedWorkers: []peer.ID{ + peer.ID("available-1"), + peer.ID("available-2"), + peer.ID("available-3"), + }, + }, + "2_available_peers_1_to_ignore": { + setupWorkerPool: func(t *testing.T) *syncWorkerPool { + ctrl := gomock.NewController(t) + networkMock := NewMockNetwork(ctrl) + networkMock.EXPECT(). + AllConnectedPeersIDs(). + Return([]peer.ID{ + peer.ID("available-1"), + peer.ID("available-2"), + peer.ID("available-3"), + }) + workerPool := newSyncWorkerPool(networkMock, nil) + workerPool.ignorePeers[peer.ID("available-3")] = struct{}{} + return workerPool + }, + exepectedWorkers: []peer.ID{ + peer.ID("available-1"), + peer.ID("available-2"), + }, + }, + "peer_already_in_workers_set": { + setupWorkerPool: func(t *testing.T) *syncWorkerPool { + ctrl := gomock.NewController(t) + networkMock := NewMockNetwork(ctrl) + networkMock.EXPECT(). + AllConnectedPeersIDs(). + Return([]peer.ID{ + peer.ID("available-1"), + peer.ID("available-2"), + peer.ID("available-3"), + }) + workerPool := newSyncWorkerPool(networkMock, nil) + syncWorker := &syncWorker{ + worker: &worker{}, + queue: make(chan *syncTask), + } + workerPool.workers[peer.ID("available-3")] = syncWorker + return workerPool + }, + exepectedWorkers: []peer.ID{ + peer.ID("available-1"), + peer.ID("available-2"), + peer.ID("available-3"), + }, + }, + } + + for tname, tt := range cases { + tt := tt + t.Run(tname, func(t *testing.T) { + t.Parallel() + + workerPool := tt.setupWorkerPool(t) + workerPool.useConnectedPeers() + defer workerPool.stop() + + require.ElementsMatch(t, + maps.Keys(workerPool.workers), + tt.exepectedWorkers) + }) + } +} + +func TestSyncWorkerPool_listenForRequests_submitRequest(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + networkMock := NewMockNetwork(ctrl) + requestMakerMock := NewMockRequestMaker(ctrl) + workerPool := newSyncWorkerPool(networkMock, requestMakerMock) + + availablePeer := peer.ID("available-peer") + workerPool.newPeer(availablePeer) + defer workerPool.stop() + + blockHash := common.MustHexToHash("0x750646b852a29e5f3668959916a03d6243a3137e91d0cd36870364931030f707") + blockRequest := network.NewBlockRequest(*variadic.MustNewUint32OrHash(blockHash), + 1, network.BootstrapRequestData, network.Descending) + mockedBlockResponse := &network.BlockResponseMessage{ + BlockData: []*types.BlockData{ + { + Hash: blockHash, + Header: &types.Header{ + ParentHash: common. + MustHexToHash("0x5895897f12e1a670609929433ac7a69dcae90e0cc2d9c32c0dce0e2a5e5e614e"), + }, + }, + }, + } + + // introduce a timeout of 5s then we can test the + // peer status change to busy + requestMakerMock.EXPECT(). + Do(availablePeer, blockRequest, &network.BlockResponseMessage{}). + DoAndReturn(func(_, _, response any) any { + responsePtr := response.(*network.BlockResponseMessage) + *responsePtr = *mockedBlockResponse + return nil + }) + + resultCh := make(chan *syncTaskResult) + workerPool.submitRequest(blockRequest, nil, resultCh) + + syncTaskResult := <-resultCh + require.NoError(t, syncTaskResult.err) + require.Equal(t, syncTaskResult.who, availablePeer) + require.Equal(t, syncTaskResult.request, blockRequest) + require.Equal(t, syncTaskResult.response, mockedBlockResponse) + +} + +func TestSyncWorkerPool_singleWorker_multipleRequests(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + networkMock := NewMockNetwork(ctrl) + requestMakerMock := NewMockRequestMaker(ctrl) + workerPool := newSyncWorkerPool(networkMock, requestMakerMock) + defer workerPool.stop() + + availablePeer := peer.ID("available-peer") + workerPool.newPeer(availablePeer) + + firstRequestBlockHash := common.MustHexToHash("0x750646b852a29e5f3668959916a03d6243a3137e91d0cd36870364931030f707") + firstBlockRequest := network.NewBlockRequest(*variadic.MustNewUint32OrHash(firstRequestBlockHash), + 1, network.BootstrapRequestData, network.Descending) + + secondRequestBlockHash := common.MustHexToHash("0x897646b852a29e5f3668959916a03d6243a3137e91d0cd36870364931030f707") + secondBlockRequest := network.NewBlockRequest(*variadic.MustNewUint32OrHash(firstRequestBlockHash), + 1, network.BootstrapRequestData, network.Descending) + + firstMockedBlockResponse := &network.BlockResponseMessage{ + BlockData: []*types.BlockData{ + { + Hash: firstRequestBlockHash, + Header: &types.Header{ + ParentHash: common. + MustHexToHash("0x5895897f12e1a670609929433ac7a69dcae90e0cc2d9c32c0dce0e2a5e5e614e"), + }, + }, + }, + } + + secondMockedBlockResponse := &network.BlockResponseMessage{ + BlockData: []*types.BlockData{ + { + Hash: secondRequestBlockHash, + Header: &types.Header{ + ParentHash: common. + MustHexToHash("0x8965897f12e1a670609929433ac7a69dcae90e0cc2d9c32c0dce0e2a5e5e614e"), + }, + }, + }, + } + + // introduce a timeout of 5s then we can test the + // then we can simulate a busy peer + requestMakerMock.EXPECT(). + Do(availablePeer, firstBlockRequest, &network.BlockResponseMessage{}). + DoAndReturn(func(_, _, response any) any { + time.Sleep(5 * time.Second) + responsePtr := response.(*network.BlockResponseMessage) + *responsePtr = *firstMockedBlockResponse + return nil + }) + + requestMakerMock.EXPECT(). + Do(availablePeer, firstBlockRequest, &network.BlockResponseMessage{}). + DoAndReturn(func(_, _, response any) any { + responsePtr := response.(*network.BlockResponseMessage) + *responsePtr = *secondMockedBlockResponse + return nil + }) + + resultCh := workerPool.submitRequests( + []*network.BlockRequestMessage{firstBlockRequest, secondBlockRequest}) + + syncTaskResult := <-resultCh + require.NoError(t, syncTaskResult.err) + require.Equal(t, syncTaskResult.who, availablePeer) + require.Equal(t, syncTaskResult.request, firstBlockRequest) + require.Equal(t, syncTaskResult.response, firstMockedBlockResponse) + + syncTaskResult = <-resultCh + require.NoError(t, syncTaskResult.err) + require.Equal(t, syncTaskResult.who, availablePeer) + require.Equal(t, syncTaskResult.request, secondBlockRequest) + require.Equal(t, syncTaskResult.response, secondMockedBlockResponse) + + require.Equal(t, uint(1), workerPool.totalWorkers()) +} diff --git a/dot/sync/worker_test.go b/dot/sync/worker_test.go new file mode 100644 index 0000000000..904be38983 --- /dev/null +++ b/dot/sync/worker_test.go @@ -0,0 +1,61 @@ +// Copyright 2023 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package sync + +import ( + "sync" + "testing" + "time" + + "github.com/ChainSafe/gossamer/dot/network" + "github.com/golang/mock/gomock" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/stretchr/testify/require" +) + +func TestWorker(t *testing.T) { + peerA := peer.ID("peerA") + ctrl := gomock.NewController(t) + + reqMaker := NewMockRequestMaker(ctrl) + reqMaker.EXPECT(). + Do(peerA, nil, gomock.AssignableToTypeOf((*network.BlockResponseMessage)(nil))). + DoAndReturn(func(_, _, _ any) any { + time.Sleep(2 * time.Second) + return nil + }). + Times(2). + Return(nil) + + sharedGuard := make(chan struct{}, 1) + w := newWorker(peerA, sharedGuard, reqMaker) + + wg := sync.WaitGroup{} + queue := make(chan *syncTask, 2) + + wg.Add(1) + go w.run(queue, &wg) + + resultCh := make(chan *syncTaskResult) + defer close(resultCh) + + queue <- &syncTask{ + resultCh: resultCh, + } + + queue <- &syncTask{ + resultCh: resultCh, + } + + time.Sleep(500 * time.Millisecond) + require.Equal(t, 1, len(sharedGuard)) + <-resultCh + + time.Sleep(500 * time.Millisecond) + require.Equal(t, 1, len(sharedGuard)) + <-resultCh + + close(queue) + wg.Wait() +} From ae86ad0d8f9441c09db5fedc8aa1320d70b0a1b3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Aug 2023 15:40:33 -0400 Subject: [PATCH 027/128] chore(deps): bump github.com/multiformats/go-multiaddr from 0.10.1 to 0.11.0 (#3441) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index f50872918c..8cabeb660a 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/libp2p/go-libp2p v0.27.7 github.com/libp2p/go-libp2p-kad-dht v0.24.2 github.com/minio/sha256-simd v1.0.1 - github.com/multiformats/go-multiaddr v0.10.1 + github.com/multiformats/go-multiaddr v0.11.0 github.com/nanobox-io/golang-scribble v0.0.0-20190309225732-aa3e7c118975 github.com/prometheus/client_golang v1.16.0 github.com/prometheus/client_model v0.4.0 @@ -41,7 +41,7 @@ require ( github.com/wasmerio/go-ext-wasm v0.3.2-0.20200326095750-0a32be6068ec github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9 golang.org/x/crypto v0.12.0 - golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df + golang.org/x/exp v0.0.0-20230725012225-302865e7556b golang.org/x/term v0.11.0 google.golang.org/protobuf v1.31.0 ) diff --git a/go.sum b/go.sum index e0850f6874..4762170d6d 100644 --- a/go.sum +++ b/go.sum @@ -532,8 +532,8 @@ github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9 github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= -github.com/multiformats/go-multiaddr v0.10.1 h1:HghtFrWyZEPrpTvgAMFJi6gFdgHfs2cb0pyfDsk+lqU= -github.com/multiformats/go-multiaddr v0.10.1/go.mod h1:jLEZsA61rwWNZQTHHnqq2HNa+4os/Hz54eqiRnsRqYQ= +github.com/multiformats/go-multiaddr v0.11.0 h1:XqGyJ8ufbCE0HmTDwx2kPdsrQ36AGPZNZX6s6xfJH10= +github.com/multiformats/go-multiaddr v0.11.0/go.mod h1:gWUm0QLR4thQ6+ZF6SXUw8YjtwQSPapICM+NmCkxHSM= github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= @@ -796,8 +796,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= -golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230725012225-302865e7556b h1:tK7yjGqVRzYdXsBcfD2MLhFAhHfDgGLm2rY1ub7FA9k= +golang.org/x/exp v0.0.0-20230725012225-302865e7556b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= From 7046c947f4c318f2b949162980fab2f88c7e77c5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Aug 2023 22:53:48 -0400 Subject: [PATCH 028/128] chore(deps): bump github.com/go-playground/validator/v10 from 10.14.1 to 10.15.0 (#3439) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8cabeb660a..275d13f0e4 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/disiqueira/gotree v1.0.0 github.com/ethereum/go-ethereum v1.12.0 github.com/fatih/color v1.15.0 - github.com/go-playground/validator/v10 v10.14.1 + github.com/go-playground/validator/v10 v10.15.0 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.9 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index 4762170d6d..3aca3ae6ba 100644 --- a/go.sum +++ b/go.sum @@ -222,8 +222,8 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= -github.com/go-playground/validator/v10 v10.14.1 h1:9c50NUPC30zyuKprjL3vNZ0m5oG+jU0zvx4AqHGnv4k= -github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/go-playground/validator/v10 v10.15.0 h1:nDU5XeOKtB3GEa+uB7GNYwhVKsgjAR7VgKoNB6ryXfw= +github.com/go-playground/validator/v10 v10.15.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= From 9dc3af5abc029aa0e44c535af8a272294c078594 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Aug 2023 23:23:33 -0400 Subject: [PATCH 029/128] chore(deps): bump github.com/ethereum/go-ethereum from 1.12.0 to 1.12.2 (#3440) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 275d13f0e4..d4c6997bdd 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/dgraph-io/badger/v4 v4.1.0 github.com/dgraph-io/ristretto v0.1.1 github.com/disiqueira/gotree v1.0.0 - github.com/ethereum/go-ethereum v1.12.0 + github.com/ethereum/go-ethereum v1.12.2 github.com/fatih/color v1.15.0 github.com/go-playground/validator/v10 v10.15.0 github.com/golang/mock v1.6.0 @@ -41,7 +41,7 @@ require ( github.com/wasmerio/go-ext-wasm v0.3.2-0.20200326095750-0a32be6068ec github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9 golang.org/x/crypto v0.12.0 - golang.org/x/exp v0.0.0-20230725012225-302865e7556b + golang.org/x/exp v0.0.0-20230810033253-352e893a4cad golang.org/x/term v0.11.0 google.golang.org/protobuf v1.31.0 ) @@ -92,7 +92,7 @@ require ( github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c // indirect + github.com/holiman/uint256 v1.2.3 // indirect github.com/huin/goupnp v1.2.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/boxo v0.10.0 // indirect @@ -185,7 +185,7 @@ require ( go.uber.org/zap v1.24.0 // indirect golang.org/x/mod v0.11.0 // indirect golang.org/x/net v0.10.0 // indirect - golang.org/x/sync v0.2.0 // indirect + golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.11.0 // indirect golang.org/x/text v0.12.0 // indirect golang.org/x/tools v0.9.1 // indirect diff --git a/go.sum b/go.sum index 3aca3ae6ba..12fdb9cdc0 100644 --- a/go.sum +++ b/go.sum @@ -180,8 +180,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/go-ethereum v1.12.0 h1:bdnhLPtqETd4m3mS8BGMNvBTf36bO5bx/hxE2zljOa0= -github.com/ethereum/go-ethereum v1.12.0/go.mod h1:/oo2X/dZLJjf2mJ6YT9wcWxa4nNJDBKDBU6sFIpx1Gs= +github.com/ethereum/go-ethereum v1.12.2 h1:eGHJ4ij7oyVqUQn48LBz3B7pvQ8sV0wGJiIE6gDq/6Y= +github.com/ethereum/go-ethereum v1.12.2/go.mod h1:1cRAEV+rp/xX0zraSCBnu9Py3HQ+geRMj3HdR+k0wfI= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= @@ -362,8 +362,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5 github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c h1:DZfsyhDK1hnSS5lH8l+JggqzEleHteTYfutAiVlSUM8= -github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= +github.com/holiman/uint256 v1.2.3 h1:K8UWO1HUJpRMXBxbmaY1Y8IAMZC/RsKB+ArEnnK4l5o= +github.com/holiman/uint256 v1.2.3/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= @@ -796,8 +796,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230725012225-302865e7556b h1:tK7yjGqVRzYdXsBcfD2MLhFAhHfDgGLm2rY1ub7FA9k= -golang.org/x/exp v0.0.0-20230725012225-302865e7556b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230810033253-352e893a4cad h1:g0bG7Z4uG+OgH2QDODnjp6ggkk1bJDsINcuWmJN1iJU= +golang.org/x/exp v0.0.0-20230810033253-352e893a4cad/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -893,8 +893,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= From 970e6aad4b4cde756fae8be8d8a2ac667ff9aa4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20Junior?= Date: Tue, 15 Aug 2023 12:02:25 -0400 Subject: [PATCH 030/128] chore(deepsource): address Deepsource suggestions (#3436) --- dot/state/grandpa.go | 4 +- dot/state/grandpa_test.go | 5 +- dot/state/service_integration_test.go | 36 ++++++++++++ dot/state/slot_test.go | 2 +- dot/state/test_helpers.go | 38 ------------- lib/grandpa/grandpa.go | 27 --------- lib/grandpa/grandpa_integration_test.go | 73 ------------------------- 7 files changed, 43 insertions(+), 142 deletions(-) diff --git a/dot/state/grandpa.go b/dot/state/grandpa.go index 68c5ee9d8f..e8bebbcaa4 100644 --- a/dot/state/grandpa.go +++ b/dot/state/grandpa.go @@ -568,7 +568,7 @@ func (s *GrandpaState) GetPrevotes(round, setID uint64) ([]types.GrandpaSignedVo return nil, err } - pvs := []types.GrandpaSignedVote{} + var pvs []types.GrandpaSignedVote err = scale.Unmarshal(data, &pvs) if err != nil { return nil, err @@ -594,7 +594,7 @@ func (s *GrandpaState) GetPrecommits(round, setID uint64) ([]types.GrandpaSigned return nil, err } - pcs := []types.GrandpaSignedVote{} + var pcs []types.GrandpaSignedVote err = scale.Unmarshal(data, &pcs) if err != nil { return nil, err diff --git a/dot/state/grandpa_test.go b/dot/state/grandpa_test.go index 53b3c5d1b8..fefc9ac451 100644 --- a/dot/state/grandpa_test.go +++ b/dot/state/grandpa_test.go @@ -143,7 +143,7 @@ func testBlockState(t *testing.T, db *chaindb.BadgerDB) *BlockState { return bs } -func TestAddScheduledChangesKeepTheRightForkTree(t *testing.T) { //nolint:tparallel +func TestAddScheduledChangesKeepTheRightForkTree(t *testing.T) { t.Parallel() keyring, err := keystore.NewSr25519Keyring() @@ -218,7 +218,10 @@ func TestAddScheduledChangesKeepTheRightForkTree(t *testing.T) { //nolint:tparal } for tname, tt := range tests { + tt := tt t.Run(tname, func(t *testing.T) { + t.Parallel() + // clear the scheduledChangeRoots after the test ends // this does not cause race condition because t.Run without // t.Parallel() blocks until this function returns diff --git a/dot/state/service_integration_test.go b/dot/state/service_integration_test.go index 9dbc25c3d4..3d696fd846 100644 --- a/dot/state/service_integration_test.go +++ b/dot/state/service_integration_test.go @@ -15,6 +15,7 @@ import ( "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/internal/log" "github.com/ChainSafe/gossamer/lib/common" + runtime "github.com/ChainSafe/gossamer/lib/runtime/storage" "github.com/ChainSafe/gossamer/lib/trie" "github.com/golang/mock/gomock" @@ -426,3 +427,38 @@ func TestService_Import(t *testing.T) { err = serv.Stop() require.NoError(t, err) } + +func generateBlockWithRandomTrie(t *testing.T, serv *Service, + parent *common.Hash, bNum uint) (*types.Block, *runtime.TrieState) { + trieState, err := serv.Storage.TrieState(nil) + require.NoError(t, err) + + // Generate random data for trie state. + rand := time.Now().UnixNano() + key := []byte("testKey" + fmt.Sprint(rand)) + value := []byte("testValue" + fmt.Sprint(rand)) + err = trieState.Put(key, value) + require.NoError(t, err) + + trieStateRoot, err := trieState.Root() + require.NoError(t, err) + + if parent == nil { + bb := serv.Block.BestBlockHash() + parent = &bb + } + + body, err := types.NewBodyFromBytes([]byte{}) + require.NoError(t, err) + + block := &types.Block{ + Header: types.Header{ + ParentHash: *parent, + Number: bNum, + StateRoot: trieStateRoot, + Digest: createPrimaryBABEDigest(t), + }, + Body: *body, + } + return block, trieState +} diff --git a/dot/state/slot_test.go b/dot/state/slot_test.go index e7f1eea43e..a1fe7f5051 100644 --- a/dot/state/slot_test.go +++ b/dot/state/slot_test.go @@ -47,7 +47,7 @@ func checkSlotToMapKeyExists(t *testing.T, db chaindb.Database, slotNumber uint6 slotEncoded := make([]byte, 8) binary.LittleEndian.PutUint64(slotEncoded, slotNumber) - slotToHeaderKey := bytes.Join([][]byte{slotHeaderMapKey, slotEncoded[:]}, nil) + slotToHeaderKey := bytes.Join([][]byte{slotHeaderMapKey, slotEncoded}, nil) _, err := db.Get(slotToHeaderKey) if err != nil { diff --git a/dot/state/test_helpers.go b/dot/state/test_helpers.go index 11818a5924..9353e2303c 100644 --- a/dot/state/test_helpers.go +++ b/dot/state/test_helpers.go @@ -5,7 +5,6 @@ package state import ( "crypto/rand" - "fmt" "math/big" "testing" "time" @@ -13,7 +12,6 @@ import ( "github.com/ChainSafe/chaindb" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" - runtime "github.com/ChainSafe/gossamer/lib/runtime/storage" "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" "github.com/ChainSafe/gossamer/pkg/scale" @@ -234,39 +232,3 @@ func AddBlocksToStateWithFixedBranches(t *testing.T, blockState *BlockState, dep } } } - -//lint:ignore U1000 this has a real reason -func generateBlockWithRandomTrie(t *testing.T, serv *Service, - parent *common.Hash, bNum uint) (*types.Block, *runtime.TrieState) { - trieState, err := serv.Storage.TrieState(nil) - require.NoError(t, err) - - // Generate random data for trie state. - rand := time.Now().UnixNano() - key := []byte("testKey" + fmt.Sprint(rand)) - value := []byte("testValue" + fmt.Sprint(rand)) - err = trieState.Put(key, value, trie.V0) - require.NoError(t, err) - - trieStateRoot, err := trieState.Root() - require.NoError(t, err) - - if parent == nil { - bb := serv.Block.BestBlockHash() - parent = &bb - } - - body, err := types.NewBodyFromBytes([]byte{}) - require.NoError(t, err) - - block := &types.Block{ - Header: types.Header{ - ParentHash: *parent, - Number: bNum, - StateRoot: trieStateRoot, - Digest: createPrimaryBABEDigest(t), - }, - Body: *body, - } - return block, trieState -} diff --git a/lib/grandpa/grandpa.go b/lib/grandpa/grandpa.go index 85d5b20ab2..f971eb4bcc 100644 --- a/lib/grandpa/grandpa.go +++ b/lib/grandpa/grandpa.go @@ -78,7 +78,6 @@ type Service struct { // channels for communication with other services finalisedCh chan *types.FinalisationInfo - //receivedCommit chan *CommitMessage telemetry Telemetry } @@ -1049,32 +1048,6 @@ func (s *Service) getVotes(stage Subround) []Vote { return maps.Keys(votes) } -// findParentWithNumber returns a Vote for an ancestor with number n given an existing Vote -func (s *Service) findParentWithNumber(v *Vote, n uint32) (*Vote, error) { - if v.Number <= n { - return v, nil - } - - b, err := s.blockState.GetHeader(v.Hash) - if err != nil { - return nil, err - } - - // # of iterations - l := int(v.Number - n) - - for i := 0; i < l; i++ { - p, err := s.blockState.GetHeader(b.ParentHash) - if err != nil { - return nil, err - } - - b = p - } - - return NewVoteFromHeader(b), nil -} - // GetSetID returns the current setID func (s *Service) GetSetID() uint64 { return s.state.setID diff --git a/lib/grandpa/grandpa_integration_test.go b/lib/grandpa/grandpa_integration_test.go index 41781c1ecd..1771e585af 100644 --- a/lib/grandpa/grandpa_integration_test.go +++ b/lib/grandpa/grandpa_integration_test.go @@ -782,31 +782,6 @@ func TestGetPreVotedBlock_EvenMoreCandidates(t *testing.T) { require.Equal(t, uint32(4), block.Number) } -func TestFindParentWithNumber(t *testing.T) { - t.Parallel() - - kr, err := keystore.NewEd25519Keyring() - require.NoError(t, err) - aliceKeyPair := kr.Alice().(*ed25519.Keypair) - - gs, st := newTestService(t, aliceKeyPair) - - // no branches needed - state.AddBlocksToStateWithFixedBranches(t, st.Block, 8, nil) - leaves := gs.blockState.(*state.BlockState).Leaves() - - v, err := NewVoteFromHash(leaves[0], st.Block) - require.NoError(t, err) - - p, err := gs.findParentWithNumber(v, 1) - require.NoError(t, err) - - expected, err := st.Block.GetBlockByNumber(1) - require.NoError(t, err) - - require.Equal(t, expected.Header.Hash(), p.Hash) -} - func TestGetBestFinalCandidate_OneBlock(t *testing.T) { kr, err := keystore.NewEd25519Keyring() require.NoError(t, err) @@ -849,54 +824,6 @@ func TestGetBestFinalCandidate_OneBlock(t *testing.T) { require.Equal(t, voteA, bfc) } -func TestGetBestFinalCandidate_PrecommitAncestor(t *testing.T) { - t.Parallel() - - kr, err := keystore.NewEd25519Keyring() - require.NoError(t, err) - aliceKeyPair := kr.Alice().(*ed25519.Keypair) - - // this tests the case when the highest precommited block is an ancestor of the prevoted block - gs, st := newTestService(t, aliceKeyPair) - - branches := map[uint]int{6: 1} - state.AddBlocksToStateWithFixedBranches(t, st.Block, 8, branches) - leaves := gs.blockState.(*state.BlockState).Leaves() - - voteA, err := NewVoteFromHash(leaves[0], st.Block) - require.NoError(t, err) - voteB, err := NewVoteFromHash(leaves[1], st.Block) - require.NoError(t, err) - - // in precommit round, 2/3 voters will vote for ancestor of A - voteC, err := gs.findParentWithNumber(voteA, 6) - require.NoError(t, err) - - for i, k := range kr.Keys { - voter := k.Public().(*ed25519.PublicKey).AsBytes() - - if i < 7 { - gs.prevotes.Store(voter, &SignedVote{ - Vote: *voteA, - }) - gs.precommits.Store(voter, &SignedVote{ - Vote: *voteC, - }) - } else { - gs.prevotes.Store(voter, &SignedVote{ - Vote: *voteB, - }) - gs.precommits.Store(voter, &SignedVote{ - Vote: *voteB, - }) - } - } - - bfc, err := gs.getBestFinalCandidate() - require.NoError(t, err) - require.Equal(t, voteC, bfc) -} - func TestGetBestFinalCandidate_NoPrecommit(t *testing.T) { t.Parallel() From 2ebf18443a7d314bdee23c0fddbd5f011ab75b97 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 15 Aug 2023 17:28:34 -0300 Subject: [PATCH 031/128] Fix linter --- dot/state/service_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dot/state/service_integration_test.go b/dot/state/service_integration_test.go index 3d696fd846..91572d482c 100644 --- a/dot/state/service_integration_test.go +++ b/dot/state/service_integration_test.go @@ -437,7 +437,7 @@ func generateBlockWithRandomTrie(t *testing.T, serv *Service, rand := time.Now().UnixNano() key := []byte("testKey" + fmt.Sprint(rand)) value := []byte("testValue" + fmt.Sprint(rand)) - err = trieState.Put(key, value) + err = trieState.Put(key, value, trie.V0) require.NoError(t, err) trieStateRoot, err := trieState.Root() From 1761922f2976a5ffdc9920576ff40708c69dc0c3 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 15 Aug 2023 21:28:10 -0300 Subject: [PATCH 032/128] Get hashed values --- dot/state/db_getter_mocks_test.go | 49 --------------------- dot/state/db_mocks_test.go | 63 +++++++++++++++++++++++++++ dot/state/interfaces.go | 3 +- dot/state/mocks_generate_test.go | 2 +- dot/state/storage.go | 2 +- dot/state/storage_test.go | 6 +-- dot/state/tries_test.go | 14 +++--- lib/runtime/wasmer/exports_test.go | 2 +- lib/trie/db/db.go | 25 +++++++++-- lib/trie/proof/database_mocks_test.go | 14 ++++++ lib/trie/proof/generate.go | 3 +- lib/trie/trie.go | 43 +++++++++++------- lib/trie/trie_test.go | 23 ++++++++-- 13 files changed, 162 insertions(+), 87 deletions(-) delete mode 100644 dot/state/db_getter_mocks_test.go create mode 100644 dot/state/db_mocks_test.go diff --git a/dot/state/db_getter_mocks_test.go b/dot/state/db_getter_mocks_test.go deleted file mode 100644 index 9493704a06..0000000000 --- a/dot/state/db_getter_mocks_test.go +++ /dev/null @@ -1,49 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/ChainSafe/gossamer/lib/trie (interfaces: DBGetter) - -// Package state is a generated GoMock package. -package state - -import ( - reflect "reflect" - - gomock "github.com/golang/mock/gomock" -) - -// MockDBGetter is a mock of DBGetter interface. -type MockDBGetter struct { - ctrl *gomock.Controller - recorder *MockDBGetterMockRecorder -} - -// MockDBGetterMockRecorder is the mock recorder for MockDBGetter. -type MockDBGetterMockRecorder struct { - mock *MockDBGetter -} - -// NewMockDBGetter creates a new mock instance. -func NewMockDBGetter(ctrl *gomock.Controller) *MockDBGetter { - mock := &MockDBGetter{ctrl: ctrl} - mock.recorder = &MockDBGetterMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockDBGetter) EXPECT() *MockDBGetterMockRecorder { - return m.recorder -} - -// Get mocks base method. -func (m *MockDBGetter) Get(arg0 []byte) ([]byte, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Get", arg0) - ret0, _ := ret[0].([]byte) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Get indicates an expected call of Get. -func (mr *MockDBGetterMockRecorder) Get(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockDBGetter)(nil).Get), arg0) -} diff --git a/dot/state/db_mocks_test.go b/dot/state/db_mocks_test.go new file mode 100644 index 0000000000..6df73ac4d8 --- /dev/null +++ b/dot/state/db_mocks_test.go @@ -0,0 +1,63 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/ChainSafe/gossamer/lib/trie (interfaces: Database) + +// Package state is a generated GoMock package. +package state + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockDatabase is a mock of Database interface. +type MockDatabase struct { + ctrl *gomock.Controller + recorder *MockDatabaseMockRecorder +} + +// MockDatabaseMockRecorder is the mock recorder for MockDatabase. +type MockDatabaseMockRecorder struct { + mock *MockDatabase +} + +// NewMockDatabase creates a new mock instance. +func NewMockDatabase(ctrl *gomock.Controller) *MockDatabase { + mock := &MockDatabase{ctrl: ctrl} + mock.recorder = &MockDatabaseMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDatabase) EXPECT() *MockDatabaseMockRecorder { + return m.recorder +} + +// Get mocks base method. +func (m *MockDatabase) Get(arg0 []byte) ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", arg0) + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockDatabaseMockRecorder) Get(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockDatabase)(nil).Get), arg0) +} + +// Put mocks base method. +func (m *MockDatabase) Put(arg0, arg1 []byte) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Put", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Put indicates an expected call of Put. +func (mr *MockDatabaseMockRecorder) Put(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*MockDatabase)(nil).Put), arg0, arg1) +} diff --git a/dot/state/interfaces.go b/dot/state/interfaces.go index bae7631e51..7c193e3af6 100644 --- a/dot/state/interfaces.go +++ b/dot/state/interfaces.go @@ -31,8 +31,9 @@ type GetPutter interface { // GetNewBatcher has methods to get values and create a // new batch. -type GetNewBatcher interface { +type Batcher interface { Getter + Putter NewBatcher } diff --git a/dot/state/mocks_generate_test.go b/dot/state/mocks_generate_test.go index e52212d053..fdac9b006f 100644 --- a/dot/state/mocks_generate_test.go +++ b/dot/state/mocks_generate_test.go @@ -7,4 +7,4 @@ package state //go:generate mockgen -destination=mocks_runtime_test.go -package $GOPACKAGE github.com/ChainSafe/gossamer/lib/runtime Instance //go:generate mockgen -destination=mock_gauge_test.go -package $GOPACKAGE github.com/prometheus/client_golang/prometheus Gauge //go:generate mockgen -destination=mock_counter_test.go -package $GOPACKAGE github.com/prometheus/client_golang/prometheus Counter -//go:generate mockgen -destination=db_getter_mocks_test.go -package=$GOPACKAGE github.com/ChainSafe/gossamer/lib/trie DBGetter +//go:generate mockgen -destination=db_mocks_test.go -package=$GOPACKAGE github.com/ChainSafe/gossamer/lib/trie Database diff --git a/dot/state/storage.go b/dot/state/storage.go index efaba3acff..15d2810cb3 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -33,7 +33,7 @@ type StorageState struct { blockState *BlockState tries *Tries - db GetNewBatcher + db Batcher sync.RWMutex // change notifiers diff --git a/dot/state/storage_test.go b/dot/state/storage_test.go index 7d6da57e65..62de71be2c 100644 --- a/dot/state/storage_test.go +++ b/dot/state/storage_test.go @@ -178,15 +178,15 @@ func TestGetStorageChildAndGetStorageFromChild(t *testing.T) { genHeader.Hash(), "0", )) - dbGetter := NewMockDBGetter(ctrl) - dbGetter.EXPECT().Get(gomock.Any()).Times(0) + trieDB := NewMockDatabase(ctrl) + trieDB.EXPECT().Get(gomock.Any()).Times(0) trieRoot := &node.Node{ PartialKey: []byte{1, 2}, StorageValue: []byte{3, 4}, Dirty: true, } - testChildTrie := trie.NewTrie(trieRoot, dbGetter) + testChildTrie := trie.NewTrie(trieRoot, trieDB) testChildTrie.Put([]byte("keyInsidechild"), []byte("voila"), trie.V0) diff --git a/dot/state/tries_test.go b/dot/state/tries_test.go index f97c4934e9..4bdc84e1d2 100644 --- a/dot/state/tries_test.go +++ b/dot/state/tries_test.go @@ -49,10 +49,10 @@ func Test_Tries_SetEmptyTrie(t *testing.T) { func Test_Tries_SetTrie(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) - dbGetter := NewMockDBGetter(ctrl) - dbGetter.EXPECT().Get(gomock.Any()).Times(0) + db := NewMockDatabase(ctrl) + db.EXPECT().Get(gomock.Any()).Times(0) - tr := trie.NewTrie(&node.Node{PartialKey: []byte{1}}, dbGetter) + tr := trie.NewTrie(&node.Node{PartialKey: []byte{1}}, db) tries := NewTries() tries.SetTrie(tr) @@ -192,8 +192,8 @@ func Test_Tries_delete(t *testing.T) { func Test_Tries_get(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) - dbGetter := NewMockDBGetter(ctrl) - dbGetter.EXPECT().Get(gomock.Any()).Times(0) + db := NewMockDatabase(ctrl) + db.EXPECT().Get(gomock.Any()).Times(0) testCases := map[string]struct { tries *Tries @@ -206,14 +206,14 @@ func Test_Tries_get(t *testing.T) { {1, 2, 3}: trie.NewTrie(&node.Node{ PartialKey: []byte{1, 2, 3}, StorageValue: []byte{1}, - }, dbGetter), + }, db), }, }, root: common.Hash{1, 2, 3}, trie: trie.NewTrie(&node.Node{ PartialKey: []byte{1, 2, 3}, StorageValue: []byte{1}, - }, dbGetter), + }, db), }, "not_found_in_map": { // similar to not found in database diff --git a/lib/runtime/wasmer/exports_test.go b/lib/runtime/wasmer/exports_test.go index d20d6eaddc..0be7186f70 100644 --- a/lib/runtime/wasmer/exports_test.go +++ b/lib/runtime/wasmer/exports_test.go @@ -627,7 +627,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1(t *testing.T) { } func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock3784(t *testing.T) { - gossTrie3783 := newTrieFromPairs(t, "../test_data/kusama/block3783.out", trie.V1) + gossTrie3783 := newTrieFromPairs(t, "../test_data/kusama/block3783.out", trie.V0) expectedRoot := common.MustHexToHash("0x948338bc0976aee78879d559a1f42385407e5a481b05a91d2a9386aa7507e7a0") require.Equal(t, expectedRoot, gossTrie3783.MustHash()) diff --git a/lib/trie/db/db.go b/lib/trie/db/db.go index 2c04b28e05..cdbd3027c7 100644 --- a/lib/trie/db/db.go +++ b/lib/trie/db/db.go @@ -10,6 +10,13 @@ import ( type MemoryDB struct { data map[common.Hash][]byte + // TODO: add lock +} + +func NewEmptyMemoryDB() *MemoryDB { + return &MemoryDB{ + data: make(map[common.Hash][]byte), + } } func NewMemoryDBFromProof(encodedNodes [][]byte) (*MemoryDB, error) { @@ -30,9 +37,9 @@ func NewMemoryDBFromProof(encodedNodes [][]byte) (*MemoryDB, error) { } -func (mdb *MemoryDB) Get(key []byte) (value []byte, err error) { - if len(key) < common.HashLength { - return nil, fmt.Errorf("expected %d bytes length key, given %d (%x)", common.HashLength, len(key), value) +func (mdb *MemoryDB) Get(key []byte) ([]byte, error) { + if len(key) != common.HashLength { + return nil, fmt.Errorf("expected %d bytes length key, given %d (%x)", common.HashLength, len(key), key) } var hash common.Hash copy(hash[:], key) @@ -43,3 +50,15 @@ func (mdb *MemoryDB) Get(key []byte) (value []byte, err error) { return nil, nil } + +func (mdb *MemoryDB) Put(key []byte, value []byte) error { + if len(key) != common.HashLength { + return fmt.Errorf("expected %d bytes length key, given %d (%x)", common.HashLength, len(key), key) + } + + var hash common.Hash + copy(hash[:], key) + + mdb.data[hash] = value + return nil +} diff --git a/lib/trie/proof/database_mocks_test.go b/lib/trie/proof/database_mocks_test.go index 69262dc315..ec07c4c1ee 100644 --- a/lib/trie/proof/database_mocks_test.go +++ b/lib/trie/proof/database_mocks_test.go @@ -47,3 +47,17 @@ func (mr *MockDatabaseMockRecorder) Get(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockDatabase)(nil).Get), arg0) } + +// Put mocks base method. +func (m *MockDatabase) Put(arg0, arg1 []byte) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Put", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Put indicates an expected call of Put. +func (mr *MockDatabaseMockRecorder) Put(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*MockDatabase)(nil).Put), arg0, arg1) +} diff --git a/lib/trie/proof/generate.go b/lib/trie/proof/generate.go index 2fd30d9ddd..6155ebac1c 100644 --- a/lib/trie/proof/generate.go +++ b/lib/trie/proof/generate.go @@ -22,7 +22,8 @@ var ( // Database defines a key value Get method used // for proof generation. type Database interface { - Get(key []byte) (value []byte, err error) + trie.DBGetter + trie.DBPutter } // Generate generates and deduplicates the encoded proof nodes diff --git a/lib/trie/trie.go b/lib/trie/trie.go index 6e8e50eb6a..a5749e6123 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -11,17 +11,23 @@ import ( "github.com/ChainSafe/gossamer/internal/trie/node" "github.com/ChainSafe/gossamer/internal/trie/tracking" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/trie/db" ) // EmptyHash is the empty trie hash. var EmptyHash = common.MustBlake2bHash([]byte{0}) +type Database interface { + DBGetter + DBPutter +} + // Trie is a base 16 modified Merkle Patricia trie. type Trie struct { generation uint64 root *Node childTries map[common.Hash]*Trie - db DBGetter + db Database // deltas stores trie deltas since the last trie snapshot. // For example node hashes that were deleted since // the last snapshot. These are used by the online @@ -32,11 +38,11 @@ type Trie struct { // NewEmptyTrie creates a trie with a nil root func NewEmptyTrie() *Trie { - return NewTrie(nil, nil) + return NewTrie(nil, db.NewEmptyMemoryDB()) } // NewTrie creates a trie with an existing root node -func NewTrie(root *Node, db DBGetter) *Trie { +func NewTrie(root *Node, db Database) *Trie { return &Trie{ root: root, childTries: make(map[common.Hash]*Trie), @@ -232,33 +238,33 @@ func entriesList(parent *Node, prefix []byte, list *[][2][]byte) { // where the keys are encoded in Little Endian. func (t *Trie) Entries() (keyValueMap map[string][]byte) { keyValueMap = make(map[string][]byte) - entries(t.root, nil, keyValueMap) + t.buildEntriesMap(t.root, nil, keyValueMap) return keyValueMap } -func entries(parent *Node, prefix []byte, kv map[string][]byte) { - if parent == nil { +func (t *Trie) buildEntriesMap(currentNode *Node, prefix []byte, kv map[string][]byte) { + if currentNode == nil { return } - if parent.Kind() == node.Leaf { - parentKey := parent.PartialKey - fullKeyNibbles := concatenateSlices(prefix, parentKey) - keyLE := string(codec.NibblesToKeyLE(fullKeyNibbles)) - kv[keyLE] = parent.StorageValue + if currentNode.Kind() == node.Leaf { + key := currentNode.PartialKey + fullKeyNibbles := concatenateSlices(prefix, key) + keyLE := codec.NibblesToKeyLE(fullKeyNibbles) + kv[string(keyLE)] = t.Get(keyLE) return } - branch := parent + branch := currentNode if branch.StorageValue != nil { fullKeyNibbles := concatenateSlices(prefix, branch.PartialKey) - keyLE := string(codec.NibblesToKeyLE(fullKeyNibbles)) - kv[keyLE] = branch.StorageValue + keyLE := codec.NibblesToKeyLE(fullKeyNibbles) + kv[string(keyLE)] = t.Get(keyLE) } for i, child := range branch.Children { childPrefix := concatenateSlices(prefix, branch.PartialKey, intToByteSlice(i)) - entries(child, childPrefix, kv) + t.buildEntriesMap(child, childPrefix, kv) } } @@ -375,7 +381,12 @@ func (t *Trie) insertKeyLE(keyLE, value []byte, isValueHashed := version.ShouldHashValue(value) if isValueHashed { - value = common.MustBlake2bHash(value).ToBytes() + hashedValue, err := common.Blake2bHash(value) + if err != nil { + return err + } + t.db.Put(hashedValue.ToBytes(), value) + value = hashedValue.ToBytes() } root, _, _, err := t.insert(t.root, nibblesKey, value, isValueHashed, pendingDeltas) diff --git a/lib/trie/trie_test.go b/lib/trie/trie_test.go index 2a58d7ebc3..1adc8993e3 100644 --- a/lib/trie/trie_test.go +++ b/lib/trie/trie_test.go @@ -12,6 +12,7 @@ import ( "github.com/ChainSafe/gossamer/internal/trie/node" "github.com/ChainSafe/gossamer/internal/trie/tracking" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/trie/db" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -33,6 +34,7 @@ func Test_NewEmptyTrie(t *testing.T) { expectedTrie := &Trie{ childTries: make(map[common.Hash]*Trie), deltas: tracking.New(), + db: db.NewEmptyMemoryDB(), } trie := NewEmptyTrie() assert.Equal(t, expectedTrie, trie) @@ -604,12 +606,12 @@ func Test_Trie_Entries(t *testing.T) { t.Parallel() root := &Node{ - PartialKey: []byte{0xa}, + PartialKey: []byte{0x0, 0xa}, StorageValue: []byte("root"), Descendants: 2, Children: padRightChildren([]*Node{ { // index 0 - PartialKey: []byte{2, 0xb}, + PartialKey: []byte{0xb}, StorageValue: []byte("leaf"), }, nil, @@ -626,7 +628,7 @@ func Test_Trie_Entries(t *testing.T) { expectedEntries := map[string][]byte{ string([]byte{0x0a}): []byte("root"), - string([]byte{0xa0, 0x2b}): []byte("leaf"), + string([]byte{0x0a, 0x0b}): []byte("leaf"), string([]byte{0x0a, 0x2b}): []byte("leaf"), } @@ -696,6 +698,7 @@ func Test_Trie_Entries(t *testing.T) { trie := Trie{ root: nil, childTries: make(map[common.Hash]*Trie), + db: db.NewEmptyMemoryDB(), } kv := map[string][]byte{ @@ -720,6 +723,7 @@ func Test_Trie_Entries(t *testing.T) { trie := Trie{ root: nil, childTries: make(map[common.Hash]*Trie), + db: db.NewEmptyMemoryDB(), } kv := map[string][]byte{ @@ -1075,6 +1079,7 @@ func Test_Trie_Put(t *testing.T) { t.Parallel() longValue := []byte("newvaluewithmorethan32byteslength") + longValueHash := common.MustBlake2bHash(longValue).ToBytes() testCases := map[string]struct { trie Trie @@ -1127,6 +1132,7 @@ func Test_Trie_Put(t *testing.T) { PartialKey: []byte{1, 2, 0, 5}, StorageValue: []byte{1}, }, + db: db.NewEmptyMemoryDB(), }, stateVersion: V1, key: []byte{0x12, 0x16}, @@ -1148,13 +1154,18 @@ func Test_Trie_Put(t *testing.T) { }, { PartialKey: []byte{6}, - StorageValue: common.MustBlake2bHash(longValue).ToBytes(), + StorageValue: longValueHash, HashedValue: true, Generation: 1, Dirty: true, }, }), }, + db: func() Database { + db := db.NewEmptyMemoryDB() + db.Put(longValueHash, longValue) + return db + }(), }, }, } @@ -1723,6 +1734,7 @@ func Test_LoadFromMap(t *testing.T) { expectedTrie: Trie{ childTries: map[common.Hash]*Trie{}, deltas: newDeltas(), + db: db.NewEmptyMemoryDB(), }, }, "empty_data": { @@ -1730,6 +1742,7 @@ func Test_LoadFromMap(t *testing.T) { expectedTrie: Trie{ childTries: map[common.Hash]*Trie{}, deltas: newDeltas(), + db: db.NewEmptyMemoryDB(), }, }, "bad_key": { @@ -1763,6 +1776,7 @@ func Test_LoadFromMap(t *testing.T) { }, childTries: map[common.Hash]*Trie{}, deltas: newDeltas(), + db: db.NewEmptyMemoryDB(), }, }, "load_key_values": { @@ -1793,6 +1807,7 @@ func Test_LoadFromMap(t *testing.T) { }, childTries: map[common.Hash]*Trie{}, deltas: newDeltas(), + db: db.NewEmptyMemoryDB(), }, }, } From 7b4b6d15045ee7d32a6c725572a64f6554f4116d Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 15 Aug 2023 21:36:01 -0300 Subject: [PATCH 033/128] Check memory db put value error --- lib/trie/trie.go | 7 ++++++- lib/trie/trie_test.go | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/trie/trie.go b/lib/trie/trie.go index a5749e6123..c21f6d45da 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -385,7 +385,12 @@ func (t *Trie) insertKeyLE(keyLE, value []byte, if err != nil { return err } - t.db.Put(hashedValue.ToBytes(), value) + + // Add original value in memory db + err = t.db.Put(hashedValue.ToBytes(), value) + if err != nil { + return err + } value = hashedValue.ToBytes() } diff --git a/lib/trie/trie_test.go b/lib/trie/trie_test.go index 1adc8993e3..6ca47ce25a 100644 --- a/lib/trie/trie_test.go +++ b/lib/trie/trie_test.go @@ -628,7 +628,7 @@ func Test_Trie_Entries(t *testing.T) { expectedEntries := map[string][]byte{ string([]byte{0x0a}): []byte("root"), - string([]byte{0x0a, 0x0b}): []byte("leaf"), + string([]byte{0x0a, 0xb}): []byte("leaf"), string([]byte{0x0a, 0x2b}): []byte("leaf"), } From 5ec446044774257099c36065e497f75820c717e3 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 15 Aug 2023 21:39:31 -0300 Subject: [PATCH 034/128] Add memory db lock --- lib/trie/db/db.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/trie/db/db.go b/lib/trie/db/db.go index cdbd3027c7..e5d38ddc2c 100644 --- a/lib/trie/db/db.go +++ b/lib/trie/db/db.go @@ -4,13 +4,14 @@ package db import ( "fmt" + "sync" "github.com/ChainSafe/gossamer/lib/common" ) type MemoryDB struct { data map[common.Hash][]byte - // TODO: add lock + l sync.RWMutex } func NewEmptyMemoryDB() *MemoryDB { @@ -44,6 +45,9 @@ func (mdb *MemoryDB) Get(key []byte) ([]byte, error) { var hash common.Hash copy(hash[:], key) + mdb.l.RLock() + defer mdb.l.RUnlock() + if value, found := mdb.data[hash]; found { return value, nil } @@ -59,6 +63,9 @@ func (mdb *MemoryDB) Put(key []byte, value []byte) error { var hash common.Hash copy(hash[:], key) + mdb.l.Lock() + defer mdb.l.Unlock() + mdb.data[hash] = value return nil } From 8d70d77750ad78a0e3e626eed58337cff381cadd Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 15 Aug 2023 21:48:51 -0300 Subject: [PATCH 035/128] Fix linter --- lib/runtime/wasmer/exports_test.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/runtime/wasmer/exports_test.go b/lib/runtime/wasmer/exports_test.go index 0be7186f70..a72800bcf1 100644 --- a/lib/runtime/wasmer/exports_test.go +++ b/lib/runtime/wasmer/exports_test.go @@ -627,7 +627,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1(t *testing.T) { } func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock3784(t *testing.T) { - gossTrie3783 := newTrieFromPairs(t, "../test_data/kusama/block3783.out", trie.V0) + gossTrie3783 := newTrieFromPairs(t, "../test_data/kusama/block3783.out") expectedRoot := common.MustHexToHash("0x948338bc0976aee78879d559a1f42385407e5a481b05a91d2a9386aa7507e7a0") require.Equal(t, expectedRoot, gossTrie3783.MustHash()) @@ -673,7 +673,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock3784(t *testing.T) { } func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock901442(t *testing.T) { - ksmTrie901441 := newTrieFromPairs(t, "../test_data/kusama/block901441.out", trie.V0) + ksmTrie901441 := newTrieFromPairs(t, "../test_data/kusama/block901441.out") expectedRoot := common.MustHexToHash("0x3a2ef7ee032f5810160bb8f3ffe3e3377bb6f2769ee9f79a5425973347acd504") require.Equal(t, expectedRoot, ksmTrie901441.MustHash()) @@ -719,7 +719,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock901442(t *testing.T) { } func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1377831(t *testing.T) { - ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block1377830.out", trie.V0) + ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block1377830.out") expectedRoot := common.MustHexToHash("0xe4de6fecda9e9e35f937d159665cf984bc1a68048b6c78912de0aeb6bd7f7e99") require.Equal(t, expectedRoot, ksmTrie.MustHash()) @@ -765,7 +765,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1377831(t *testing.T) { } func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1482003(t *testing.T) { - ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block1482002.out", trie.V0) + ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block1482002.out") expectedRoot := common.MustHexToHash("0x09f9ca28df0560c2291aa16b56e15e07d1e1927088f51356d522722aa90ca7cb") require.Equal(t, expectedRoot, ksmTrie.MustHash()) @@ -813,7 +813,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1482003(t *testing.T) { func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock4939774(t *testing.T) { t.Skip("skip for now as block4939773 is too large") - ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block4939773.out", trie.V0) + ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block4939773.out") expectedRoot := common.MustHexToHash("0xc45748e6e8632b44fc32b04cc4380098a9584cbd63ffbc59adce189574fc36fe") require.Equal(t, expectedRoot, ksmTrie.MustHash()) @@ -856,7 +856,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock4939774(t *testing.T) { } func TestInstance_ExecuteBlock_PolkadotBlock1089328(t *testing.T) { - dotTrie := newTrieFromPairs(t, "../test_data/polkadot/block1089327.json", trie.V0) + dotTrie := newTrieFromPairs(t, "../test_data/polkadot/block1089327.json") expectedRoot := common.MustHexToHash("0x87ed9ebe7fb645d3b5b0255cc16e78ed022d9fbb52486105436e15a74557535b") require.Equal(t, expectedRoot, dotTrie.MustHash()) @@ -983,7 +983,7 @@ func TestInstance_PaymentQueryInfo(t *testing.T) { } } -func newTrieFromPairs(t *testing.T, filename string, version trie.Version) *trie.Trie { +func newTrieFromPairs(t *testing.T, filename string) *trie.Trie { data, err := os.ReadFile(filename) require.NoError(t, err) @@ -998,7 +998,8 @@ func newTrieFromPairs(t *testing.T, filename string, version trie.Version) *trie entries[pairArr[0].(string)] = pairArr[1].(string) } - tr, err := trie.LoadFromMap(entries, version) + // We need to use V0 for all tests using this function + tr, err := trie.LoadFromMap(entries, trie.V0) require.NoError(t, err) return &tr } From 461596e9fac0512fb726e61f3e0e3d7157d819e7 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 15 Aug 2023 22:05:48 -0300 Subject: [PATCH 036/128] Deepsource --- dot/state/interfaces.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dot/state/interfaces.go b/dot/state/interfaces.go index 7c193e3af6..913a057504 100644 --- a/dot/state/interfaces.go +++ b/dot/state/interfaces.go @@ -29,7 +29,7 @@ type GetPutter interface { Putter } -// GetNewBatcher has methods to get values and create a +// Batcher has methods to get values and create a // new batch. type Batcher interface { Getter From c9b42e9d2c5dce1a3b1af8a589954b3f3d7f6f3c Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 15 Aug 2023 22:08:41 -0300 Subject: [PATCH 037/128] Add default version --- cmd/gossamer/commands/import_state.go | 4 +--- lib/trie/version.go | 3 +++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/gossamer/commands/import_state.go b/cmd/gossamer/commands/import_state.go index 9f3f118b7b..0141271af7 100644 --- a/cmd/gossamer/commands/import_state.go +++ b/cmd/gossamer/commands/import_state.go @@ -15,6 +15,7 @@ import ( func init() { ImportStateCmd.Flags().String("chain", "", "Chain id used to load default configuration for specified chain") ImportStateCmd.Flags().String("state-file", "", "Path to JSON file consisting of key-value pairs") + ImportStateCmd.Flags().String("state-version", trie.DefaultStateVersion.String(), "State version v0 or v1") ImportStateCmd.Flags().String("header-file", "", "Path to JSON file of block header corresponding to the given state") ImportStateCmd.Flags().Uint64("first-slot", 0, "The first BABE slot of the network") } @@ -59,9 +60,6 @@ func execImportState(cmd *cobra.Command) error { if err != nil { return fmt.Errorf("failed to get state-version: %s", err) } - if stateVersionFlag == "" { - return fmt.Errorf("state-version must be specified") - } stateVersion, err := trie.ParseVersion(stateVersionFlag) if err != nil { return fmt.Errorf("failed to parse state-version: %s", err) diff --git a/lib/trie/version.go b/lib/trie/version.go index 16974e36fe..c3b5aa78cb 100644 --- a/lib/trie/version.go +++ b/lib/trie/version.go @@ -26,6 +26,9 @@ const ( V1 ) +// See https://github.com/paritytech/substrate/blob/5e76587825b9a9d52d8cb02ba38828adf606157b/primitives/storage/src/lib.rs#L435-L439 +const DefaultStateVersion = V1 + func (v Version) String() string { switch v { case V0: From 503fc2798928a208042902b53ffbce0b8c595ed7 Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 16 Aug 2023 10:26:31 -0300 Subject: [PATCH 038/128] Fix doc --- lib/trie/version.go | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/trie/version.go b/lib/trie/version.go index c3b5aa78cb..ffc782b507 100644 --- a/lib/trie/version.go +++ b/lib/trie/version.go @@ -26,6 +26,7 @@ const ( V1 ) +// DefaultStateVersion sets the state version we should use as default // See https://github.com/paritytech/substrate/blob/5e76587825b9a9d52d8cb02ba38828adf606157b/primitives/storage/src/lib.rs#L435-L439 const DefaultStateVersion = V1 From 3f6cdcc31d9be0b01480824fc2eec0262a910831 Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 16 Aug 2023 10:41:03 -0300 Subject: [PATCH 039/128] Add test to cover case for #2329 --- lib/trie/proof/proof_test.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/lib/trie/proof/proof_test.go b/lib/trie/proof/proof_test.go index f808c999b4..f47b797ed2 100644 --- a/lib/trie/proof/proof_test.go +++ b/lib/trie/proof/proof_test.go @@ -99,3 +99,37 @@ func TestParachainHeaderStateProof(t *testing.T) { require.NoError(t, err) } + +func TestTrieProof(t *testing.T) { + key, err := hex.DecodeString("f0c365c3cf59d671eb72da0e7a4113c49f1f0515f462cdcf84e0f1d6045dfcbb") + if err != nil { + panic(err) + } + root, err := hex.DecodeString("dc4887669c2a6b3462e9557aa3105a66a02b6ec3b21784613de78c95dc3cbbe0") + if err != nil { + panic(err) + } + proof1, err := hex.DecodeString("80fffd8028b54b9a0a90d41b7941c43e6a0597d5914e3b62bdcb244851b9fc806c28ea2480d5ba6d50586692888b0c2f5b3c3fc345eb3a2405996f025ed37982ca396f5ed580bd281c12f20f06077bffd56b2f8b6431ee6c9fd11fed9c22db86cea849aeff2280afa1e1b5ce72ea1675e5e69be85e98fbfb660691a76fee9229f758a75315f2bc80aafc60caa3519d4b861e6b8da226266a15060e2071bba4184e194da61dfb208e809d3f6ae8f655009551de95ae1ef863f6771522fd5c0475a50ff53c5c8169b5888024a760a8f6c27928ae9e2fed9968bc5f6e17c3ae647398d8a615e5b2bb4b425f8085a0da830399f25fca4b653de654ffd3c92be39f3ae4f54e7c504961b5bd00cf80c2d44d371e5fc1f50227d7491ad65ad049630361cefb4ab1844831237609f08380c644938921d14ae611f3a90991af8b7f5bdb8fa361ee2c646c849bca90f491e6806e729ad43a591cd1321762582782bbe4ed193c6f583ec76013126f7f786e376280509bb016f2887d12137e73d26d7ddcd7f9c8ff458147cb9d309494655fe68de180009f8697d760fbe020564b07f407e6aad58ba9451b3d2d88b3ee03e12db7c47480952dcc0804e1120508a1753f1de4aa5b7481026a3320df8b48e918f0cecbaed3803360bf948fddc403d345064082e8393d7a1aad7a19081f6d02d94358f242b86c") + if err != nil { + panic(err) + } + proof2, err := hex.DecodeString("9ec365c3cf59d671eb72da0e7a4113c41002505f0e7b9012096b41c4eb3aaf947f6ea429080000685f0f1f0515f462cdcf84e0f1d6045dfcbb20865c4a2b7f010000") + if err != nil { + panic(err) + } + proof3, err := hex.DecodeString("8005088076c66e2871b4fe037d112ebffb3bfc8bd83a4ec26047f58ee2df7be4e9ebe3d680c1638f702aaa71e4b78cc8538ecae03e827bb494cc54279606b201ec071a5e24806d2a1e6d5236e1e13c5a5c84831f5f5383f97eba32df6f9faf80e32cf2f129bc") + if err != nil { + panic(err) + } + + proof := [][]byte{proof1, proof2, proof3} + proofDB, err := db.NewMemoryDBFromProof(proof) + + require.NoError(t, err) + + trie, err := buildTrie(proof, root, proofDB) + require.NoError(t, err) + value := trie.Get(key) + + require.Equal(t, []byte{0x86, 0x5c, 0x4a, 0x2b, 0x7f, 0x1, 0x0, 0x0}, value) +} From 0c5f54268649a5d994f995ea51221ea736c9460e Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 16 Aug 2023 10:47:56 -0300 Subject: [PATCH 040/128] Fix linter --- lib/trie/proof/proof_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/trie/proof/proof_test.go b/lib/trie/proof/proof_test.go index f47b797ed2..be8517b71e 100644 --- a/lib/trie/proof/proof_test.go +++ b/lib/trie/proof/proof_test.go @@ -109,15 +109,15 @@ func TestTrieProof(t *testing.T) { if err != nil { panic(err) } - proof1, err := hex.DecodeString("80fffd8028b54b9a0a90d41b7941c43e6a0597d5914e3b62bdcb244851b9fc806c28ea2480d5ba6d50586692888b0c2f5b3c3fc345eb3a2405996f025ed37982ca396f5ed580bd281c12f20f06077bffd56b2f8b6431ee6c9fd11fed9c22db86cea849aeff2280afa1e1b5ce72ea1675e5e69be85e98fbfb660691a76fee9229f758a75315f2bc80aafc60caa3519d4b861e6b8da226266a15060e2071bba4184e194da61dfb208e809d3f6ae8f655009551de95ae1ef863f6771522fd5c0475a50ff53c5c8169b5888024a760a8f6c27928ae9e2fed9968bc5f6e17c3ae647398d8a615e5b2bb4b425f8085a0da830399f25fca4b653de654ffd3c92be39f3ae4f54e7c504961b5bd00cf80c2d44d371e5fc1f50227d7491ad65ad049630361cefb4ab1844831237609f08380c644938921d14ae611f3a90991af8b7f5bdb8fa361ee2c646c849bca90f491e6806e729ad43a591cd1321762582782bbe4ed193c6f583ec76013126f7f786e376280509bb016f2887d12137e73d26d7ddcd7f9c8ff458147cb9d309494655fe68de180009f8697d760fbe020564b07f407e6aad58ba9451b3d2d88b3ee03e12db7c47480952dcc0804e1120508a1753f1de4aa5b7481026a3320df8b48e918f0cecbaed3803360bf948fddc403d345064082e8393d7a1aad7a19081f6d02d94358f242b86c") + proof1, err := hex.DecodeString("80fffd8028b54b9a0a90d41b7941c43e6a0597d5914e3b62bdcb244851b9fc806c28ea2480d5ba6d50586692888b0c2f5b3c3fc345eb3a2405996f025ed37982ca396f5ed580bd281c12f20f06077bffd56b2f8b6431ee6c9fd11fed9c22db86cea849aeff2280afa1e1b5ce72ea1675e5e69be85e98fbfb660691a76fee9229f758a75315f2bc80aafc60caa3519d4b861e6b8da226266a15060e2071bba4184e194da61dfb208e809d3f6ae8f655009551de95ae1ef863f6771522fd5c0475a50ff53c5c8169b5888024a760a8f6c27928ae9e2fed9968bc5f6e17c3ae647398d8a615e5b2bb4b425f8085a0da830399f25fca4b653de654ffd3c92be39f3ae4f54e7c504961b5bd00cf80c2d44d371e5fc1f50227d7491ad65ad049630361cefb4ab1844831237609f08380c644938921d14ae611f3a90991af8b7f5bdb8fa361ee2c646c849bca90f491e6806e729ad43a591cd1321762582782bbe4ed193c6f583ec76013126f7f786e376280509bb016f2887d12137e73d26d7ddcd7f9c8ff458147cb9d309494655fe68de180009f8697d760fbe020564b07f407e6aad58ba9451b3d2d88b3ee03e12db7c47480952dcc0804e1120508a1753f1de4aa5b7481026a3320df8b48e918f0cecbaed3803360bf948fddc403d345064082e8393d7a1aad7a19081f6d02d94358f242b86c") //nolint:lll if err != nil { panic(err) } - proof2, err := hex.DecodeString("9ec365c3cf59d671eb72da0e7a4113c41002505f0e7b9012096b41c4eb3aaf947f6ea429080000685f0f1f0515f462cdcf84e0f1d6045dfcbb20865c4a2b7f010000") + proof2, err := hex.DecodeString("9ec365c3cf59d671eb72da0e7a4113c41002505f0e7b9012096b41c4eb3aaf947f6ea429080000685f0f1f0515f462cdcf84e0f1d6045dfcbb20865c4a2b7f010000") //nolint:lll if err != nil { panic(err) } - proof3, err := hex.DecodeString("8005088076c66e2871b4fe037d112ebffb3bfc8bd83a4ec26047f58ee2df7be4e9ebe3d680c1638f702aaa71e4b78cc8538ecae03e827bb494cc54279606b201ec071a5e24806d2a1e6d5236e1e13c5a5c84831f5f5383f97eba32df6f9faf80e32cf2f129bc") + proof3, err := hex.DecodeString("8005088076c66e2871b4fe037d112ebffb3bfc8bd83a4ec26047f58ee2df7be4e9ebe3d680c1638f702aaa71e4b78cc8538ecae03e827bb494cc54279606b201ec071a5e24806d2a1e6d5236e1e13c5a5c84831f5f5383f97eba32df6f9faf80e32cf2f129bc") //nolint:lll if err != nil { panic(err) } From 3e3878a77f5cc4b3f41df731642b5378640aed95 Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 16 Aug 2023 11:56:55 -0300 Subject: [PATCH 041/128] Use state version param in runtime functions --- lib/runtime/wazero/imports.go | 11 +++++++++-- lib/trie/version.go | 11 +++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/lib/runtime/wazero/imports.go b/lib/runtime/wazero/imports.go index 32d2cbda8a..7d4e1697ef 100644 --- a/lib/runtime/wazero/imports.go +++ b/lib/runtime/wazero/imports.go @@ -865,6 +865,10 @@ func ext_trie_blake2_256_root_version_2(ctx context.Context, m api.Module, dataS panic("nil runtime context") } + stateVersion := uint8(version) + + trie.EnforceValidVersion(stateVersion) + data := read(m, dataSpan) t := trie.NewEmptyTrie() @@ -882,7 +886,7 @@ func ext_trie_blake2_256_root_version_2(ctx context.Context, m api.Module, dataS for _, kv := range kvs { //TODO: use version parameter here - err := t.Put(kv.Key, kv.Value) + err := t.Put(kv.Key, kv.Value, trie.Version(stateVersion)) if err != nil { logger.Errorf("failed putting key 0x%x and value 0x%x into trie: %s", kv.Key, kv.Value, err) @@ -968,6 +972,9 @@ func ext_trie_blake2_256_ordered_root_version_2( panic("nil runtime context") } + stateVersion := uint8(version) + trie.EnforceValidVersion(stateVersion) + data := read(m, dataSpan) t := trie.NewEmptyTrie() @@ -989,7 +996,7 @@ func ext_trie_blake2_256_ordered_root_version_2( key, value) //TODO: use version parameter here - err = t.Put(key, value) + err = t.Put(key, value, trie.Version(stateVersion)) if err != nil { logger.Errorf("failed putting key 0x%x and value 0x%x into trie: %s", key, value, err) diff --git a/lib/trie/version.go b/lib/trie/version.go index ffc782b507..84e1105a80 100644 --- a/lib/trie/version.go +++ b/lib/trie/version.go @@ -66,3 +66,14 @@ func ParseVersion(s string) (version Version, err error) { ErrParseVersion, s, V0, V1) } } + +func EnforceValidVersion(version uint8) { + v := Version(version) + + switch v { + case V0, V1: + return + default: + panic("Invalid state version") + } +} From 4f8d81c74f517d244d4556c995f16d0ba1e4b1a5 Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 16 Aug 2023 12:12:50 -0300 Subject: [PATCH 042/128] PR comments --- lib/trie/trie.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/trie/trie.go b/lib/trie/trie.go index c21f6d45da..9316b3a059 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -386,7 +386,7 @@ func (t *Trie) insertKeyLE(keyLE, value []byte, return err } - // Add original value in memory db + // Add original value in db using the hashed value as key err = t.db.Put(hashedValue.ToBytes(), value) if err != nil { return err From 2da48d7ed4dba5a793129d5c9edc29e7bf67264d Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 16 Aug 2023 15:14:49 -0300 Subject: [PATCH 043/128] Fix merge --- dot/state/storage_notify_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dot/state/storage_notify_test.go b/dot/state/storage_notify_test.go index 02216ecd34..45965916ae 100644 --- a/dot/state/storage_notify_test.go +++ b/dot/state/storage_notify_test.go @@ -9,11 +9,7 @@ import ( "time" "github.com/ChainSafe/gossamer/lib/common" -<<<<<<< HEAD "github.com/ChainSafe/gossamer/lib/trie" - "github.com/dgraph-io/badger/v4/pb" -======= ->>>>>>> 344461dfca4d7cc341379cc778f4eeabd411e4cb "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" ) From 7e1e42fb36bce8097a01750192fb6d3ed9a03460 Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 17 Aug 2023 10:25:50 -0300 Subject: [PATCH 044/128] Fix merge --- lib/runtime/wasmer/exports_test.go | 1258 --------------- lib/runtime/wasmer/helpers.go | 275 ---- lib/runtime/wasmer/imports.go | 2288 ---------------------------- lib/runtime/wasmer/imports_test.go | 1961 ------------------------ 4 files changed, 5782 deletions(-) delete mode 100644 lib/runtime/wasmer/exports_test.go delete mode 100644 lib/runtime/wasmer/helpers.go delete mode 100644 lib/runtime/wasmer/imports.go delete mode 100644 lib/runtime/wasmer/imports_test.go diff --git a/lib/runtime/wasmer/exports_test.go b/lib/runtime/wasmer/exports_test.go deleted file mode 100644 index a72800bcf1..0000000000 --- a/lib/runtime/wasmer/exports_test.go +++ /dev/null @@ -1,1258 +0,0 @@ -// Copyright 2021 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package wasmer - -import ( - "bytes" - "encoding/json" - "math/big" - "os" - "testing" - - "github.com/ChainSafe/gossamer/dot/types" - "github.com/ChainSafe/gossamer/internal/log" - "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/crypto/ed25519" - "github.com/ChainSafe/gossamer/lib/genesis" - "github.com/ChainSafe/gossamer/lib/runtime" - "github.com/ChainSafe/gossamer/lib/runtime/storage" - "github.com/ChainSafe/gossamer/lib/runtime/wasmer/testdata" - "github.com/ChainSafe/gossamer/lib/trie" - "github.com/ChainSafe/gossamer/lib/utils" - "github.com/ChainSafe/gossamer/pkg/scale" - "github.com/centrifuge/go-substrate-rpc-client/v4/signature" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// this is generated by printing key ownership proof while running `test_generate_equivocation_report_blob` -// https://github.com/paritytech/substrate/blob/ded44948e2d5a398abcb4e342b0513cb690961bb/frame/grandpa/src/benchmarking.rs#L85 -var testKeyOwnershipProof types.OpaqueKeyOwnershipProof = types.OpaqueKeyOwnershipProof([]byte{64, 138, 252, 29, 127, 102, 189, 129, 207, 47, 157, 60, 17, 138, 194, 121, 139, 92, 176, 175, 224, 16, 185, 93, 175, 251, 224, 81, 209, 61, 0, 71}) //nolint:lll - -func Test_Instance_Version(t *testing.T) { - t.Parallel() - - type instanceVersioner interface { - Version() (runtime.Version, error) - } - - testCases := map[string]struct { - instanceBuilder func(t *testing.T) instanceVersioner - expectedVersion runtime.Version - }{ - "kusama": { - instanceBuilder: func(t *testing.T) instanceVersioner { - genesisPath := utils.GetKusamaGenesisPath(t) - kusamaGenesis := genesisFromRawJSON(t, genesisPath) - genesisTrie, err := runtime.NewTrieFromGenesis(kusamaGenesis) - require.NoError(t, err) - - cfg := Config{ - Storage: storage.NewTrieState(&genesisTrie), - LogLvl: log.Critical, - } - - instance, err := NewRuntimeFromGenesis(cfg) - require.NoError(t, err) - return instance - }, - expectedVersion: runtime.Version{ - SpecName: []byte("kusama"), - ImplName: []byte("parity-kusama"), - AuthoringVersion: 2, - SpecVersion: 1020, - ImplVersion: 0, - APIItems: []runtime.APIItem{ - {Name: [8]uint8{0xdf, 0x6a, 0xcb, 0x68, 0x99, 0x7, 0x60, 0x9b}, Ver: 0x2}, - {Name: [8]uint8{0x37, 0xe3, 0x97, 0xfc, 0x7c, 0x91, 0xf5, 0xe4}, Ver: 0x1}, - {Name: [8]uint8{0x40, 0xfe, 0x3a, 0xd4, 0x1, 0xf8, 0x95, 0x9a}, Ver: 0x4}, - {Name: [8]uint8{0xd2, 0xbc, 0x98, 0x97, 0xee, 0xd0, 0x8f, 0x15}, Ver: 0x1}, - {Name: [8]uint8{0xf7, 0x8b, 0x27, 0x8b, 0xe5, 0x3f, 0x45, 0x4c}, Ver: 0x1}, - {Name: [8]uint8{0xaf, 0x2c, 0x2, 0x97, 0xa2, 0x3e, 0x6d, 0x3d}, Ver: 0x1}, - {Name: [8]uint8{0xed, 0x99, 0xc5, 0xac, 0xb2, 0x5e, 0xed, 0xf5}, Ver: 0x2}, - {Name: [8]uint8{0xcb, 0xca, 0x25, 0xe3, 0x9f, 0x14, 0x23, 0x87}, Ver: 0x1}, - {Name: [8]uint8{0x68, 0x7a, 0xd4, 0x4a, 0xd3, 0x7f, 0x3, 0xc2}, Ver: 0x1}, - {Name: [8]uint8{0xab, 0x3c, 0x5, 0x72, 0x29, 0x1f, 0xeb, 0x8b}, Ver: 0x1}, - {Name: [8]uint8{0xbc, 0x9d, 0x89, 0x90, 0x4f, 0x5b, 0x92, 0x3f}, Ver: 0x1}, - {Name: [8]uint8{0x37, 0xc8, 0xbb, 0x13, 0x50, 0xa9, 0xa2, 0xa8}, Ver: 0x1}, - }, - }, - }, - "polkadot_v0929": { - instanceBuilder: func(t *testing.T) instanceVersioner { - return NewTestInstance(t, runtime.POLKADOT_RUNTIME_v0929) - }, - expectedVersion: runtime.Version{ - SpecName: []byte("polkadot"), - ImplName: []byte("parity-polkadot"), - AuthoringVersion: 0, - SpecVersion: 9290, - ImplVersion: 0, - APIItems: []runtime.APIItem{ - {Name: [8]uint8{0xdf, 0x6a, 0xcb, 0x68, 0x99, 0x7, 0x60, 0x9b}, Ver: 0x4}, - {Name: [8]uint8{0x37, 0xe3, 0x97, 0xfc, 0x7c, 0x91, 0xf5, 0xe4}, Ver: 0x1}, - {Name: [8]uint8{0x40, 0xfe, 0x3a, 0xd4, 0x1, 0xf8, 0x95, 0x9a}, Ver: 0x6}, - {Name: [8]uint8{0x17, 0xa6, 0xbc, 0xd, 0x0, 0x62, 0xae, 0xb3}, Ver: 0x1}, - {Name: [8]uint8{0xd2, 0xbc, 0x98, 0x97, 0xee, 0xd0, 0x8f, 0x15}, Ver: 0x3}, - {Name: [8]uint8{0xf7, 0x8b, 0x27, 0x8b, 0xe5, 0x3f, 0x45, 0x4c}, Ver: 0x2}, - {Name: [8]uint8{0xaf, 0x2c, 0x2, 0x97, 0xa2, 0x3e, 0x6d, 0x3d}, Ver: 0x2}, - {Name: [8]uint8{0x49, 0xea, 0xaf, 0x1b, 0x54, 0x8a, 0xc, 0xb0}, Ver: 0x1}, - {Name: [8]uint8{0x91, 0xd5, 0xdf, 0x18, 0xb0, 0xd2, 0xcf, 0x58}, Ver: 0x1}, - {Name: [8]uint8{0xed, 0x99, 0xc5, 0xac, 0xb2, 0x5e, 0xed, 0xf5}, Ver: 0x3}, - {Name: [8]uint8{0xcb, 0xca, 0x25, 0xe3, 0x9f, 0x14, 0x23, 0x87}, Ver: 0x2}, - {Name: [8]uint8{0x68, 0x7a, 0xd4, 0x4a, 0xd3, 0x7f, 0x3, 0xc2}, Ver: 0x1}, - {Name: [8]uint8{0xab, 0x3c, 0x5, 0x72, 0x29, 0x1f, 0xeb, 0x8b}, Ver: 0x1}, - {Name: [8]uint8{0xbc, 0x9d, 0x89, 0x90, 0x4f, 0x5b, 0x92, 0x3f}, Ver: 0x1}, - {Name: [8]uint8{0x37, 0xc8, 0xbb, 0x13, 0x50, 0xa9, 0xa2, 0xa8}, Ver: 0x1}, - {Name: [8]uint8{0xf3, 0xff, 0x14, 0xd5, 0xab, 0x52, 0x70, 0x59}, Ver: 0x1}, - }, - TransactionVersion: 14, - }, - }, - "westend_v0929": { - instanceBuilder: func(t *testing.T) instanceVersioner { - return NewTestInstance(t, runtime.WESTEND_RUNTIME_v0929) - }, - expectedVersion: runtime.Version{ - SpecName: []byte("westend"), - ImplName: []byte("parity-westend"), - AuthoringVersion: 2, - SpecVersion: 9290, - ImplVersion: 0, - APIItems: []runtime.APIItem{ - {Name: [8]uint8{0xdf, 0x6a, 0xcb, 0x68, 0x99, 0x7, 0x60, 0x9b}, Ver: 0x4}, - {Name: [8]uint8{0x37, 0xe3, 0x97, 0xfc, 0x7c, 0x91, 0xf5, 0xe4}, Ver: 0x1}, - {Name: [8]uint8{0x40, 0xfe, 0x3a, 0xd4, 0x1, 0xf8, 0x95, 0x9a}, Ver: 0x6}, - {Name: [8]uint8{0xd2, 0xbc, 0x98, 0x97, 0xee, 0xd0, 0x8f, 0x15}, Ver: 0x3}, - {Name: [8]uint8{0xf7, 0x8b, 0x27, 0x8b, 0xe5, 0x3f, 0x45, 0x4c}, Ver: 0x2}, - {Name: [8]uint8{0xaf, 0x2c, 0x2, 0x97, 0xa2, 0x3e, 0x6d, 0x3d}, Ver: 0x2}, - {Name: [8]uint8{0x49, 0xea, 0xaf, 0x1b, 0x54, 0x8a, 0xc, 0xb0}, Ver: 0x1}, - {Name: [8]uint8{0x91, 0xd5, 0xdf, 0x18, 0xb0, 0xd2, 0xcf, 0x58}, Ver: 0x1}, - {Name: [8]uint8{0xed, 0x99, 0xc5, 0xac, 0xb2, 0x5e, 0xed, 0xf5}, Ver: 0x3}, - {Name: [8]uint8{0xcb, 0xca, 0x25, 0xe3, 0x9f, 0x14, 0x23, 0x87}, Ver: 0x2}, - {Name: [8]uint8{0x68, 0x7a, 0xd4, 0x4a, 0xd3, 0x7f, 0x3, 0xc2}, Ver: 0x1}, - {Name: [8]uint8{0xab, 0x3c, 0x5, 0x72, 0x29, 0x1f, 0xeb, 0x8b}, Ver: 0x1}, - {Name: [8]uint8{0xbc, 0x9d, 0x89, 0x90, 0x4f, 0x5b, 0x92, 0x3f}, Ver: 0x1}, - {Name: [8]uint8{0x37, 0xc8, 0xbb, 0x13, 0x50, 0xa9, 0xa2, 0xa8}, Ver: 0x1}, - {Name: [8]uint8{0xf3, 0xff, 0x14, 0xd5, 0xab, 0x52, 0x70, 0x59}, Ver: 0x1}, - {Name: [8]uint8{0x17, 0xa6, 0xbc, 0xd, 0x0, 0x62, 0xae, 0xb3}, Ver: 0x1}, - }, - TransactionVersion: 12, - }, - }, - } - - for name, testCase := range testCases { - testCase := testCase - t.Run(name, func(t *testing.T) { - t.Parallel() - - instance := testCase.instanceBuilder(t) - version, err := instance.Version() - require.NoError(t, err) - assert.Equal(t, testCase.expectedVersion, version) - }) - } -} - -func balanceKey(t *testing.T, pub []byte) []byte { - h0, err := common.Twox128Hash([]byte("System")) - require.NoError(t, err) - h1, err := common.Twox128Hash([]byte("Account")) - require.NoError(t, err) - h2, err := common.Blake2b128(pub) - require.NoError(t, err) - return append(append(append(h0, h1...), h2...), pub...) -} - -func TestWestendRuntime_ValidateTransaction(t *testing.T) { - genesisPath := utils.GetWestendDevRawGenesisPath(t) - gen := genesisFromRawJSON(t, genesisPath) - genTrie, err := runtime.NewTrieFromGenesis(gen) - require.NoError(t, err) - - // set state to genesis state - genState := storage.NewTrieState(&genTrie) - - cfg := Config{ - Storage: genState, - LogLvl: log.Critical, - } - - nodeStorage := runtime.NodeStorage{} - nodeStorage.BaseDB = runtime.NewInMemoryDB(t) - cfg.NodeStorage = nodeStorage - - rt, err := NewRuntimeFromGenesis(cfg) - require.NoError(t, err) - - alicePub := common.MustHexToBytes("0xd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d") - aliceBalanceKey := balanceKey(t, alicePub) - - accInfo := types.AccountInfo{ - Nonce: 0, - Data: types.AccountData{ - Free: scale.MustNewUint128(big.NewInt(1152921504606846976)), - Reserved: scale.MustNewUint128(big.NewInt(0)), - MiscFrozen: scale.MustNewUint128(big.NewInt(0)), - FreeFrozen: scale.MustNewUint128(big.NewInt(0)), - }, - } - - encBal, err := scale.Marshal(accInfo) - require.NoError(t, err) - - rt.ctx.Storage.Put(aliceBalanceKey, encBal, trie.V0) - // this key is System.UpgradedToDualRefCount -> set to true since all accounts have been upgraded to v0.9 format - rt.ctx.Storage.Put(common.UpgradedToDualRefKey, []byte{1}, trie.V0) - - genesisHeader := &types.Header{ - Number: 0, - StateRoot: genTrie.MustHash(), - } - - extHex := runtime.NewTestExtrinsic(t, rt, genesisHeader.Hash(), genesisHeader.Hash(), - 0, signature.TestKeyringPairAlice, "System.remark", []byte{0xab, 0xcd}) - - genesisHashBytes := genesisHeader.Hash().ToBytes() - - validateTransactionArguments := [][]byte{ - {byte(types.TxnExternal)}, - common.MustHexToBytes(extHex), - genesisHashBytes} - - extrinsicsBytes := bytes.Join(validateTransactionArguments, nil) - - runtime.InitializeRuntimeToTest(t, rt, genesisHeader) - _, err = rt.ValidateTransaction(extrinsicsBytes) - require.NoError(t, err) -} - -func TestInstance_GrandpaAuthorities_NodeRuntime(t *testing.T) { - tt := trie.NewEmptyTrie() - - value, err := common.HexToBytes("0x0108eea1eabcac7d2c8a6459b7322cf997874482bfc3d2ec7a80888a3a7d714103640100000000000000b64994460e59b30364cad3c92e3df6052f9b0ebbb8f88460c194dc5794d6d7170100000000000000") //nolint:lll - require.NoError(t, err) - - key := common.MustHexToBytes(genesis.GrandpaAuthoritiesKeyHex) - tt.Put(key, value, trie.V0) - - rt := NewTestInstanceWithTrie(t, runtime.WESTEND_RUNTIME_v0929, tt) - - auths, err := rt.GrandpaAuthorities() - require.NoError(t, err) - - authABytes, _ := common.HexToBytes("0xeea1eabcac7d2c8a6459b7322cf997874482bfc3d2ec7a80888a3a7d71410364") - authBBytes, _ := common.HexToBytes("0xb64994460e59b30364cad3c92e3df6052f9b0ebbb8f88460c194dc5794d6d717") - - authA, _ := ed25519.NewPublicKey(authABytes) - authB, _ := ed25519.NewPublicKey(authBBytes) - - expected := []types.Authority{ - {Key: authA, Weight: 1}, - {Key: authB, Weight: 1}, - } - - require.Equal(t, expected, auths) -} - -func TestInstance_GrandpaAuthorities_PolkadotRuntime(t *testing.T) { - tt := trie.NewEmptyTrie() - - value, err := common.HexToBytes("0x0108eea1eabcac7d2c8a6459b7322cf997874482bfc3d2ec7a80888a3a7d714103640100000000000000b64994460e59b30364cad3c92e3df6052f9b0ebbb8f88460c194dc5794d6d7170100000000000000") //nolint:lll - require.NoError(t, err) - - key := common.MustHexToBytes(genesis.GrandpaAuthoritiesKeyHex) - tt.Put(key, value, trie.V0) - - rt := NewTestInstanceWithTrie(t, runtime.POLKADOT_RUNTIME_v0929, tt) - - auths, err := rt.GrandpaAuthorities() - require.NoError(t, err) - - authABytes, _ := common.HexToBytes("0xeea1eabcac7d2c8a6459b7322cf997874482bfc3d2ec7a80888a3a7d71410364") - authBBytes, _ := common.HexToBytes("0xb64994460e59b30364cad3c92e3df6052f9b0ebbb8f88460c194dc5794d6d717") - - authA, _ := ed25519.NewPublicKey(authABytes) - authB, _ := ed25519.NewPublicKey(authBBytes) - - expected := []types.Authority{ - {Key: authA, Weight: 1}, - {Key: authB, Weight: 1}, - } - - require.Equal(t, expected, auths) -} - -func TestInstance_BabeGenerateKeyOwnershipProof(t *testing.T) { - t.Parallel() - - testCases := []struct { - name string - targetRuntime string - }{ - { - name: "with_polkadot_runtime", - targetRuntime: runtime.POLKADOT_RUNTIME_v0929, - }, - { - name: "with_westend_runtime", - targetRuntime: runtime.WESTEND_RUNTIME_v0929, - }, - } - for _, testCase := range testCases { - testCase := testCase - t.Run(testCase.name, func(t *testing.T) { - t.Parallel() - - tt := trie.NewEmptyTrie() - - randomnessValue, err := common.HexToHash("0x01") - require.NoError(t, err) - key := common.MustHexToBytes(genesis.BABERandomnessKeyHex) - tt.Put(key, randomnessValue[:], trie.V0) - - authorityValue, err := common.HexToBytes("0x08eea1eabcac7d2c8a6459b7322cf997874482bfc3d2ec7a80888a3a7d714103640100000000000000b64994460e59b30364cad3c92e3df6052f9b0ebbb8f88460c194dc5794d6d7170100000000000000") //nolint:lll - require.NoError(t, err) - - key = common.MustHexToBytes(genesis.BABEAuthoritiesKeyHex) - tt.Put(key, authorityValue, trie.V0) - - rt := NewTestInstanceWithTrie(t, testCase.targetRuntime, tt) - - babeConfig, err := rt.BabeConfiguration() - require.NoError(t, err) - - require.NotEmpty(t, babeConfig.GenesisAuthorities) - - authorityID := babeConfig.GenesisAuthorities[0].Key - - const slot = uint64(10) - res, err := rt.BabeGenerateKeyOwnershipProof(slot, authorityID) - require.NoError(t, err) - require.Nil(t, res) - }) - } -} - -func TestInstance_BabeSubmitReportEquivocationUnsignedExtrinsic(t *testing.T) { - t.Parallel() - - testCases := []struct { - name string - targetRuntime string - }{ - { - name: "with_polkadot_runtime", - targetRuntime: runtime.POLKADOT_RUNTIME_v0929, - }, - { - name: "with_westend_runtime", - targetRuntime: runtime.WESTEND_RUNTIME_v0929, - }, - } - for _, testCase := range testCases { - testCase := testCase - t.Run(testCase.name, func(t *testing.T) { - t.Parallel() - - tt := trie.NewEmptyTrie() - rt := NewTestInstanceWithTrie(t, testCase.targetRuntime, tt) - authorityID := types.AuthorityID{1} - const slot = uint64(1) - - keyOwnershipProof := testKeyOwnershipProof - - equivocationProof := types.BabeEquivocationProof{ - Offender: authorityID, - Slot: slot, - } - - err := rt.BabeSubmitReportEquivocationUnsignedExtrinsic(equivocationProof, keyOwnershipProof) - require.NoError(t, err) - }) - } -} - -func TestInstance_BabeConfiguration_WestendRuntime_NoAuthorities(t *testing.T) { - rt := NewTestInstance(t, runtime.WESTEND_RUNTIME_v0929) - cfg, err := rt.BabeConfiguration() - require.NoError(t, err) - - expected := &types.BabeConfiguration{ - SlotDuration: 6000, - EpochLength: 600, - C1: 1, - C2: 4, - GenesisAuthorities: nil, - Randomness: [32]byte{}, - SecondarySlots: 2, - } - require.Equal(t, expected, cfg) -} - -func TestInstance_BabeConfiguration_WestendRuntime_WithAuthorities(t *testing.T) { - tt := trie.NewEmptyTrie() - - randomnessValue, err := common.HexToHash("0x01") - require.NoError(t, err) - key := common.MustHexToBytes(genesis.BABERandomnessKeyHex) - tt.Put(key, randomnessValue[:], trie.V0) - - authorityValue, err := common.HexToBytes("0x08eea1eabcac7d2c8a6459b7322cf997874482bfc3d2ec7a80888a3a7d714103640100000000000000b64994460e59b30364cad3c92e3df6052f9b0ebbb8f88460c194dc5794d6d7170100000000000000") //nolint:lll - require.NoError(t, err) - - key = common.MustHexToBytes(genesis.BABEAuthoritiesKeyHex) - tt.Put(key, authorityValue, trie.V0) - - rt := NewTestInstanceWithTrie(t, runtime.WESTEND_RUNTIME_v0929, tt) - - cfg, err := rt.BabeConfiguration() - require.NoError(t, err) - - authA, _ := common.HexToHash("0xeea1eabcac7d2c8a6459b7322cf997874482bfc3d2ec7a80888a3a7d71410364") - authB, _ := common.HexToHash("0xb64994460e59b30364cad3c92e3df6052f9b0ebbb8f88460c194dc5794d6d717") - - expectedAuthData := []types.AuthorityRaw{ - {Key: authA, Weight: 1}, - {Key: authB, Weight: 1}, - } - - expected := &types.BabeConfiguration{ - SlotDuration: 6000, - EpochLength: 600, - C1: 1, - C2: 4, - GenesisAuthorities: expectedAuthData, - Randomness: [32]byte{1}, - SecondarySlots: 2, - } - - require.Equal(t, expected, cfg) -} - -func TestInstance_InitializeBlock_NodeRuntime(t *testing.T) { - rt := NewTestInstance(t, runtime.WESTEND_RUNTIME_v0929) - - header := &types.Header{ - Number: 1, - Digest: types.NewDigest(), - } - - err := rt.InitializeBlock(header) - require.NoError(t, err) -} - -func TestInstance_InitializeBlock_PolkadotRuntime(t *testing.T) { - rt := NewTestInstance(t, runtime.POLKADOT_RUNTIME_v0929) - - header := &types.Header{ - Number: 1, - Digest: types.NewDigest(), - } - - err := rt.InitializeBlock(header) - require.NoError(t, err) -} - -func TestInstance_ExecuteBlock_WestendRuntime(t *testing.T) { - instance := NewTestInstance(t, runtime.WESTEND_RUNTIME_v0929) - block := runtime.InitializeRuntimeToTest(t, instance, &types.Header{}) - - // reset state back to parent state before executing - parentState := storage.NewTrieState(nil) - instance.SetContextStorage(parentState) - - _, err := instance.ExecuteBlock(block) - require.NoError(t, err) -} - -func TestInstance_ApplyExtrinsic_WestendRuntime(t *testing.T) { - genesisPath := utils.GetWestendDevRawGenesisPath(t) - gen := genesisFromRawJSON(t, genesisPath) - genTrie, err := runtime.NewTrieFromGenesis(gen) - require.NoError(t, err) - - // set state to genesis state - genState := storage.NewTrieState(&genTrie) - - cfg := Config{ - Storage: genState, - LogLvl: log.Critical, - } - - instance, err := NewRuntimeFromGenesis(cfg) - require.NoError(t, err) - - // reset state back to parent state before executing - parentState := storage.NewTrieState(&genTrie) - instance.SetContextStorage(parentState) - - genesisHeader := &types.Header{ - Number: 0, - StateRoot: genTrie.MustHash(), - } - header := &types.Header{ - ParentHash: genesisHeader.Hash(), - Number: 1, - Digest: types.NewDigest(), - } - - err = instance.InitializeBlock(header) - require.NoError(t, err) - - extHex := runtime.NewTestExtrinsic(t, instance, genesisHeader.Hash(), genesisHeader.Hash(), - 0, signature.TestKeyringPairAlice, "System.remark", []byte{0xab, 0xcd}) - - res, err := instance.ApplyExtrinsic(common.MustHexToBytes(extHex)) - require.NoError(t, err) - require.Equal(t, []byte{0, 0}, res) -} - -func TestInstance_ExecuteBlock_PolkadotRuntime(t *testing.T) { - DefaultTestLogLvl = 0 - - instance := NewTestInstance(t, runtime.POLKADOT_RUNTIME_v0929) - - block := runtime.InitializeRuntimeToTest(t, instance, &types.Header{}) - - // reset state back to parent state before executing - parentState := storage.NewTrieState(nil) - instance.SetContextStorage(parentState) - - _, err := instance.ExecuteBlock(block) - require.NoError(t, err) -} - -func TestInstance_ExecuteBlock_PolkadotRuntime_PolkadotBlock1(t *testing.T) { - genesisPath := utils.GetPolkadotGenesisPath(t) - gen := genesisFromRawJSON(t, genesisPath) - genTrie, err := runtime.NewTrieFromGenesis(gen) - require.NoError(t, err) - - expectedGenesisRoot := common.MustHexToHash("0x29d0d972cd27cbc511e9589fcb7a4506d5eb6a9e8df205f00472e5ab354a4e17") - require.Equal(t, expectedGenesisRoot, genTrie.MustHash()) - - // set state to genesis state - genState := storage.NewTrieState(&genTrie) - - cfg := Config{ - Storage: genState, - LogLvl: log.Critical, - } - - instance, err := NewRuntimeFromGenesis(cfg) - require.NoError(t, err) - - // block data is received from querying a polkadot node - body := []byte{8, 40, 4, 3, 0, 11, 80, 149, 160, 81, 114, 1, 16, 4, 20, 0, 0} - var exts [][]byte - err = scale.Unmarshal(body, &exts) - require.NoError(t, err) - require.Equal(t, 2, len(exts)) - - // digest data received from querying polkadot node - digestBytes := common.MustHexToBytes("0x0c0642414245b501010000000093decc0f00000000362ed8d6055645487fe42e9c8640be651f70a3a2a03658046b2b43f021665704501af9b1ca6e974c257e3d26609b5f68b5b0a1da53f7f252bbe5d94948c39705c98ffa4b869dd44ac29528e3723d619cc7edf1d3f7b7a57a957f6a7e9bdb270a044241424549040118fa3437b10f6e7af8f31362df3a179b991a8c56313d1bcd6307a4d0c734c1ae310100000000000000d2419bc8835493ac89eb09d5985281f5dff4bc6c7a7ea988fd23af05f301580a0100000000000000ccb6bef60defc30724545d57440394ed1c71ea7ee6d880ed0e79871a05b5e40601000000000000005e67b64cf07d4d258a47df63835121423551712844f5b67de68e36bb9a21e12701000000000000006236877b05370265640c133fec07e64d7ca823db1dc56f2d3584b3d7c0f1615801000000000000006c52d02d95c30aa567fda284acf25025ca7470f0b0c516ddf94475a1807c4d250100000000000000000000000000000000000000000000000000000000000000000000000000000005424142450101d468680c844b19194d4dfbdc6697a35bf2b494bda2c5a6961d4d4eacfbf74574379ba0d97b5bb650c2e8670a63791a727943bcb699dc7a228bdb9e0a98c9d089") //nolint:lll - - digest := types.NewDigest() - err = scale.Unmarshal(digestBytes, &digest) - require.NoError(t, err) - - // polkadot block 1, from polkadot.js - block := &types.Block{ - Header: types.Header{ - ParentHash: common.MustHexToHash("0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3"), - Number: 1, - StateRoot: common.MustHexToHash("0xc56fcd6e7a757926ace3e1ecff9b4010fc78b90d459202a339266a7f6360002f"), - ExtrinsicsRoot: common.MustHexToHash("0x9a87f6af64ef97aff2d31bebfdd59f8fe2ef6019278b634b2515a38f1c4c2420"), - Digest: digest, - }, - Body: *types.NewBody(types.BytesArrayToExtrinsics(exts)), - } - - _, err = instance.ExecuteBlock(block) - require.NoError(t, err) -} - -func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1(t *testing.T) { - genesisPath := utils.GetKusamaGenesisPath(t) - gen := genesisFromRawJSON(t, genesisPath) - genTrie, err := runtime.NewTrieFromGenesis(gen) - require.NoError(t, err) - - expectedGenesisRoot := common.MustHexToHash("0xb0006203c3a6e6bd2c6a17b1d4ae8ca49a31da0f4579da950b127774b44aef6b") - require.Equal(t, expectedGenesisRoot, genTrie.MustHash()) - - // set state to genesis state - genState := storage.NewTrieState(&genTrie) - - cfg := Config{ - Storage: genState, - LogLvl: log.Critical, - } - - instance, err := NewRuntimeFromGenesis(cfg) - require.NoError(t, err) - - // block data is received from querying a polkadot node - body := []byte{8, 40, 4, 2, 0, 11, 144, 17, 14, 179, 110, 1, 16, 4, 20, 0, 0} - var exts [][]byte - err = scale.Unmarshal(body, &exts) - require.NoError(t, err) - require.Equal(t, 2, len(exts)) - - // digest from polkadot.js - digestBytes := common.MustHexToBytes("0x0c0642414245340201000000ef55a50f00000000044241424549040118ca239392960473fe1bc65f94ee27d890a49c1b200c006ff5dcc525330ecc16770100000000000000b46f01874ce7abbb5220e8fd89bede0adad14c73039d91e28e881823433e723f0100000000000000d684d9176d6eb69887540c9a89fa6097adea82fc4b0ff26d1062b488f352e179010000000000000068195a71bdde49117a616424bdc60a1733e96acb1da5aeab5d268cf2a572e94101000000000000001a0575ef4ae24bdfd31f4cb5bd61239ae67c12d4e64ae51ac756044aa6ad8200010000000000000018168f2aad0081a25728961ee00627cfe35e39833c805016632bf7c14da5800901000000000000000000000000000000000000000000000000000000000000000000000000000000054241424501014625284883e564bc1e4063f5ea2b49846cdddaa3761d04f543b698c1c3ee935c40d25b869247c36c6b8a8cbbd7bb2768f560ab7c276df3c62df357a7e3b1ec8d") //nolint:lll - - digest := types.NewDigest() - err = scale.Unmarshal(digestBytes, &digest) - require.NoError(t, err) - - // kusama block 1, from polkadot.js - block := &types.Block{ - Header: types.Header{ - ParentHash: common.MustHexToHash("0xb0a8d493285c2df73290dfb7e61f870f17b41801197a149ca93654499ea3dafe"), - Number: 1, - StateRoot: common.MustHexToHash("0xfabb0c6e92d29e8bb2167f3c6fb0ddeb956a4278a3cf853661af74a076fc9cb7"), - ExtrinsicsRoot: common.MustHexToHash("0xa35fb7f7616f5c979d48222b3d2fa7cb2331ef73954726714d91ca945cc34fd8"), - Digest: digest, - }, - Body: *types.NewBody(types.BytesArrayToExtrinsics(exts)), - } - - _, err = instance.ExecuteBlock(block) - require.NoError(t, err) -} - -func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock3784(t *testing.T) { - gossTrie3783 := newTrieFromPairs(t, "../test_data/kusama/block3783.out") - expectedRoot := common.MustHexToHash("0x948338bc0976aee78879d559a1f42385407e5a481b05a91d2a9386aa7507e7a0") - require.Equal(t, expectedRoot, gossTrie3783.MustHash()) - - // set state to genesis state - state3783 := storage.NewTrieState(gossTrie3783) - - cfg := Config{ - Storage: state3783, - LogLvl: log.Critical, - } - - instance, err := NewInstanceFromTrie(gossTrie3783, cfg) - require.NoError(t, err) - - // block data is received from querying a polkadot node - body := common.MustHexToBytes("0x10280402000bb00d69b46e0114040900193b10041400009101041300eaaec5728cd6ea9160ff92a49bb45972c532d2163241746134726aaa5b2f72129d8650715320f23765c6306503669f69bf684b188dea73b1e247dd1dd166513b1c13daa387c35f24ac918d2fa772b73cffd20204a8875e48a1b11bb3229deb7f00") //nolint:lll - var exts [][]byte - err = scale.Unmarshal(body, &exts) - require.NoError(t, err) - require.Equal(t, 4, len(exts)) - - // digest from polkadot.js - digestBytes := common.MustHexToBytes("0x080642414245340203000000bd64a50f0000000005424142450101bc0d6850dba8d32ea1dbe26cb4ac56da6cca662c7cc642dc8eed32d2bddd65029f0721436eafeebdf9b4f17d1673c6bc6c3c51fe3dda3121a5fc60c657a5808b") //nolint:lll - - digest := types.NewDigest() - err = scale.Unmarshal(digestBytes, &digest) - require.NoError(t, err) - - // kusama block 3784, from polkadot.js - block := &types.Block{ - Header: types.Header{ - ParentHash: common.MustHexToHash("0x4843b4aa38cf2e3e2f6fae401b98dd705bed668a82dd3751dc38f1601c814ca8"), - Number: 3784, - StateRoot: common.MustHexToHash("0xac44cc18ec22f0f3fca39dfe8725c0383af1c982a833e081fbb2540e46eb09a5"), - ExtrinsicsRoot: common.MustHexToHash("0x52b7d4852fc648cb8f908901e1e36269593c25050c31718454bca74b69115d12"), - Digest: digest, - }, - Body: *types.NewBody(types.BytesArrayToExtrinsics(exts)), - } - - _, err = instance.ExecuteBlock(block) - require.NoError(t, err) -} - -func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock901442(t *testing.T) { - ksmTrie901441 := newTrieFromPairs(t, "../test_data/kusama/block901441.out") - expectedRoot := common.MustHexToHash("0x3a2ef7ee032f5810160bb8f3ffe3e3377bb6f2769ee9f79a5425973347acd504") - require.Equal(t, expectedRoot, ksmTrie901441.MustHash()) - - // set state to genesis state - state901441 := storage.NewTrieState(ksmTrie901441) - - cfg := Config{ - Storage: state901441, - LogLvl: log.Critical, - } - - instance, err := NewInstanceFromTrie(ksmTrie901441, cfg) - require.NoError(t, err) - - body := common.MustHexToBytes("0x0c280402000b207eb80a70011c040900fa0437001004140000") - var exts [][]byte - err = scale.Unmarshal(body, &exts) - require.NoError(t, err) - require.Equal(t, 3, len(exts)) - - // digest from polkadot.js - digestBytes := common.MustHexToBytes("0x080642414245340244000000aeffb30f00000000054241424501011cbef2a084a774c34d9990c7bfc6b4d2d5e9f5b59feca792cd2bb89a890c2a6f09668b5e8224879f007f49f299d25fbb3c0f30d94fb8055e07fa8a4ed10f8083") //nolint:lll - - digest := types.NewDigest() - err = scale.Unmarshal(digestBytes, &digest) - require.NoError(t, err) - require.Equal(t, 2, len(digest.Types)) - - // kusama block 901442, from polkadot.js - block := &types.Block{ - Header: types.Header{ - ParentHash: common.MustHexToHash("0x68d9c5f75225f09d7ce493eff8aabac7bae8b65cb81a2fd532a99fbb8c663931"), - Number: 901442, - StateRoot: common.MustHexToHash("0x6ea065f850894c5b58cb1a73ec887e56842851943641149c57cea357cae4f596"), - ExtrinsicsRoot: common.MustHexToHash("0x13483a4c148fff5f072e86b5af52bf031556514e9c87ea19f9e31e7b13c0c414"), - Digest: digest, - }, - Body: *types.NewBody(types.BytesArrayToExtrinsics(exts)), - } - - _, err = instance.ExecuteBlock(block) - require.NoError(t, err) -} - -func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1377831(t *testing.T) { - ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block1377830.out") - expectedRoot := common.MustHexToHash("0xe4de6fecda9e9e35f937d159665cf984bc1a68048b6c78912de0aeb6bd7f7e99") - require.Equal(t, expectedRoot, ksmTrie.MustHash()) - - // set state to genesis state - state := storage.NewTrieState(ksmTrie) - - cfg := Config{ - Storage: state, - LogLvl: log.Critical, - } - - instance, err := NewInstanceFromTrie(ksmTrie, cfg) - require.NoError(t, err) - - body := common.MustHexToBytes("0x08280402000b60c241c070011004140000") - var exts [][]byte - err = scale.Unmarshal(body, &exts) - require.NoError(t, err) - require.Equal(t, 2, len(exts)) - - // digest from polkadot.js - digestBytes := common.MustHexToBytes("0x080642414245b50101020000008abebb0f00000000045553c32a949242580161bcc35d7c3e492e66defdcf4525d7a338039590012f42660acabf1952a2d5d01725601705404d6ac671507a6aa2cf09840afbdfbb006f48062dae16c56b8dc5c6ea6ffba854b7e8f46e153e98c238cbe7bbb1556f0b0542414245010136914c6832dd5ba811a975a3b654d76a1ec81684f4b03d115ce2e694feadc96411930438fde4beb008c5f8e26cfa2f5b554fa3814b5b73d31f348446fd4fd688") //nolint:lll - - digest := types.NewDigest() - err = scale.Unmarshal(digestBytes, &digest) - require.NoError(t, err) - require.Equal(t, 2, len(digest.Types)) - - // kusama block 1377831, from polkadot.js - block := &types.Block{ - Header: types.Header{ - ParentHash: common.MustHexToHash("0xca387b3cc045e8848277069d8794cbf077b08218c0b55f74d81dd750b14e768c"), - Number: 1377831, - StateRoot: common.MustHexToHash("0x7e5569e652c4b1a3cecfcf5e5e64a97fe55071d34bab51e25626ec20cae05a02"), - ExtrinsicsRoot: common.MustHexToHash("0x7f3ea0ed63b4053d9b75e7ee3e5b3f6ce916e8f59b7b6c5e966b7a56ea0a563a"), - Digest: digest, - }, - Body: *types.NewBody(types.BytesArrayToExtrinsics(exts)), - } - - _, err = instance.ExecuteBlock(block) - require.NoError(t, err) -} - -func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1482003(t *testing.T) { - ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block1482002.out") - expectedRoot := common.MustHexToHash("0x09f9ca28df0560c2291aa16b56e15e07d1e1927088f51356d522722aa90ca7cb") - require.Equal(t, expectedRoot, ksmTrie.MustHash()) - - // set state to genesis state - state := storage.NewTrieState(ksmTrie) - - cfg := Config{ - Storage: state, - LogLvl: log.Critical, - } - - instance, err := NewInstanceFromTrie(ksmTrie, cfg) - require.NoError(t, err) - - body := common.MustHexToBytes("0x0c280402000b10c3e3e570011c04090042745a001004140000") - var exts [][]byte - err = scale.Unmarshal(body, &exts) - require.NoError(t, err) - require.Equal(t, 3, len(exts)) - - // digest from polkadot.js - digestBytes := testdata.DigestKusama1482002(t) - - digest := types.NewDigest() - err = scale.Unmarshal(digestBytes, &digest) - require.NoError(t, err) - - require.Equal(t, 4, len(digest.Types)) - - // kusama block 1482003, from polkadot.js - block := &types.Block{ - Header: types.Header{ - ParentHash: common.MustHexToHash("0x587f6da1bfa71a675f10dfa0f63edfcf168e8ece97eb5f526aaf0e8a8e82db3f"), - Number: 1482003, - StateRoot: common.MustHexToHash("0xd2de750002f33968437bdd54912dd4f55c3bddc5a391a8e0b8332568e1efea8d"), - ExtrinsicsRoot: common.MustHexToHash("0xdf5da95780b77e83ad0bf820d5838f07a0d5131aa95a75f8dfbd01fbccb300bd"), - Digest: digest, - }, - Body: *types.NewBody(types.BytesArrayToExtrinsics(exts)), - } - - _, err = instance.ExecuteBlock(block) - require.NoError(t, err) -} - -func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock4939774(t *testing.T) { - t.Skip("skip for now as block4939773 is too large") - ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block4939773.out") - expectedRoot := common.MustHexToHash("0xc45748e6e8632b44fc32b04cc4380098a9584cbd63ffbc59adce189574fc36fe") - require.Equal(t, expectedRoot, ksmTrie.MustHash()) - - // set state to genesis state - state := storage.NewTrieState(ksmTrie) - - cfg := Config{ - Storage: state, - LogLvl: log.Critical, - } - - instance, err := NewInstanceFromTrie(ksmTrie, cfg) - require.NoError(t, err) - - body := common.MustHexToBytes("0x08280402000b80eb3cd17501710984c2292bcf6f34fc2d25f7a1ebaec41c3239536f12f75417c73f7c5aca53308668016ec90c2318ee45af373755527436c4d7a257c481fdc3214634eb4b5c6711ae181827c378843da82c72191647667607ee97e0f0335f14d0876c63503b5f2b8986650304001f010200083e1f2bfd408d3b8d2266ce9b6f2d40acef27b773414537be72576ee3e6108b256eb45e26258d7ac737c3ad3af8cd1b2208d45c472ba19ebfc3e2fb834a6e904d01de574b00010000007506180228040052dac5497bbdd42583d07aa46102790d54aacdcbfac8877189e3b609117a29150b00a0724e180904001cf8853df87ca8588405e30c46a434d636c86561b955b09e2e9b27fc296bf4290b005039278c040400f49db9c8894863a7dd213be93b1c440b145cc19d4927b4c29fe5fa25e8a1667f0b005039278c040400e05f031d874257a24232076830a073a6af6851c07735de201edfc412ca8853180b005039278c0404009289e88ec986066d04f7d93d80f7a3c9794580b5e59d2a7af6b19745dd148f6f0b005039278c0404006c8aff52c496b64b476ca22e58fc54822b435abbbbcaf0c9dd7cf1ab573227790b005039278c04040044e31f7c4afa3b055696923ccb405da2ee2d9eefccf568aa3c6855dbff573e5f0b005039278c040400469ec0f872af2503a9251666fd089d0e84d3f6c8b761ee94b0e868788e0f60500b005039278c040400b41cc00e4ee2945ce9974dbb355265e39c9cf325c176147d7f6b1631af38ce590b005039278c040400d8e2f26a12d4bfc513fd32c1e5a7f14e930c3ef37997bf4e3de2fed51eed515a0b005039278c040048227b8300000000") //nolint:lll - var exts [][]byte - err = scale.Unmarshal(body, &exts) - require.NoError(t, err) - require.Equal(t, 2, len(exts)) - - digestBytes := common.MustHexToBytes("0x080642414245b50101ef0100000815f30f000000004014ed1a99f017ea2c0d879d7317f51106938f879b296ff92c64319c0c70fe453d72035395da8d53e885def26e63cf90461ee549d0864f9691a4f401b31c1801730c014bc0641b307e8a30692e7d074b4656993b40d6f08698bc49dea40c11090542414245010192ed24972a8108b9bad1a8785b443efe72d4bc2069ab40eac65519fb01ff04250f44f6202d30ca88c30fee385bc8d7f51df15dddacf4e5d53788d260ce758c89") //nolint:lll - digest := types.NewDigest() - err = scale.Unmarshal(digestBytes, &digest) - require.NoError(t, err) - require.Equal(t, 2, len(digest.Types)) - - block := &types.Block{ - Header: types.Header{ - ParentHash: common.MustHexToHash("0xac08290f49cb9760a3a4c5a49351af76ba9432add29178e5cc27d4451f9126c9"), - Number: 4939774, - StateRoot: common.MustHexToHash("0x5d66f43cdbf1740b8ca41f0cd016602f1648fb08b74fe49f5f078845071d0a54"), - ExtrinsicsRoot: common.MustHexToHash("0x5d887e118ee6320aca38e49cbd98adc25472c6efbf77a695ab0d6c476a4ec6e9"), - Digest: digest, - }, - Body: *types.NewBody(types.BytesArrayToExtrinsics(exts)), - } - - _, err = instance.ExecuteBlock(block) - require.NoError(t, err) -} - -func TestInstance_ExecuteBlock_PolkadotBlock1089328(t *testing.T) { - dotTrie := newTrieFromPairs(t, "../test_data/polkadot/block1089327.json") - expectedRoot := common.MustHexToHash("0x87ed9ebe7fb645d3b5b0255cc16e78ed022d9fbb52486105436e15a74557535b") - require.Equal(t, expectedRoot, dotTrie.MustHash()) - - // set state to genesis state - state := storage.NewTrieState(dotTrie) - - cfg := Config{ - Storage: state, - LogLvl: log.Critical, - } - - instance, err := NewInstanceFromTrie(dotTrie, cfg) - require.NoError(t, err) - - body := common.MustHexToBytes("0x0c280403000be02ab6d873011004140000b90384468e34dbdcc8da24e44b0f0d34d97ccad5ce0281e465db0cc1d8e1423d50d90a018a89185c693f77b050fa35d1f80b19608b72a6e626110e835caedf949668a12b0ad7b786accf2caac0ec874941ccea9825d50b6bb5870e1400f0e56bb4c18b87a5021501001d00862e432e0cf75693899c62691ac0f48967f815add97ae85659dcde8332708551001b000cf4da8aea0e5649a8bedbc1f08e8a8c0febe50cd5b1c9ce0da2164f19aef40f01014a87a7d3673e5c80aec79973682140828a0d1c3899f4f3cc953bd02673e11a022aaa4f269e3f1a90156db29df88f780b1540b610aeb5cd347ee703c5dff48485") //nolint:lll - var exts [][]byte - err = scale.Unmarshal(body, &exts) - require.NoError(t, err) - require.Equal(t, 3, len(exts)) - - // digest from polkadot.js - digestBytes := common.MustHexToBytes("0x080642414245b501017b000000428edd0f00000000c4fd75c7535d8eec375d70d21cc62262247b599aa67d8a9cf2f7d1b8cb93cd1f9539f04902c33d4c0fe47f723dfed8505d31de1c04d0036a9df233ff902fce0d70060908faa4b3f481e54cbd6a52dfc20c3faac82f746d84dc03c2f824a89a0d0542414245010122041949669a56c8f11b3e3e7c803e477ad24a71ed887bc81c956b59ea8f2b30122e6042494aab60a75e0db8fdff45951e456e6053bd64eb5722600e4a13038b") //nolint:lll - - digest := types.NewDigest() - err = scale.Unmarshal(digestBytes, &digest) - require.NoError(t, err) - require.Equal(t, 2, len(digest.Types)) - - block := &types.Block{ - Header: types.Header{ - ParentHash: common.MustHexToHash("0x21dc35454805411be396debf3e1d5aad8d6e9d0d7679cce0cc632ba8a647d07c"), - Number: 1089328, - StateRoot: common.MustHexToHash("0x257b1a7f6bc0287fcbf50676dd29817f2f7ae193cb65b31962e351917406fa23"), - ExtrinsicsRoot: common.MustHexToHash("0x950173af1d9fdcd0be5428fc3eaf05d5f34376bd3882d9a61b348fa2dc641012"), - Digest: digest, - }, - Body: *types.NewBody(types.BytesArrayToExtrinsics(exts)), - } - - _, err = instance.ExecuteBlock(block) - require.NoError(t, err) -} - -func TestInstance_DecodeSessionKeys(t *testing.T) { - keys := "0xd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d34309a9d2a24213896ff06895db16aade8b6502f3a71cf56374cc3852042602634309a9d2a24213896ff06895db16aade8b6502f3a71cf56374cc3852042602634309a9d2a24213896ff06895db16aade8b6502f3a71cf56374cc3852042602634309a9d2a24213896ff06895db16aade8b6502f3a71cf56374cc3852042602634309a9d2a24213896ff06895db16aade8b6502f3a71cf56374cc38520426026" //nolint:lll - pubkeys, err := common.HexToBytes(keys) - require.NoError(t, err) - - pukeysBytes, err := scale.Marshal(pubkeys) - require.NoError(t, err) - - instance := NewTestInstance(t, runtime.WESTEND_RUNTIME_v0929) - decoded, err := instance.DecodeSessionKeys(pukeysBytes) - require.NoError(t, err) - - var decodedKeys *[]struct { - Data []uint8 - Type [4]uint8 - } - - err = scale.Unmarshal(decoded, &decodedKeys) - require.NoError(t, err) - - require.NotNil(t, decodedKeys) - require.Len(t, *decodedKeys, 6) -} - -func TestInstance_PaymentQueryInfo(t *testing.T) { - tests := []struct { - extB []byte - ext string - errMessage string - expect *types.RuntimeDispatchInfo - }{ - { - // Was made with @polkadot/api on https://github.com/danforbes/polkadot-js-scripts/tree/create-signed-tx - ext: "0xd1018400d43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d01bc2b6e35929aabd5b8bc4e5b0168c9bee59e2bb9d6098769f6683ecf73e44c776652d947a270d59f3d37eb9f9c8c17ec1b4cc473f2f9928ffdeef0f3abd43e85d502000000012844616e20466f72626573", //nolint:lll - expect: &types.RuntimeDispatchInfo{ - Weight: 0, - Class: 0, - PartialFee: &scale.Uint128{ - Upper: 0, - Lower: uint64(12800000000), - }, - }, - }, - { - // incomplete extrinsic - ext: "0x4ccde39a5684e7a56da23b22d4d9fbadb023baa19c56495432884d0640000000000000000000000000000000", - errMessage: "running runtime function: " + - "Failed to call the `TransactionPaymentApi_query_info` exported function.", - }, - { - // incomplete extrinsic - extB: nil, - errMessage: "running runtime function: " + - "Failed to call the `TransactionPaymentApi_query_info` exported function.", - }, - } - - for _, test := range tests { - var err error - var extBytes []byte - - if test.ext == "" { - extBytes = test.extB - } else { - extBytes, err = common.HexToBytes(test.ext) - require.NoError(t, err) - } - - ins := NewTestInstance(t, runtime.WESTEND_RUNTIME_v0929) - info, err := ins.PaymentQueryInfo(extBytes) - - if test.errMessage != "" { - assert.EqualError(t, err, test.errMessage) - continue - } - require.NoError(t, err) - - require.NoError(t, err) - require.NotNil(t, info) - require.Equal(t, test.expect, info) - } -} - -func newTrieFromPairs(t *testing.T, filename string) *trie.Trie { - data, err := os.ReadFile(filename) - require.NoError(t, err) - - rpcPairs := make(map[string]interface{}) - err = json.Unmarshal(data, &rpcPairs) - require.NoError(t, err) - pairs := rpcPairs["result"].([]interface{}) - - entries := make(map[string]string) - for _, pair := range pairs { - pairArr := pair.([]interface{}) - entries[pairArr[0].(string)] = pairArr[1].(string) - } - - // We need to use V0 for all tests using this function - tr, err := trie.LoadFromMap(entries, trie.V0) - require.NoError(t, err) - return &tr -} - -func TestInstance_TransactionPaymentCallApi_QueryCallInfo(t *testing.T) { - t.Parallel() - ins := NewTestInstance(t, runtime.WESTEND_RUNTIME_v0929) - tests := []struct { - callHex string - errMessage string - expect *types.RuntimeDispatchInfo - }{ - { - // call generated by using palkadot.js/api v9.5.1: api.tx.system.remark("Ed") - // and removing first byte (encoding) and second byte (unknown) - callHex: "0x0001084564", - expect: &types.RuntimeDispatchInfo{ - Weight: 0, - Class: 0, - PartialFee: &scale.Uint128{ - Upper: 0, - Lower: uint64(1500000000), - }, - }, - }, - { - // call removing encoding (first byte), polkadot.js/api v9.5.1: api.tx.system.remark("Ed") - // polkadot.js/api returns error: RPC-CORE: call(method: Text, data: Bytes, at?: BlockHash): - // Bytes:: -32000: Client error: Execution failed: Execution aborted due to trap: wasm trap: wasm - //`unreachable` instruction executed - callHex: "0x040001084564", - errMessage: "running runtime function: " + - "Failed to call the `TransactionPaymentCallApi_query_call_info` exported function.", - }, - { - // call without removing any bytes, polkadot.js/api v9.5.1: api.tx.system.remark("Ed test") - // polkadot.js/api returns error: Error: createType(Call):: findMetaCall: Unable to find Call with index - // [44, 4]/[44,4] - callHex: "0x2c0400011c45642074657374", - errMessage: "running runtime function: " + - "Failed to call the `TransactionPaymentCallApi_query_call_info` exported function.", - }, - } - - for _, test := range tests { - var err error - var callBytes []byte - - callBytes, err = common.HexToBytes(test.callHex) - require.NoError(t, err) - - info, err := ins.QueryCallInfo(callBytes) - - if test.errMessage != "" { - assert.EqualError(t, err, test.errMessage) - continue - } - - require.NoError(t, err) - require.NotNil(t, info) - require.Equal(t, test.expect, info) - } -} - -func TestInstance_TransactionPaymentCallApi_QueryCallFeeDetails(t *testing.T) { - t.Parallel() - ins := NewTestInstance(t, runtime.WESTEND_RUNTIME_v0929) - tests := []struct { - callHex string - errMessage string - expect *types.FeeDetails - }{ - { - // call generated by using palkadot.js/api v9.5.1: api.tx.system.remark("Ed") - // and removing first byte (encoding) and second byte (unknown) - callHex: "0x0001084564", - expect: &types.FeeDetails{ - InclusionFee: types.InclusionFee{ - BaseFee: &scale.Uint128{ - Upper: 0, - Lower: uint64(256000000001), - }, - LenFee: &scale.Uint128{ - Upper: 0, - Lower: uint64(128000000000), - }, - AdjustedWeightFee: &scale.Uint128{}, - }, - Tip: &scale.Uint128{}, - }, - }, - { - // call removing encoding (first byte), polkadot.js/api v9.5.1: api.tx.system.remark("Ed") - // when calling polkadot node (v0.9.29) with polkadot.js/api the node returns error: RPC-CORE: call( - // method: Text, data: Bytes, at?: BlockHash): Bytes:: -32000: Client error: Execution failed: - // Execution aborted due to trap: wasm trap: wasm `unreachable` instruction executed - callHex: "0x040001084564", - errMessage: "running runtime function: " + - "Failed to call the `TransactionPaymentCallApi_query_call_fee_details` exported function.", - }, - { - // call without removing any bytes, polkadot.js/api v9.5.1: api.tx.system.remark("Ed test") - // when calling polkadot (v0.9.29) with polkadot.js/api the node returns error: Error: createType( - //Call):: findMetaCall: Unable to find Call with index [44, 4]/[44,4] - callHex: "0x18040001084564", - errMessage: "running runtime function: " + - "Failed to call the `TransactionPaymentCallApi_query_call_fee_details` exported function.", - }, - } - - for _, test := range tests { - extBytes, err := common.HexToBytes(test.callHex) - require.NoError(t, err) - - details, err := ins.QueryCallFeeDetails(extBytes) - - if test.errMessage != "" { - assert.EqualError(t, err, test.errMessage) - continue - } - - require.NoError(t, err) - require.NotNil(t, details) - require.Equal(t, test.expect, details) - } -} - -func TestInstance_GrandpaGenerateKeyOwnershipProof(t *testing.T) { - t.Parallel() - instance := NewTestInstance(t, runtime.WESTEND_RUNTIME_v0929) - identity := common.MustHexToBytes("0x88dc3417d5058ec4b4503e0c12ea1a0a89be200fe98922423d4334014fa6b0ee") - identityPubKey, _ := ed25519.NewPublicKey(identity) - authorityID := identityPubKey.AsBytes() - - opaqueKeyOwnershipProof, err := instance.GrandpaGenerateKeyOwnershipProof(uint64(0), authorityID) - // Since the input is not valid with respect to the instance, an empty proof is returned - require.NoError(t, err) - require.Nil(t, opaqueKeyOwnershipProof) -} - -func TestInstance_GrandpaSubmitReportEquivocationUnsignedExtrinsic(t *testing.T) { - t.Parallel() - identity := common.MustHexToBytes("0x88dc3417d5058ec4b4503e0c12ea1a0a89be200fe98922423d4334014fa6b0ee") - identityPubKey, _ := ed25519.NewPublicKey(identity) - runtime := NewTestInstance(t, runtime.WESTEND_RUNTIME_v0929) - - keyOwnershipProofRaw := types.GrandpaOpaqueKeyOwnershipProof([]byte{64, 138, 252, 29, 127, 102, 189, 129, 207, 47, - 157, 60, 17, 138, 194, 121, 139, 92, 176, 175, 224, 16, 185, 93, 175, 251, 224, 81, 209, 61, 0, 71}) - keyOwnershipProof := scale.MustMarshal(keyOwnershipProofRaw) - - var opaqueKeyOwnershipProof types.GrandpaOpaqueKeyOwnershipProof - err := scale.Unmarshal(keyOwnershipProof, &opaqueKeyOwnershipProof) - require.NoError(t, err) - - firstVote := types.GrandpaVote{ - Hash: common.MustHexToHash("0x4801b8e62d31167d30c893cc1970f6a0e289420282a4b245b75f2c46fb308af1"), - Number: 10, - } - secondVote := types.GrandpaVote{ - Hash: common.MustHexToHash("0xc314327941fdd924bc67fd72651c40aececd485ca3e878c21e02abb40feae5bd"), - Number: 10, - } - - firstSignatureArray := mustHexTo64BArray(t, "0xd7292caacc62504365f179892a7399f233944bf261f8a3f66260f70e0016f2d"+ - "b63922726b015c82dc7131f4730fbec61f71672a571453e51029bfb469070900f") - - secondSignatureArray := mustHexTo64BArray(t, "0xb3c408b74905dfedfffa66f99f16fe8b938fd8df76a92225228a1ca07523"+ - "0b99a2d9e173c561952e1e378b701915ca188d2c832ef92a3fab8e455f32570c0807") - - grandpaEquivocation := types.GrandpaEquivocation{ - RoundNumber: 1, - ID: identityPubKey.AsBytes(), - FirstVote: firstVote, - FirstSignature: firstSignatureArray, - SecondVote: secondVote, - SecondSignature: secondSignatureArray, - } - preVoteEquivocation := types.PreVote(grandpaEquivocation) - equivocationVote := types.NewGrandpaEquivocation() - err = equivocationVote.Set(preVoteEquivocation) - require.NoError(t, err) - - equivocationProof := types.GrandpaEquivocationProof{ - SetID: 1, - Equivocation: *equivocationVote, - } - err = runtime.GrandpaSubmitReportEquivocationUnsignedExtrinsic(equivocationProof, opaqueKeyOwnershipProof) - require.NoError(t, err) -} - -func loadEntries(t *testing.T, filename string) *trie.Trie { - data, err := os.ReadFile(filename) - require.NoError(t, err) - - entries := make([][2][]byte, 0) - err = json.Unmarshal(data, &entries) - require.NoError(t, err) - - tr, err := trie.LoadFromEntries(entries, trie.V0) - require.NoError(t, err) - return tr -} - -func TestInstance_ExecuteBlock_WestendBlock1097836(t *testing.T) { - trie := loadEntries(t, "../test_data/westend/entrieslist-1097836.json") - expectedRoot := common.MustHexToHash("0x97d68d4c5a01571cf4beea9b058d53dfd6d6ce5382d0657a185041d736880e03") - require.Equal(t, expectedRoot, trie.MustHash()) - - // set state to genesis state - state := storage.NewTrieState(trie) - - cfg := Config{ - Storage: state, - LogLvl: log.Critical, - } - - instance, err := NewInstanceFromTrie(trie, cfg) - require.NoError(t, err) - - digest := types.NewDigest() - digest.Add( - *types.NewBABEPreRuntimeDigest(common.MustHexToBytes("0x02020000006522d30f00000000")), - types.SealDigest{ - ConsensusEngineID: types.BabeEngineID, - Data: common.MustHexToBytes("0xa06e4a23ae347c7edd24f84153c007563895d64a06b8781b1e21bf2c8bc426676bfd4bf20c08f276e54598aff3b64541d84c4b5da7bfa39479d0a45585a75388"), //nolint:lll - }, - ) - - exts := make([]types.Extrinsic, 0) - ext, err := common.HexToBytes("0x0402000b301f76e47201") - require.NoError(t, err) - exts = append(exts, ext) - ext, err = common.HexToBytes("0x040d0000") - require.NoError(t, err) - exts = append(exts, ext) - ext, err = common.HexToBytes("0x84c2c9676ac7a4e56bcb861cf443899114a17baa817ae5c295b9bb6f947fc427fd02db25d6a66498b5e81ac3a52ae4025eef7bb36eee31195c60c7ebdb6f0bac4177baf32e73719fe3e5fdce1059e9e16bb85bad0a0c4e34c4b0e680fb7a132cc4f7018502000004000c60f6d9b5a5ee5a0c0428d1940936a1b91cd29924578f7ac9e793db6dab0cd70700e40b5402") //nolint:lll - require.NoError(t, err) - exts = append(exts, ext) - - header := types.NewHeader( - common.MustHexToHash("0x805b7d8f50e4f30b014a25460c3a4057072ed902058d1f49fbbd99d3d6925ba3"), - common.MustHexToHash("0xb4df4c5bbceb56923f77d003e5064a839d3a319c0c930b955159637fdd74c393"), - common.MustHexToHash("0x2b0ee636bae60b595392d330a57e20524a13614605d3b3665433b7875f7a06fc"), - 1097836, - digest, - ) - require.Equal(t, "0x32ad1878e447c67f12395cef8d3f9e1c7871170711ba61350f40766e00998f89", header.Hash().String()) - - block := &types.Block{ - Header: *header, - Body: types.Body(exts), - } - - _, err = instance.ExecuteBlock(block) - require.NoError(t, err) -} diff --git a/lib/runtime/wasmer/helpers.go b/lib/runtime/wasmer/helpers.go deleted file mode 100644 index 9c95f220e2..0000000000 --- a/lib/runtime/wasmer/helpers.go +++ /dev/null @@ -1,275 +0,0 @@ -// Copyright 2022 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package wasmer - -// #include -import "C" //skipcq: SCC-compile - -import ( - "fmt" - "math/big" - - "github.com/ChainSafe/gossamer/lib/common/types" - "github.com/ChainSafe/gossamer/lib/runtime" - "github.com/ChainSafe/gossamer/lib/trie" - "github.com/ChainSafe/gossamer/pkg/scale" - "github.com/wasmerio/go-ext-wasm/wasmer" -) - -// toPointerSize converts an uint32 pointer and uint32 size -// to an int64 pointer size. -func toPointerSize(ptr, size uint32) (pointerSize int64) { - return int64(ptr) | (int64(size) << 32) -} - -// splitPointerSize converts an int64 pointer size to an -// uint32 pointer and an uint32 size. -func splitPointerSize(pointerSize int64) (ptr, size uint32) { - return uint32(pointerSize), uint32(pointerSize >> 32) -} - -// asMemorySlice converts a 64 bit pointer size to a Go byte slice. -func asMemorySlice(context wasmer.InstanceContext, pointerSize C.int64_t) (data []byte) { - memory := context.Memory().Data() - ptr, size := splitPointerSize(int64(pointerSize)) - return memory[ptr : ptr+size] -} - -// toWasmMemory copies a Go byte slice to wasm memory and returns the corresponding -// 64 bit pointer size. -func toWasmMemory(context wasmer.InstanceContext, data []byte) ( - pointerSize int64, err error) { - allocator := context.Data().(*runtime.Context).Allocator - size := uint32(len(data)) - - ptr, err := allocator.Allocate(size) - if err != nil { - return 0, fmt.Errorf("allocating: %w", err) - } - - memory := context.Memory().Data() - - if uint32(len(memory)) < ptr+size { - panic(fmt.Sprintf("length of memory is less than expected, want %d have %d", ptr+size, len(memory))) - } - - copy(memory[ptr:ptr+size], data) - pointerSize = toPointerSize(ptr, size) - return pointerSize, nil -} - -// toWasmMemorySized copies a Go byte slice to wasm memory and returns the corresponding -// 32 bit pointer. Note the data must have a well known fixed length in the runtime. -func toWasmMemorySized(context wasmer.InstanceContext, data []byte) ( - pointer uint32, err error) { - allocator := context.Data().(*runtime.Context).Allocator - - size := uint32(len(data)) - pointer, err = allocator.Allocate(size) - if err != nil { - return 0, fmt.Errorf("allocating: %w", err) - } - - memory := context.Memory().Data() - copy(memory[pointer:pointer+size], data) - - return pointer, nil -} - -// toWasmMemoryOptional scale encodes the byte slice `data`, writes it to wasm memory -// and returns the corresponding 64 bit pointer size. -func toWasmMemoryOptional(context wasmer.InstanceContext, data []byte) ( - pointerSize int64, err error) { - var optionalSlice *[]byte - if data != nil { - optionalSlice = &data - } - - encoded, err := scale.Marshal(optionalSlice) - if err != nil { - return 0, err - } - - return toWasmMemory(context, encoded) -} - -// toWasmMemoryResult wraps the data byte slice in a Result type, scale encodes it, -// copies it to wasm memory and returns the corresponding 64 bit pointer size. -func toWasmMemoryResult(context wasmer.InstanceContext, data []byte) ( - pointerSize int64, err error) { - var result *types.Result - if len(data) == 0 { - result = types.NewResult(byte(1), nil) - } else { - result = types.NewResult(byte(0), data) - } - - encodedResult, err := result.Encode() - if err != nil { - return 0, fmt.Errorf("encoding result: %w", err) - } - - return toWasmMemory(context, encodedResult) -} - -// toWasmMemoryOptional scale encodes the uint32 pointer `data`, writes it to wasm memory -// and returns the corresponding 64 bit pointer size. -func toWasmMemoryOptionalUint32(context wasmer.InstanceContext, data *uint32) ( - pointerSize int64, err error) { - enc, err := scale.Marshal(data) - if err != nil { - return 0, fmt.Errorf("scale encoding: %w", err) - } - return toWasmMemory(context, enc) -} - -func mustToWasmMemoryNil(context wasmer.InstanceContext) ( - cPointerSize C.int64_t) { - allocator := context.Data().(*runtime.Context).Allocator - ptr, err := allocator.Allocate(0) - if err != nil { - // we allocate 0 byte, this should never fail - panic(err) - } - pointerSize := toPointerSize(ptr, 0) - return C.int64_t(pointerSize) -} - -func toWasmMemoryOptionalNil(context wasmer.InstanceContext) ( - cPointerSize C.int64_t, err error) { - pointerSize, err := toWasmMemoryOptional(context, nil) - if err != nil { - return 0, err - } - - return C.int64_t(pointerSize), nil -} - -func mustToWasmMemoryOptionalNil(context wasmer.InstanceContext) ( - cPointerSize C.int64_t) { - cPointerSize, err := toWasmMemoryOptionalNil(context) - if err != nil { - panic(err) - } - return cPointerSize -} - -func toWasmMemoryResultEmpty(context wasmer.InstanceContext) ( - cPointerSize C.int64_t, err error) { - pointerSize, err := toWasmMemoryResult(context, nil) - if err != nil { - return 0, err - } - return C.int64_t(pointerSize), nil -} - -func mustToWasmMemoryResultEmpty(context wasmer.InstanceContext) ( - cPointerSize C.int64_t) { - cPointerSize, err := toWasmMemoryResultEmpty(context) - if err != nil { - panic(err) - } - return cPointerSize -} - -// toKillStorageResultEnum encodes the `allRemoved` flag and -// the `numRemoved` uint32 to a byte slice and returns it. -// The format used is: -// Byte 0: 1 if allRemoved is false, 0 otherwise -// Byte 1-5: scale encoding of numRemoved (up to 4 bytes) -func toKillStorageResultEnum(allRemoved bool, numRemoved uint32) ( - encodedEnumValue []byte, err error) { - encodedNumRemoved, err := scale.Marshal(numRemoved) - if err != nil { - return nil, fmt.Errorf("scale encoding: %w", err) - } - - encodedEnumValue = make([]byte, len(encodedNumRemoved)+1) - if !allRemoved { - // At least one key resides in the child trie due to the supplied limit. - encodedEnumValue[0] = 1 - } - copy(encodedEnumValue[1:], encodedNumRemoved) - - return encodedEnumValue, nil -} - -// toWasmMemoryFixedSizeOptional copies the `data` byte slice to a 64B array, -// scale encodes the pointer to the resulting array, writes it to wasm memory -// and returns the corresponding 64 bit pointer size. -func toWasmMemoryFixedSizeOptional(context wasmer.InstanceContext, data []byte) ( - pointerSize int64, err error) { - var optionalFixedSize [64]byte - copy(optionalFixedSize[:], data) - encodedOptionalFixedSize, err := scale.Marshal(&optionalFixedSize) - if err != nil { - return 0, fmt.Errorf("scale encoding: %w", err) - } - return toWasmMemory(context, encodedOptionalFixedSize) -} - -func storageAppend(storage runtime.Storage, key, valueToAppend []byte, version trie.Version) (err error) { - // this function assumes the item in storage is a SCALE encoded array of items - // the valueToAppend is a new item, so it appends the item and increases the length prefix by 1 - currentValue := storage.Get(key) - - var value []byte - if len(currentValue) == 0 { - nextLength := big.NewInt(1) - encodedLength, err := scale.Marshal(nextLength) - if err != nil { - return fmt.Errorf("scale encoding: %w", err) - } - value = make([]byte, len(encodedLength)+len(valueToAppend)) - // append new length prefix to start of items array - copy(value, encodedLength) - copy(value[len(encodedLength):], valueToAppend) - } else { - var currentLength *big.Int - err := scale.Unmarshal(currentValue, ¤tLength) - if err != nil { - logger.Tracef( - "item in storage is not SCALE encoded, overwriting at key 0x%x", key) - value = make([]byte, 1+len(valueToAppend)) - value[0] = 4 - copy(value[1:], valueToAppend) - } else { - lengthBytes, err := scale.Marshal(currentLength) - if err != nil { - return fmt.Errorf("scale encoding: %w", err) - } - - // increase length by 1 - nextLength := big.NewInt(0).Add(currentLength, big.NewInt(1)) - nextLengthBytes, err := scale.Marshal(nextLength) - if err != nil { - return fmt.Errorf("scale encoding next length bytes: %w", err) - } - - // append new item, pop off number of bytes required for length encoding, - // since we're not using old scale.Decoder - value = make([]byte, len(nextLengthBytes)+len(currentValue)-len(lengthBytes)+len(valueToAppend)) - // append new length prefix to start of items array - i := 0 - copy(value[i:], nextLengthBytes) - i += len(nextLengthBytes) - copy(value[i:], currentValue[len(lengthBytes):]) - i += len(currentValue) - len(lengthBytes) - copy(value[i:], valueToAppend) - } - } - - err = storage.Put(key, value, version) - if err != nil { - return fmt.Errorf("putting key and value in storage: %w", err) - } - - return nil -} - -func panicOnError(err error) { - if err != nil { - panic(err) - } -} diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go deleted file mode 100644 index a0a0568a57..0000000000 --- a/lib/runtime/wasmer/imports.go +++ /dev/null @@ -1,2288 +0,0 @@ -// Copyright 2021 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package wasmer - -// #include -// -// extern void ext_logging_log_version_1(void *context, int32_t level, int64_t target, int64_t msg); -// extern int32_t ext_logging_max_level_version_1(void *context); -// -// extern void ext_sandbox_instance_teardown_version_1(void *context, int32_t a); -// extern int32_t ext_sandbox_instantiate_version_1(void *context, int32_t a, int64_t b, int64_t c, int32_t d); -// extern int32_t ext_sandbox_invoke_version_1(void *context, int32_t a, int64_t b, int64_t c, int32_t d, int32_t e, int32_t f); -// extern int32_t ext_sandbox_memory_get_version_1(void *context, int32_t a, int32_t b, int32_t c, int32_t d); -// extern int32_t ext_sandbox_memory_new_version_1(void *context, int32_t a, int32_t b); -// extern int32_t ext_sandbox_memory_set_version_1(void *context, int32_t a, int32_t b, int32_t c, int32_t d); -// extern void ext_sandbox_memory_teardown_version_1(void *context, int32_t a); -// -// extern int32_t ext_crypto_ed25519_generate_version_1(void *context, int32_t a, int64_t b); -// extern int64_t ext_crypto_ed25519_public_keys_version_1(void *context, int32_t a); -// extern int64_t ext_crypto_ed25519_sign_version_1(void *context, int32_t a, int32_t b, int64_t c); -// extern int32_t ext_crypto_ed25519_verify_version_1(void *context, int32_t a, int64_t b, int32_t c); -// extern int32_t ext_crypto_finish_batch_verify_version_1(void *context); -// extern int64_t ext_crypto_secp256k1_ecdsa_recover_version_1(void *context, int32_t a, int32_t b); -// extern int64_t ext_crypto_secp256k1_ecdsa_recover_version_2(void *context, int32_t a, int32_t b); -// extern int64_t ext_crypto_secp256k1_ecdsa_recover_compressed_version_1(void *context, int32_t a, int32_t b); -// extern int64_t ext_crypto_secp256k1_ecdsa_recover_compressed_version_2(void *context, int32_t a, int32_t b); -// extern int32_t ext_crypto_ecdsa_verify_version_2(void *context, int32_t a, int64_t b, int32_t c); -// extern int32_t ext_crypto_sr25519_generate_version_1(void *context, int32_t a, int64_t b); -// extern int64_t ext_crypto_sr25519_public_keys_version_1(void *context, int32_t a); -// extern int64_t ext_crypto_sr25519_sign_version_1(void *context, int32_t a, int32_t b, int64_t c); -// extern int32_t ext_crypto_sr25519_verify_version_1(void *context, int32_t a, int64_t b, int32_t c); -// extern int32_t ext_crypto_sr25519_verify_version_2(void *context, int32_t a, int64_t b, int32_t c); -// extern void ext_crypto_start_batch_verify_version_1(void *context); -// -// extern int32_t ext_trie_blake2_256_root_version_1(void *context, int64_t a); -// extern int32_t ext_trie_blake2_256_ordered_root_version_1(void *context, int64_t a); -// extern int32_t ext_trie_blake2_256_ordered_root_version_2(void *context, int64_t a, int32_t b); -// extern int32_t ext_trie_blake2_256_verify_proof_version_1(void *context, int32_t a, int64_t b, int64_t c, int64_t d); -// -// extern int64_t ext_misc_runtime_version_version_1(void *context, int64_t a); -// extern void ext_misc_print_hex_version_1(void *context, int64_t a); -// extern void ext_misc_print_num_version_1(void *context, int64_t a); -// extern void ext_misc_print_utf8_version_1(void *context, int64_t a); -// -// extern void ext_default_child_storage_clear_version_1(void *context, int64_t a, int64_t b); -// extern int64_t ext_default_child_storage_get_version_1(void *context, int64_t a, int64_t b); -// extern int64_t ext_default_child_storage_next_key_version_1(void *context, int64_t a, int64_t b); -// extern int64_t ext_default_child_storage_read_version_1(void *context, int64_t a, int64_t b, int64_t c, int32_t d); -// extern int64_t ext_default_child_storage_root_version_1(void *context, int64_t a); -// extern int64_t ext_default_child_storage_root_version_2(void *context, int64_t a, int32_t b); -// extern void ext_default_child_storage_set_version_1(void *context, int64_t a, int64_t b, int64_t c); -// extern void ext_default_child_storage_storage_kill_version_1(void *context, int64_t a); -// extern int32_t ext_default_child_storage_storage_kill_version_2(void *context, int64_t a, int64_t b); -// extern int64_t ext_default_child_storage_storage_kill_version_3(void *context, int64_t a, int64_t b); -// extern void ext_default_child_storage_clear_prefix_version_1(void *context, int64_t a, int64_t b); -// extern int64_t ext_default_child_storage_clear_prefix_version_2(void *context, int64_t a, int64_t b, int64_t c); -// extern int32_t ext_default_child_storage_exists_version_1(void *context, int64_t a, int64_t b); -// -// extern void ext_allocator_free_version_1(void *context, int32_t a); -// extern int32_t ext_allocator_malloc_version_1(void *context, int32_t a); -// -// extern int32_t ext_hashing_blake2_128_version_1(void *context, int64_t a); -// extern int32_t ext_hashing_blake2_256_version_1(void *context, int64_t a); -// extern int32_t ext_hashing_keccak_256_version_1(void *context, int64_t a); -// extern int32_t ext_hashing_sha2_256_version_1(void *context, int64_t a); -// extern int32_t ext_hashing_twox_256_version_1(void *context, int64_t a); -// extern int32_t ext_hashing_twox_128_version_1(void *context, int64_t a); -// extern int32_t ext_hashing_twox_64_version_1(void *context, int64_t a); -// -// extern void ext_offchain_index_clear_version_1(void *context, int64_t a); -// extern void ext_offchain_index_set_version_1(void *context, int64_t a, int64_t b); -// extern int32_t ext_offchain_is_validator_version_1(void *context); -// extern void ext_offchain_local_storage_clear_version_1(void *context, int32_t a, int64_t b); -// extern int32_t ext_offchain_local_storage_compare_and_set_version_1(void *context, int32_t a, int64_t b, int64_t c, int64_t d); -// extern int64_t ext_offchain_local_storage_get_version_1(void *context, int32_t a, int64_t b); -// extern void ext_offchain_local_storage_set_version_1(void *context, int32_t a, int64_t b, int64_t c); -// extern int64_t ext_offchain_network_state_version_1(void *context); -// extern int32_t ext_offchain_random_seed_version_1(void *context); -// extern int64_t ext_offchain_submit_transaction_version_1(void *context, int64_t a); -// extern int64_t ext_offchain_timestamp_version_1(void *context); -// extern void ext_offchain_sleep_until_version_1(void *context, int64_t a); -// extern int64_t ext_offchain_http_request_start_version_1(void *context, int64_t a, int64_t b, int64_t c); -// extern int64_t ext_offchain_http_request_add_header_version_1(void *context, int32_t a, int64_t k, int64_t v); -// -// extern void ext_storage_append_version_1(void *context, int64_t a, int64_t b); -// extern int64_t ext_storage_changes_root_version_1(void *context, int64_t a); -// extern void ext_storage_clear_version_1(void *context, int64_t a); -// extern void ext_storage_clear_prefix_version_1(void *context, int64_t a); -// extern int64_t ext_storage_clear_prefix_version_2(void *context, int64_t a, int64_t b); -// extern void ext_storage_commit_transaction_version_1(void *context); -// extern int32_t ext_storage_exists_version_1(void *context, int64_t a); -// extern int64_t ext_storage_get_version_1(void *context, int64_t a); -// extern int64_t ext_storage_next_key_version_1(void *context, int64_t a); -// extern int64_t ext_storage_read_version_1(void *context, int64_t a, int64_t b, int32_t c); -// extern void ext_storage_rollback_transaction_version_1(void *context); -// extern int64_t ext_storage_root_version_1(void *context); -// extern int64_t ext_storage_root_version_2(void *context, int32_t a); -// extern void ext_storage_set_version_1(void *context, int64_t a, int64_t b); -// extern void ext_storage_start_transaction_version_1(void *context); -// -// extern void ext_transaction_index_index_version_1(void *context, int32_t a, int32_t b, int32_t c); -// extern void ext_transaction_index_renew_version_1(void *context, int32_t a, int32_t b); -import "C" //skipcq: SCC-compile - -import ( - "encoding/binary" - "fmt" - "math/big" - "math/rand" - "reflect" - "time" - "unsafe" - - "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/crypto" - "github.com/ChainSafe/gossamer/lib/crypto/ed25519" - "github.com/ChainSafe/gossamer/lib/crypto/secp256k1" - "github.com/ChainSafe/gossamer/lib/crypto/sr25519" - "github.com/ChainSafe/gossamer/lib/runtime" - "github.com/ChainSafe/gossamer/lib/transaction" - "github.com/ChainSafe/gossamer/lib/trie" - "github.com/ChainSafe/gossamer/lib/trie/proof" - "github.com/ChainSafe/gossamer/pkg/scale" - - wasm "github.com/wasmerio/go-ext-wasm/wasmer" -) - -const ( - validateSignatureFail = "failed to validate signature" -) - -//export ext_logging_log_version_1 -func ext_logging_log_version_1(context unsafe.Pointer, level C.int32_t, targetData, msgData C.int64_t) { - logger.Trace("executing...") - instanceContext := wasm.IntoInstanceContext(context) - - target := string(asMemorySlice(instanceContext, targetData)) - msg := string(asMemorySlice(instanceContext, msgData)) - - switch int(level) { - case 0: - logger.Critical("target=" + target + " message=" + msg) - case 1: - logger.Warn("target=" + target + " message=" + msg) - case 2: - logger.Info("target=" + target + " message=" + msg) - case 3: - logger.Debug("target=" + target + " message=" + msg) - case 4: - logger.Trace("target=" + target + " message=" + msg) - default: - logger.Errorf("level=%d target=%s message=%s", int(level), target, msg) - } -} - -//export ext_logging_max_level_version_1 -func ext_logging_max_level_version_1(context unsafe.Pointer) C.int32_t { - logger.Trace("executing...") - return 4 -} - -//export ext_transaction_index_index_version_1 -func ext_transaction_index_index_version_1(context unsafe.Pointer, a, b, c C.int32_t) { - logger.Trace("executing...") - logger.Warn("unimplemented") -} - -//export ext_transaction_index_renew_version_1 -func ext_transaction_index_renew_version_1(context unsafe.Pointer, a, b C.int32_t) { - logger.Trace("executing...") - logger.Warn("unimplemented") -} - -//export ext_sandbox_instance_teardown_version_1 -func ext_sandbox_instance_teardown_version_1(context unsafe.Pointer, a C.int32_t) { - logger.Trace("executing...") - logger.Warn("unimplemented") -} - -//export ext_sandbox_instantiate_version_1 -func ext_sandbox_instantiate_version_1(context unsafe.Pointer, a C.int32_t, x, y C.int64_t, z C.int32_t) C.int32_t { - logger.Trace("executing...") - logger.Warn("unimplemented") - return 0 -} - -//export ext_sandbox_invoke_version_1 -func ext_sandbox_invoke_version_1(context unsafe.Pointer, a C.int32_t, x, y C.int64_t, z, d, e C.int32_t) C.int32_t { - logger.Trace("executing...") - logger.Warn("unimplemented") - return 0 -} - -//export ext_sandbox_memory_get_version_1 -func ext_sandbox_memory_get_version_1(context unsafe.Pointer, a, z, d, e C.int32_t) C.int32_t { - logger.Trace("executing...") - logger.Warn("unimplemented") - return 0 -} - -//export ext_sandbox_memory_new_version_1 -func ext_sandbox_memory_new_version_1(context unsafe.Pointer, a, z C.int32_t) C.int32_t { - logger.Trace("executing...") - logger.Warn("unimplemented") - return 0 -} - -//export ext_sandbox_memory_set_version_1 -func ext_sandbox_memory_set_version_1(context unsafe.Pointer, a, z, d, e C.int32_t) C.int32_t { - logger.Trace("executing...") - logger.Warn("unimplemented") - return 0 -} - -//export ext_sandbox_memory_teardown_version_1 -func ext_sandbox_memory_teardown_version_1(context unsafe.Pointer, a C.int32_t) { - logger.Trace("executing...") - logger.Warn("unimplemented") -} - -//export ext_crypto_ed25519_generate_version_1 -func ext_crypto_ed25519_generate_version_1(context unsafe.Pointer, keyTypeID C.int32_t, seedSpan C.int64_t) C.int32_t { - logger.Trace("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - runtimeCtx := instanceContext.Data().(*runtime.Context) - memory := instanceContext.Memory().Data() - - id := memory[keyTypeID : keyTypeID+4] - seedBytes := asMemorySlice(instanceContext, seedSpan) - - var seed *[]byte - err := scale.Unmarshal(seedBytes, &seed) - if err != nil { - logger.Warnf("cannot generate key: %s", err) - return 0 - } - - var kp *ed25519.Keypair - - if seed != nil { - kp, err = ed25519.NewKeypairFromMnenomic(string(*seed), "") - } else { - kp, err = ed25519.GenerateKeypair() - } - - if err != nil { - logger.Warnf("cannot generate key: %s", err) - return 0 - } - - ks, err := runtimeCtx.Keystore.GetKeystore(id) - if err != nil { - logger.Warnf("error for id 0x%x: %s", id, err) - return 0 - } - - err = ks.Insert(kp) - if err != nil { - logger.Warnf("failed to insert key: %s", err) - return 0 - } - - ret, err := toWasmMemorySized(instanceContext, kp.Public().Encode()) - if err != nil { - logger.Warnf("failed to allocate memory: %s", err) - return 0 - } - - logger.Debug("generated ed25519 keypair with public key: " + kp.Public().Hex()) - return C.int32_t(ret) -} - -//export ext_crypto_ed25519_public_keys_version_1 -func ext_crypto_ed25519_public_keys_version_1(context unsafe.Pointer, keyTypeID C.int32_t) C.int64_t { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - runtimeCtx := instanceContext.Data().(*runtime.Context) - memory := instanceContext.Memory().Data() - - id := memory[keyTypeID : keyTypeID+4] - - ks, err := runtimeCtx.Keystore.GetKeystore(id) - if err != nil { - logger.Warnf("error for id 0x%x: %s", id, err) - ret, _ := toWasmMemory(instanceContext, []byte{0}) - return C.int64_t(ret) - } - - if ks.Type() != crypto.Ed25519Type && ks.Type() != crypto.UnknownType { - logger.Warnf( - "error for id 0x%x: keystore type is %s and not the expected ed25519", - id, ks.Type()) - ret, _ := toWasmMemory(instanceContext, []byte{0}) - return C.int64_t(ret) - } - - keys := ks.PublicKeys() - - var encodedKeys []byte - for _, key := range keys { - encodedKeys = append(encodedKeys, key.Encode()...) - } - - prefix, err := scale.Marshal(big.NewInt(int64(len(keys)))) - if err != nil { - logger.Errorf("failed to allocate memory: %s", err) - ret, _ := toWasmMemory(instanceContext, []byte{0}) - return C.int64_t(ret) - } - - ret, err := toWasmMemory(instanceContext, append(prefix, encodedKeys...)) - if err != nil { - logger.Errorf("failed to allocate memory: %s", err) - ret, _ = toWasmMemory(instanceContext, []byte{0}) - return C.int64_t(ret) - } - - return C.int64_t(ret) -} - -//export ext_crypto_ed25519_sign_version_1 -func ext_crypto_ed25519_sign_version_1(context unsafe.Pointer, keyTypeID, key C.int32_t, msg C.int64_t) C.int64_t { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - runtimeCtx := instanceContext.Data().(*runtime.Context) - memory := instanceContext.Memory().Data() - - id := memory[keyTypeID : keyTypeID+4] - - pubKeyData := memory[key : key+32] - pubKey, err := ed25519.NewPublicKey(pubKeyData) - if err != nil { - logger.Errorf("failed to get public keys: %s", err) - return 0 - } - - ks, err := runtimeCtx.Keystore.GetKeystore(id) - if err != nil { - logger.Warnf("error for id 0x%x: %s", id, err) - return mustToWasmMemoryOptionalNil(instanceContext) - } - - signingKey := ks.GetKeypair(pubKey) - if signingKey == nil { - logger.Error("could not find public key " + pubKey.Hex() + " in keystore") - ret, err := toWasmMemoryOptionalNil(instanceContext) - if err != nil { - logger.Errorf("failed to allocate memory: %s", err) - return 0 - } - return ret - } - - sig, err := signingKey.Sign(asMemorySlice(instanceContext, msg)) - if err != nil { - logger.Error("could not sign message") - } - - ret, err := toWasmMemoryFixedSizeOptional(instanceContext, sig) - if err != nil { - logger.Errorf("failed to allocate memory: %s", err) - return 0 - } - - return C.int64_t(ret) -} - -//export ext_crypto_ed25519_verify_version_1 -func ext_crypto_ed25519_verify_version_1(context unsafe.Pointer, sig C.int32_t, - msg C.int64_t, key C.int32_t) C.int32_t { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - memory := instanceContext.Memory().Data() - sigVerifier := instanceContext.Data().(*runtime.Context).SigVerifier - - signature := memory[sig : sig+64] - message := asMemorySlice(instanceContext, msg) - pubKeyData := memory[key : key+32] - - pubKey, err := ed25519.NewPublicKey(pubKeyData) - if err != nil { - logger.Error("failed to create public key") - return 0 - } - - if sigVerifier.IsStarted() { - signature := crypto.SignatureInfo{ - PubKey: pubKey.Encode(), - Sign: signature, - Msg: message, - VerifyFunc: ed25519.VerifySignature, - } - sigVerifier.Add(&signature) - return 1 - } - - if ok, err := pubKey.Verify(message, signature); err != nil || !ok { - logger.Error("failed to verify") - return 0 - } - - logger.Debug("verified ed25519 signature") - return 1 -} - -//export ext_crypto_secp256k1_ecdsa_recover_version_1 -func ext_crypto_secp256k1_ecdsa_recover_version_1(context unsafe.Pointer, sig, msg C.int32_t) C.int64_t { - logger.Trace("executing...") - instanceContext := wasm.IntoInstanceContext(context) - memory := instanceContext.Memory().Data() - - // msg must be the 32-byte hash of the message to be signed. - // sig must be a 65-byte compact ECDSA signature containing the - // recovery id as the last element - message := memory[msg : msg+32] - signature := memory[sig : sig+65] - - pub, err := secp256k1.RecoverPublicKey(message, signature) - if err != nil { - logger.Errorf("failed to recover public key: %s", err) - ret, err := toWasmMemoryResultEmpty(instanceContext) - if err != nil { - logger.Errorf("failed to allocate memory: %s", err) - return 0 - } - return ret - } - - logger.Debugf( - "recovered public key of length %d: 0x%x", - len(pub), pub) - - ret, err := toWasmMemoryResult(instanceContext, pub[1:]) - if err != nil { - logger.Errorf("failed to allocate memory: %s", err) - return 0 - } - - return C.int64_t(ret) -} - -//export ext_crypto_secp256k1_ecdsa_recover_version_2 -func ext_crypto_secp256k1_ecdsa_recover_version_2(context unsafe.Pointer, sig, msg C.int32_t) C.int64_t { - logger.Trace("executing...") - return ext_crypto_secp256k1_ecdsa_recover_version_1(context, sig, msg) -} - -//export ext_crypto_ecdsa_verify_version_2 -func ext_crypto_ecdsa_verify_version_2(context unsafe.Pointer, sig C.int32_t, msg C.int64_t, key C.int32_t) C.int32_t { - logger.Trace("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - memory := instanceContext.Memory().Data() - sigVerifier := instanceContext.Data().(*runtime.Context).SigVerifier - - message := asMemorySlice(instanceContext, msg) - signature := memory[sig : sig+64] - pubKey := memory[key : key+33] - - pub := new(secp256k1.PublicKey) - err := pub.Decode(pubKey) - if err != nil { - logger.Errorf("failed to decode public key: %s", err) - return C.int32_t(0) - } - - logger.Debugf("pub=%s, message=0x%x, signature=0x%x", - pub.Hex(), fmt.Sprintf("0x%x", message), fmt.Sprintf("0x%x", signature)) - - hash, err := common.Blake2bHash(message) - if err != nil { - logger.Errorf("failed to hash message: %s", err) - return C.int32_t(0) - } - - if sigVerifier.IsStarted() { - signature := crypto.SignatureInfo{ - PubKey: pub.Encode(), - Sign: signature, - Msg: hash[:], - VerifyFunc: secp256k1.VerifySignature, - } - sigVerifier.Add(&signature) - return C.int32_t(1) - } - - ok, err := pub.Verify(hash[:], signature) - if err != nil || !ok { - message := validateSignatureFail - if err != nil { - message += ": " + err.Error() - } - logger.Errorf(message) - return C.int32_t(0) - } - - logger.Debug("validated signature") - return C.int32_t(1) -} - -//export ext_crypto_secp256k1_ecdsa_recover_compressed_version_1 -func ext_crypto_secp256k1_ecdsa_recover_compressed_version_1(context unsafe.Pointer, sig, msg C.int32_t) C.int64_t { - logger.Trace("executing...") - instanceContext := wasm.IntoInstanceContext(context) - memory := instanceContext.Memory().Data() - - // msg must be the 32-byte hash of the message to be signed. - // sig must be a 65-byte compact ECDSA signature containing the - // recovery id as the last element - message := memory[msg : msg+32] - signature := memory[sig : sig+65] - - cpub, err := secp256k1.RecoverPublicKeyCompressed(message, signature) - if err != nil { - logger.Errorf("failed to recover public key: %s", err) - return mustToWasmMemoryResultEmpty(instanceContext) - } - - logger.Debugf( - "recovered public key of length %d: 0x%x", - len(cpub), cpub) - - ret, err := toWasmMemoryResult(instanceContext, cpub) - if err != nil { - logger.Errorf("failed to allocate memory: %s", err) - return 0 - } - - return C.int64_t(ret) -} - -//export ext_crypto_secp256k1_ecdsa_recover_compressed_version_2 -func ext_crypto_secp256k1_ecdsa_recover_compressed_version_2(context unsafe.Pointer, sig, msg C.int32_t) C.int64_t { - logger.Trace("executing...") - return ext_crypto_secp256k1_ecdsa_recover_compressed_version_1(context, sig, msg) -} - -//export ext_crypto_sr25519_generate_version_1 -func ext_crypto_sr25519_generate_version_1(context unsafe.Pointer, keyTypeID C.int32_t, seedSpan C.int64_t) C.int32_t { - logger.Trace("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - runtimeCtx := instanceContext.Data().(*runtime.Context) - memory := instanceContext.Memory().Data() - - id := memory[keyTypeID : keyTypeID+4] - seedBytes := asMemorySlice(instanceContext, seedSpan) - - var seed *[]byte - err := scale.Unmarshal(seedBytes, &seed) - if err != nil { - logger.Warnf("cannot generate key: %s", err) - return 0 - } - - var kp *sr25519.Keypair - if seed != nil { - kp, err = sr25519.NewKeypairFromMnenomic(string(*seed), "") - } else { - kp, err = sr25519.GenerateKeypair() - } - - if err != nil { - logger.Tracef("cannot generate key: %s", err) - panic(err) - } - - ks, err := runtimeCtx.Keystore.GetKeystore(id) - if err != nil { - logger.Warnf("error for id "+common.BytesToHex(id)+": %s", err) - return 0 - } - - err = ks.Insert(kp) - if err != nil { - logger.Warnf("failed to insert key: %s", err) - return 0 - } - - ret, err := toWasmMemorySized(instanceContext, kp.Public().Encode()) - if err != nil { - logger.Errorf("failed to allocate memory: %s", err) - return 0 - } - - logger.Debug("generated sr25519 keypair with public key: " + kp.Public().Hex()) - return C.int32_t(ret) -} - -//export ext_crypto_sr25519_public_keys_version_1 -func ext_crypto_sr25519_public_keys_version_1(context unsafe.Pointer, keyTypeID C.int32_t) C.int64_t { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - runtimeCtx := instanceContext.Data().(*runtime.Context) - memory := instanceContext.Memory().Data() - - id := memory[keyTypeID : keyTypeID+4] - - ks, err := runtimeCtx.Keystore.GetKeystore(id) - if err != nil { - logger.Warnf("error for id "+common.BytesToHex(id)+": %s", err) - ret, _ := toWasmMemory(instanceContext, []byte{0}) - return C.int64_t(ret) - } - - if ks.Type() != crypto.Sr25519Type && ks.Type() != crypto.UnknownType { - logger.Warnf( - "keystore type for id 0x%x is %s and not expected sr25519", - id, ks.Type()) - ret, _ := toWasmMemory(instanceContext, []byte{0}) - return C.int64_t(ret) - } - - keys := ks.PublicKeys() - - var encodedKeys []byte - for _, key := range keys { - encodedKeys = append(encodedKeys, key.Encode()...) - } - - prefix, err := scale.Marshal(big.NewInt(int64(len(keys)))) - if err != nil { - logger.Errorf("failed to allocate memory: %s", err) - ret, _ := toWasmMemory(instanceContext, []byte{0}) - return C.int64_t(ret) - } - - ret, err := toWasmMemory(instanceContext, append(prefix, encodedKeys...)) - if err != nil { - logger.Errorf("failed to allocate memory: %s", err) - ret, _ = toWasmMemory(instanceContext, []byte{0}) - return C.int64_t(ret) - } - - return C.int64_t(ret) -} - -//export ext_crypto_sr25519_sign_version_1 -func ext_crypto_sr25519_sign_version_1(context unsafe.Pointer, keyTypeID, key C.int32_t, msg C.int64_t) C.int64_t { - logger.Debug("executing...") - instanceContext := wasm.IntoInstanceContext(context) - runtimeCtx := instanceContext.Data().(*runtime.Context) - memory := instanceContext.Memory().Data() - - id := memory[keyTypeID : keyTypeID+4] - - ks, err := runtimeCtx.Keystore.GetKeystore(id) - if err != nil { - logger.Warnf("error for id 0x%x: %s", id, err) - return mustToWasmMemoryOptionalNil(instanceContext) - } - - var ret int64 - pubKey, err := sr25519.NewPublicKey(memory[key : key+32]) - if err != nil { - logger.Errorf("failed to get public key: %s", err) - return mustToWasmMemoryOptionalNil(instanceContext) - } - - signingKey := ks.GetKeypair(pubKey) - if signingKey == nil { - logger.Error("could not find public key " + pubKey.Hex() + " in keystore") - return mustToWasmMemoryOptionalNil(instanceContext) - } - - msgData := asMemorySlice(instanceContext, msg) - sig, err := signingKey.Sign(msgData) - if err != nil { - logger.Errorf("could not sign message: %s", err) - return mustToWasmMemoryOptionalNil(instanceContext) - } - - ret, err = toWasmMemoryFixedSizeOptional(instanceContext, sig) - if err != nil { - logger.Errorf("failed to allocate memory: %s", err) - return mustToWasmMemoryOptionalNil(instanceContext) - } - - return C.int64_t(ret) -} - -//export ext_crypto_sr25519_verify_version_1 -func ext_crypto_sr25519_verify_version_1(context unsafe.Pointer, sig C.int32_t, - msg C.int64_t, key C.int32_t) C.int32_t { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - memory := instanceContext.Memory().Data() - sigVerifier := instanceContext.Data().(*runtime.Context).SigVerifier - - message := asMemorySlice(instanceContext, msg) - signature := memory[sig : sig+64] - - pub, err := sr25519.NewPublicKey(memory[key : key+32]) - if err != nil { - logger.Error("invalid sr25519 public key") - return 0 - } - - logger.Debugf( - "pub=%s message=0x%x signature=0x%x", - pub.Hex(), message, signature) - - if sigVerifier.IsStarted() { - signature := crypto.SignatureInfo{ - PubKey: pub.Encode(), - Sign: signature, - Msg: message, - VerifyFunc: sr25519.VerifySignature, - } - sigVerifier.Add(&signature) - return 1 - } - - ok, err := pub.VerifyDeprecated(message, signature) - if err != nil || !ok { - message := validateSignatureFail - if err != nil { - message += ": " + err.Error() - } - logger.Debugf(message) - // this fails at block 3876, which seems to be expected, based on discussions - return 1 - } - - logger.Debug("verified sr25519 signature") - return 1 -} - -//export ext_crypto_sr25519_verify_version_2 -func ext_crypto_sr25519_verify_version_2(context unsafe.Pointer, sig C.int32_t, - msg C.int64_t, key C.int32_t) C.int32_t { - logger.Trace("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - memory := instanceContext.Memory().Data() - sigVerifier := instanceContext.Data().(*runtime.Context).SigVerifier - - message := asMemorySlice(instanceContext, msg) - signature := memory[sig : sig+64] - - pub, err := sr25519.NewPublicKey(memory[key : key+32]) - if err != nil { - logger.Error("invalid sr25519 public key") - return 0 - } - - logger.Debugf( - "pub=%s; message=0x%x; signature=0x%x", - pub.Hex(), message, signature) - - if sigVerifier.IsStarted() { - signature := crypto.SignatureInfo{ - PubKey: pub.Encode(), - Sign: signature, - Msg: message, - VerifyFunc: sr25519.VerifySignature, - } - sigVerifier.Add(&signature) - return 1 - } - - ok, err := pub.Verify(message, signature) - if err != nil || !ok { - message := validateSignatureFail - if err != nil { - message += ": " + err.Error() - } - logger.Errorf(message) - return 0 - } - - logger.Debug("validated signature") - return C.int32_t(1) -} - -//export ext_crypto_start_batch_verify_version_1 -func ext_crypto_start_batch_verify_version_1(context unsafe.Pointer) { - logger.Debug("executing...") - - // TODO: fix and re-enable signature verification (#1405) - // beginBatchVerify(context) -} - -//export ext_crypto_finish_batch_verify_version_1 -func ext_crypto_finish_batch_verify_version_1(context unsafe.Pointer) C.int32_t { - logger.Debug("executing...") - - // TODO: fix and re-enable signature verification (#1405) - // return finishBatchVerify(context) - return 1 -} - -//export ext_trie_blake2_256_root_version_1 -func ext_trie_blake2_256_root_version_1(context unsafe.Pointer, dataSpan C.int64_t) C.int32_t { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - memory := instanceContext.Memory().Data() - runtimeCtx := instanceContext.Data().(*runtime.Context) - data := asMemorySlice(instanceContext, dataSpan) - - t := trie.NewEmptyTrie() - - type kv struct { - Key, Value []byte - } - - // this function is expecting an array of (key, value) tuples - var kvs []kv - if err := scale.Unmarshal(data, &kvs); err != nil { - logger.Errorf("failed scale decoding data: %s", err) - return 0 - } - - for _, kv := range kvs { - err := t.Put(kv.Key, kv.Value, trie.V0) - if err != nil { - logger.Errorf("failed putting key 0x%x and value 0x%x into trie: %s", - kv.Key, kv.Value, err) - return 0 - } - } - - // allocate memory for value and copy value to memory - ptr, err := runtimeCtx.Allocator.Allocate(32) - if err != nil { - logger.Errorf("failed allocating: %s", err) - return 0 - } - - hash, err := t.Hash() - if err != nil { - logger.Errorf("failed computing trie Merkle root hash: %s", err) - return 0 - } - - logger.Debugf("root hash is %s", hash) - copy(memory[ptr:ptr+32], hash[:]) - return C.int32_t(ptr) -} - -//export ext_trie_blake2_256_ordered_root_version_1 -func ext_trie_blake2_256_ordered_root_version_1(context unsafe.Pointer, dataSpan C.int64_t) C.int32_t { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - memory := instanceContext.Memory().Data() - runtimeCtx := instanceContext.Data().(*runtime.Context) - data := asMemorySlice(instanceContext, dataSpan) - - t := trie.NewEmptyTrie() - var values [][]byte - err := scale.Unmarshal(data, &values) - if err != nil { - logger.Errorf("failed scale decoding data: %s", err) - return 0 - } - - for i, value := range values { - key, err := scale.Marshal(big.NewInt(int64(i))) - if err != nil { - logger.Errorf("failed scale encoding value index %d: %s", i, err) - return 0 - } - logger.Tracef( - "put key=0x%x and value=0x%x", - key, value) - - err = t.Put(key, value, trie.V0) - if err != nil { - logger.Errorf("failed putting key 0x%x and value 0x%x into trie: %s", - key, value, err) - return 0 - } - } - - // allocate memory for value and copy value to memory - ptr, err := runtimeCtx.Allocator.Allocate(32) - if err != nil { - logger.Errorf("failed allocating: %s", err) - return 0 - } - - hash, err := t.Hash() - if err != nil { - logger.Errorf("failed computing trie Merkle root hash: %s", err) - return 0 - } - - logger.Debugf("root hash is %s", hash) - copy(memory[ptr:ptr+32], hash[:]) - return C.int32_t(ptr) -} - -//export ext_trie_blake2_256_ordered_root_version_2 -func ext_trie_blake2_256_ordered_root_version_2(context unsafe.Pointer, - dataSpan C.int64_t, version C.int32_t) C.int32_t { - // TODO: update to use state trie version 1 (#2418) - return ext_trie_blake2_256_ordered_root_version_1(context, dataSpan) -} - -//export ext_trie_blake2_256_verify_proof_version_1 -func ext_trie_blake2_256_verify_proof_version_1(context unsafe.Pointer, - rootSpan C.int32_t, proofSpan, keySpan, valueSpan C.int64_t) C.int32_t { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - - toDecProofs := asMemorySlice(instanceContext, proofSpan) - var encodedProofNodes [][]byte - err := scale.Unmarshal(toDecProofs, &encodedProofNodes) - if err != nil { - logger.Errorf("failed scale decoding proof data: %s", err) - return C.int32_t(0) - } - - key := asMemorySlice(instanceContext, keySpan) - value := asMemorySlice(instanceContext, valueSpan) - - mem := instanceContext.Memory().Data() - trieRoot := mem[rootSpan : rootSpan+32] - - err = proof.Verify(encodedProofNodes, trieRoot, key, value) - if err != nil { - logger.Errorf("failed proof verification: %s", err) - return C.int32_t(0) - } - - return C.int32_t(1) -} - -//export ext_misc_print_hex_version_1 -func ext_misc_print_hex_version_1(context unsafe.Pointer, dataSpan C.int64_t) { - logger.Trace("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - data := asMemorySlice(instanceContext, dataSpan) - logger.Debugf("data: 0x%x", data) -} - -//export ext_misc_print_num_version_1 -func ext_misc_print_num_version_1(_ unsafe.Pointer, data C.int64_t) { - logger.Trace("executing...") - - logger.Debugf("num: %d", int64(data)) -} - -//export ext_misc_print_utf8_version_1 -func ext_misc_print_utf8_version_1(context unsafe.Pointer, dataSpan C.int64_t) { - logger.Trace("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - data := asMemorySlice(instanceContext, dataSpan) - logger.Debug("utf8: " + string(data)) -} - -//export ext_misc_runtime_version_version_1 -func ext_misc_runtime_version_version_1(context unsafe.Pointer, dataSpan C.int64_t) C.int64_t { - logger.Trace("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - code := asMemorySlice(instanceContext, dataSpan) - - version, err := GetRuntimeVersion(code) - if err != nil { - logger.Errorf("failed to get runtime version: %s", err) - return mustToWasmMemoryOptionalNil(instanceContext) - } - - // Note the encoding contains all the latest Core_version fields as defined in - // https://spec.polkadot.network/#defn-rt-core-version - // In other words, decoding older version data with missing fields - // and then encoding it will result in a longer encoding due to the - // extra version fields. This however remains compatible since the - // version fields are still encoded in the same order and an older - // decoder would succeed with the longer encoding. - encodedData, err := scale.Marshal(version) - if err != nil { - logger.Errorf("failed to encode result: %s", err) - return 0 - } - - out, err := toWasmMemoryOptional(instanceContext, encodedData) - if err != nil { - logger.Errorf("failed to allocate: %s", err) - return 0 - } - - return C.int64_t(out) -} - -//export ext_default_child_storage_read_version_1 -func ext_default_child_storage_read_version_1(context unsafe.Pointer, - childStorageKey, key, valueOut C.int64_t, offset C.int32_t) C.int64_t { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - storage := instanceContext.Data().(*runtime.Context).Storage - memory := instanceContext.Memory().Data() - - keyToChild := asMemorySlice(instanceContext, childStorageKey) - keyBytes := asMemorySlice(instanceContext, key) - value, err := storage.GetChildStorage(keyToChild, keyBytes) - if err != nil { - logger.Errorf("failed to get child storage: %s", err) - return 0 - } - - valueBuf, valueLen := splitPointerSize(int64(valueOut)) - copy(memory[valueBuf:valueBuf+valueLen], value[offset:]) - - size := uint32(len(value[offset:])) - sizeBuf := make([]byte, 4) - binary.LittleEndian.PutUint32(sizeBuf, size) - - sizeSpan, err := toWasmMemoryOptional(instanceContext, sizeBuf) - if err != nil { - logger.Errorf("failed to allocate: %s", err) - return 0 - } - - return C.int64_t(sizeSpan) -} - -//export ext_default_child_storage_clear_version_1 -func ext_default_child_storage_clear_version_1(context unsafe.Pointer, childStorageKey, keySpan C.int64_t) { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - ctx := instanceContext.Data().(*runtime.Context) - storage := ctx.Storage - - keyToChild := asMemorySlice(instanceContext, childStorageKey) - key := asMemorySlice(instanceContext, keySpan) - - err := storage.ClearChildStorage(keyToChild, key) - if err != nil { - logger.Errorf("failed to clear child storage: %s", err) - } -} - -//export ext_default_child_storage_clear_prefix_version_1 -func ext_default_child_storage_clear_prefix_version_1(context unsafe.Pointer, childStorageKey, prefixSpan C.int64_t) { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - ctx := instanceContext.Data().(*runtime.Context) - storage := ctx.Storage - - keyToChild := asMemorySlice(instanceContext, childStorageKey) - prefix := asMemorySlice(instanceContext, prefixSpan) - - err := storage.ClearPrefixInChild(keyToChild, prefix) - if err != nil { - logger.Errorf("failed to clear prefix in child: %s", err) - } -} - -// NewDigestItem returns a new VaryingDataType to represent a DigestItem -func NewKillStorageResult(deleted uint32, allDeleted bool) scale.VaryingDataType { - killStorageResult := scale.MustNewVaryingDataType(new(noneRemain), new(someRemain)) - - var err error - if allDeleted { - err = killStorageResult.Set(noneRemain(deleted)) - } else { - err = killStorageResult.Set(someRemain(deleted)) - } - - if err != nil { - panic(err) - } - return killStorageResult -} - -//export ext_default_child_storage_clear_prefix_version_2 -func ext_default_child_storage_clear_prefix_version_2(context unsafe.Pointer, childStorageKey, prefixSpan, - limitSpan C.int64_t) C.int64_t { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - ctx := instanceContext.Data().(*runtime.Context) - storage := ctx.Storage - - keyToChild := asMemorySlice(instanceContext, childStorageKey) - prefix := asMemorySlice(instanceContext, prefixSpan) - - limitBytes := asMemorySlice(instanceContext, limitSpan) - - var limit []byte - err := scale.Unmarshal(limitBytes, &limit) - if err != nil { - logger.Warnf("failed scale decoding limit: %s", err) - return mustToWasmMemoryNil(instanceContext) - } - - if len(limit) == 0 { - // limit is None, set limit to max - limit = []byte{0xff, 0xff, 0xff, 0xff} - } - - limitUint := binary.LittleEndian.Uint32(limit) - - deleted, allDeleted, err := storage.ClearPrefixInChildWithLimit( - keyToChild, prefix, limitUint) - if err != nil { - logger.Errorf("failed to clear prefix in child with limit: %s", err) - } - - killStorageResult := NewKillStorageResult(deleted, allDeleted) - - encodedKillStorageResult, err := scale.Marshal(killStorageResult) - if err != nil { - logger.Errorf("failed to encode result: %s", err) - return 0 - } - - resultSpan, err := toWasmMemoryOptional(instanceContext, encodedKillStorageResult) - if err != nil { - logger.Errorf("failed to allocate: %s", err) - return 0 - } - - return C.int64_t(resultSpan) -} - -//export ext_default_child_storage_exists_version_1 -func ext_default_child_storage_exists_version_1(context unsafe.Pointer, - childStorageKey, key C.int64_t) C.int32_t { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - storage := instanceContext.Data().(*runtime.Context).Storage - - keyToChild := asMemorySlice(instanceContext, childStorageKey) - keyBytes := asMemorySlice(instanceContext, key) - child, err := storage.GetChildStorage(keyToChild, keyBytes) - if err != nil { - logger.Errorf("failed to get child from child storage: %s", err) - return 0 - } - if child != nil { - return 1 - } - return 0 -} - -//export ext_default_child_storage_get_version_1 -func ext_default_child_storage_get_version_1(context unsafe.Pointer, childStorageKey, key C.int64_t) C.int64_t { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - storage := instanceContext.Data().(*runtime.Context).Storage - - keyToChild := asMemorySlice(instanceContext, childStorageKey) - keyBytes := asMemorySlice(instanceContext, key) - child, err := storage.GetChildStorage(keyToChild, keyBytes) - if err != nil { - logger.Errorf("failed to get child from child storage: %s", err) - return 0 - } - - value, err := toWasmMemoryOptional(instanceContext, child) - if err != nil { - logger.Errorf("failed to allocate: %s", err) - return 0 - } - - return C.int64_t(value) -} - -//export ext_default_child_storage_next_key_version_1 -func ext_default_child_storage_next_key_version_1(context unsafe.Pointer, childStorageKey, key C.int64_t) C.int64_t { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - storage := instanceContext.Data().(*runtime.Context).Storage - - keyToChild := asMemorySlice(instanceContext, childStorageKey) - keyBytes := asMemorySlice(instanceContext, key) - child, err := storage.GetChildNextKey(keyToChild, keyBytes) - if err != nil { - logger.Errorf("failed to get child's next key: %s", err) - return 0 - } - - value, err := toWasmMemoryOptional(instanceContext, child) - if err != nil { - logger.Errorf("failed to allocate: %s", err) - return 0 - } - - return C.int64_t(value) -} - -//export ext_default_child_storage_root_version_1 -func ext_default_child_storage_root_version_1(context unsafe.Pointer, - childStorageKey C.int64_t) (ptrSize C.int64_t) { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - storage := instanceContext.Data().(*runtime.Context).Storage - - child, err := storage.GetChild(asMemorySlice(instanceContext, childStorageKey)) - if err != nil { - logger.Errorf("failed to retrieve child: %s", err) - return 0 - } - - childRoot, err := child.Hash() - if err != nil { - logger.Errorf("failed to encode child root: %s", err) - return 0 - } - - root, err := toWasmMemoryOptional(instanceContext, childRoot[:]) - if err != nil { - logger.Errorf("failed to allocate: %s", err) - return 0 - } - - return C.int64_t(root) -} - -//export ext_default_child_storage_root_version_2 -func ext_default_child_storage_root_version_2(context unsafe.Pointer, - childStorageKey C.int64_t, stateVersion C.int32_t) (ptrSize C.int64_t) { - // TODO: Implement this after we have storage trie version 1 implemented #2418 - return ext_default_child_storage_root_version_1(context, childStorageKey) -} - -//export ext_default_child_storage_set_version_1 -func ext_default_child_storage_set_version_1(context unsafe.Pointer, - childStorageKeySpan, keySpan, valueSpan C.int64_t) { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - ctx := instanceContext.Data().(*runtime.Context) - storage := ctx.Storage - - childStorageKey := asMemorySlice(instanceContext, childStorageKeySpan) - key := asMemorySlice(instanceContext, keySpan) - value := asMemorySlice(instanceContext, valueSpan) - - cp := make([]byte, len(value)) - copy(cp, value) - - err := storage.SetChildStorage(childStorageKey, key, cp) - if err != nil { - logger.Errorf("failed to set value in child storage: %s", err) - return - } -} - -//export ext_default_child_storage_storage_kill_version_1 -func ext_default_child_storage_storage_kill_version_1(context unsafe.Pointer, childStorageKeySpan C.int64_t) { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - ctx := instanceContext.Data().(*runtime.Context) - storage := ctx.Storage - - childStorageKey := asMemorySlice(instanceContext, childStorageKeySpan) - err := storage.DeleteChild(childStorageKey) - panicOnError(err) -} - -//export ext_default_child_storage_storage_kill_version_2 -func ext_default_child_storage_storage_kill_version_2(context unsafe.Pointer, - childStorageKeySpan, lim C.int64_t) (allDeleted C.int32_t) { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - ctx := instanceContext.Data().(*runtime.Context) - storage := ctx.Storage - childStorageKey := asMemorySlice(instanceContext, childStorageKeySpan) - - limitBytes := asMemorySlice(instanceContext, lim) - - var limit *[]byte - err := scale.Unmarshal(limitBytes, &limit) - if err != nil { - logger.Warnf("cannot generate limit: %s", err) - return 0 - } - - _, all, err := storage.DeleteChildLimit(childStorageKey, limit) - if err != nil { - logger.Warnf("cannot get child storage: %s", err) - } - - if all { - return 1 - } - - return 0 -} - -type noneRemain uint32 - -func (noneRemain) Index() uint { return 0 } -func (nr noneRemain) String() string { return fmt.Sprintf("noneRemain(%d)", nr) } - -type someRemain uint32 - -func (someRemain) Index() uint { return 1 } -func (sr someRemain) String() string { return fmt.Sprintf("someRemain(%d)", sr) } - -//export ext_default_child_storage_storage_kill_version_3 -func ext_default_child_storage_storage_kill_version_3(context unsafe.Pointer, - childStorageKeySpan, lim C.int64_t) (pointerSize C.int64_t) { - logger.Debug("executing...") - instanceContext := wasm.IntoInstanceContext(context) - ctx := instanceContext.Data().(*runtime.Context) - storage := ctx.Storage - childStorageKey := asMemorySlice(instanceContext, childStorageKeySpan) - - limitBytes := asMemorySlice(instanceContext, lim) - - var limit *[]byte - err := scale.Unmarshal(limitBytes, &limit) - if err != nil { - logger.Warnf("cannot generate limit: %s", err) - } - - deleted, all, err := storage.DeleteChildLimit(childStorageKey, limit) - if err != nil { - logger.Warnf("cannot get child storage: %s", err) - return C.int64_t(0) - } - - vdt, err := scale.NewVaryingDataType(noneRemain(0), someRemain(0)) - if err != nil { - logger.Warnf("cannot create new varying data type: %s", err) - } - - if all { - err = vdt.Set(noneRemain(deleted)) - } else { - err = vdt.Set(someRemain(deleted)) - } - if err != nil { - logger.Warnf("cannot set varying data type: %s", err) - return C.int64_t(0) - } - - encoded, err := scale.Marshal(vdt) - if err != nil { - logger.Warnf("problem marshalling varying data type: %s", err) - return C.int64_t(0) - } - - out, err := toWasmMemoryOptional(instanceContext, encoded) - if err != nil { - logger.Warnf("failed to allocate: %s", err) - return 0 - } - - return C.int64_t(out) -} - -//export ext_allocator_free_version_1 -func ext_allocator_free_version_1(context unsafe.Pointer, addr C.int32_t) { - logger.Trace("executing...") - instanceContext := wasm.IntoInstanceContext(context) - runtimeCtx := instanceContext.Data().(*runtime.Context) - - // Deallocate memory - err := runtimeCtx.Allocator.Deallocate(uint32(addr)) - if err != nil { - logger.Errorf("failed to free memory: %s", err) - } -} - -//export ext_allocator_malloc_version_1 -func ext_allocator_malloc_version_1(context unsafe.Pointer, size C.int32_t) C.int32_t { - logger.Tracef("executing with size %d...", int64(size)) - - instanceContext := wasm.IntoInstanceContext(context) - ctx := instanceContext.Data().(*runtime.Context) - - // Allocate memory - res, err := ctx.Allocator.Allocate(uint32(size)) - if err != nil { - logger.Criticalf("failed to allocate memory: %s", err) - panic(err) - } - - return C.int32_t(res) -} - -//export ext_hashing_blake2_128_version_1 -func ext_hashing_blake2_128_version_1(context unsafe.Pointer, dataSpan C.int64_t) C.int32_t { - logger.Trace("executing...") - instanceContext := wasm.IntoInstanceContext(context) - - data := asMemorySlice(instanceContext, dataSpan) - - hash, err := common.Blake2b128(data) - if err != nil { - logger.Errorf("failed hashing data: %s", err) - return 0 - } - - logger.Debugf( - "data 0x%x has hash 0x%x", - data, hash) - - out, err := toWasmMemorySized(instanceContext, hash) - if err != nil { - logger.Errorf("failed to allocate: %s", err) - return 0 - } - - return C.int32_t(out) -} - -//export ext_hashing_blake2_256_version_1 -func ext_hashing_blake2_256_version_1(context unsafe.Pointer, dataSpan C.int64_t) C.int32_t { - logger.Trace("executing...") - instanceContext := wasm.IntoInstanceContext(context) - - data := asMemorySlice(instanceContext, dataSpan) - - hash, err := common.Blake2bHash(data) - if err != nil { - logger.Errorf("failed hashing data: %s", err) - return 0 - } - - logger.Debugf("data 0x%x has hash %s", data, hash) - - out, err := toWasmMemorySized(instanceContext, hash[:]) - if err != nil { - logger.Errorf("failed to allocate: %s", err) - return 0 - } - - return C.int32_t(out) -} - -//export ext_hashing_keccak_256_version_1 -func ext_hashing_keccak_256_version_1(context unsafe.Pointer, dataSpan C.int64_t) C.int32_t { - logger.Trace("executing...") - instanceContext := wasm.IntoInstanceContext(context) - - data := asMemorySlice(instanceContext, dataSpan) - - hash, err := common.Keccak256(data) - if err != nil { - logger.Errorf("failed hashing data: %s", err) - return 0 - } - - logger.Debugf("data 0x%x has hash %s", data, hash) - - out, err := toWasmMemorySized(instanceContext, hash[:]) - if err != nil { - logger.Errorf("failed to allocate: %s", err) - return 0 - } - - return C.int32_t(out) -} - -//export ext_hashing_sha2_256_version_1 -func ext_hashing_sha2_256_version_1(context unsafe.Pointer, dataSpan C.int64_t) C.int32_t { - logger.Trace("executing...") - instanceContext := wasm.IntoInstanceContext(context) - - data := asMemorySlice(instanceContext, dataSpan) - hash := common.Sha256(data) - - logger.Debugf("data 0x%x has hash %s", data, hash) - - out, err := toWasmMemorySized(instanceContext, hash[:]) - if err != nil { - logger.Errorf("failed to allocate: %s", err) - return 0 - } - - return C.int32_t(out) -} - -//export ext_hashing_twox_256_version_1 -func ext_hashing_twox_256_version_1(context unsafe.Pointer, dataSpan C.int64_t) C.int32_t { - logger.Trace("executing...") - instanceContext := wasm.IntoInstanceContext(context) - - data := asMemorySlice(instanceContext, dataSpan) - - hash, err := common.Twox256(data) - if err != nil { - logger.Errorf("failed hashing data: %s", err) - return 0 - } - - logger.Debugf("data 0x%x has hash %s", data, hash) - - out, err := toWasmMemorySized(instanceContext, hash[:]) - if err != nil { - logger.Errorf("failed to allocate: %s", err) - return 0 - } - - return C.int32_t(out) -} - -//export ext_hashing_twox_128_version_1 -func ext_hashing_twox_128_version_1(context unsafe.Pointer, dataSpan C.int64_t) C.int32_t { - logger.Trace("executing...") - instanceContext := wasm.IntoInstanceContext(context) - data := asMemorySlice(instanceContext, dataSpan) - - hash, err := common.Twox128Hash(data) - if err != nil { - logger.Errorf("failed hashing data: %s", err) - return 0 - } - - logger.Debugf( - "data 0x%x hash hash 0x%x", - data, hash) - - out, err := toWasmMemorySized(instanceContext, hash) - if err != nil { - logger.Errorf("failed to allocate: %s", err) - return 0 - } - - return C.int32_t(out) -} - -//export ext_hashing_twox_64_version_1 -func ext_hashing_twox_64_version_1(context unsafe.Pointer, dataSpan C.int64_t) C.int32_t { - logger.Trace("executing...") - instanceContext := wasm.IntoInstanceContext(context) - - data := asMemorySlice(instanceContext, dataSpan) - - hash, err := common.Twox64(data) - if err != nil { - logger.Errorf("failed hashing data: %s", err) - return 0 - } - - logger.Debugf( - "data 0x%x has hash 0x%x", - data, hash) - - out, err := toWasmMemorySized(instanceContext, hash) - if err != nil { - logger.Errorf("failed to allocate: %s", err) - return 0 - } - - return C.int32_t(out) -} - -//export ext_offchain_index_set_version_1 -func ext_offchain_index_set_version_1(context unsafe.Pointer, keySpan, valueSpan C.int64_t) { - logger.Trace("executing...") - instanceContext := wasm.IntoInstanceContext(context) - runtimeCtx := instanceContext.Data().(*runtime.Context) - - storageKey := asMemorySlice(instanceContext, keySpan) - newValue := asMemorySlice(instanceContext, valueSpan) - cp := make([]byte, len(newValue)) - copy(cp, newValue) - - err := runtimeCtx.NodeStorage.BaseDB.Put(storageKey, cp) - if err != nil { - logger.Errorf("failed to set value in raw storage: %s", err) - } -} - -//export ext_offchain_index_clear_version_1 -func ext_offchain_index_clear_version_1(context unsafe.Pointer, keySpan C.int64_t) { - // Remove a key and its associated value from the Offchain DB. - // https://github.com/paritytech/substrate/blob/4d608f9c42e8d70d835a748fa929e59a99497e90/primitives/io/src/lib.rs#L1213 - logger.Trace("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - runtimeCtx := instanceContext.Data().(*runtime.Context) - - storageKey := asMemorySlice(instanceContext, keySpan) - err := runtimeCtx.NodeStorage.BaseDB.Del(storageKey) - if err != nil { - logger.Errorf("failed to set value in raw storage: %s", err) - } -} - -//export ext_offchain_local_storage_clear_version_1 -func ext_offchain_local_storage_clear_version_1(context unsafe.Pointer, kind C.int32_t, key C.int64_t) { - logger.Trace("executing...") - instanceContext := wasm.IntoInstanceContext(context) - runtimeCtx := instanceContext.Data().(*runtime.Context) - - storageKey := asMemorySlice(instanceContext, key) - - memory := instanceContext.Memory().Data() - kindInt := binary.LittleEndian.Uint32(memory[kind : kind+4]) - - var err error - - switch runtime.NodeStorageType(kindInt) { - case runtime.NodeStorageTypePersistent: - err = runtimeCtx.NodeStorage.PersistentStorage.Del(storageKey) - case runtime.NodeStorageTypeLocal: - err = runtimeCtx.NodeStorage.LocalStorage.Del(storageKey) - } - - if err != nil { - logger.Errorf("failed to clear value from storage: %s", err) - } -} - -//export ext_offchain_is_validator_version_1 -func ext_offchain_is_validator_version_1(context unsafe.Pointer) C.int32_t { - logger.Debug("executing...") - instanceContext := wasm.IntoInstanceContext(context) - - runtimeCtx := instanceContext.Data().(*runtime.Context) - if runtimeCtx.Validator { - return 1 - } - return 0 -} - -//export ext_offchain_local_storage_compare_and_set_version_1 -func ext_offchain_local_storage_compare_and_set_version_1(context unsafe.Pointer, - kind C.int32_t, key, oldValue, newValue C.int64_t) (newValueSet C.int32_t) { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - runtimeCtx := instanceContext.Data().(*runtime.Context) - - storageKey := asMemorySlice(instanceContext, key) - - var storedValue []byte - var err error - - switch runtime.NodeStorageType(kind) { - case runtime.NodeStorageTypePersistent: - storedValue, err = runtimeCtx.NodeStorage.PersistentStorage.Get(storageKey) - case runtime.NodeStorageTypeLocal: - storedValue, err = runtimeCtx.NodeStorage.LocalStorage.Get(storageKey) - } - - if err != nil { - logger.Errorf("failed to get value from storage: %s", err) - return 0 - } - - oldVal := asMemorySlice(instanceContext, oldValue) - newVal := asMemorySlice(instanceContext, newValue) - if reflect.DeepEqual(storedValue, oldVal) { - cp := make([]byte, len(newVal)) - copy(cp, newVal) - err = runtimeCtx.NodeStorage.LocalStorage.Put(storageKey, cp) - if err != nil { - logger.Errorf("failed to set value in storage: %s", err) - return 0 - } - } - - return 1 -} - -//export ext_offchain_local_storage_get_version_1 -func ext_offchain_local_storage_get_version_1(context unsafe.Pointer, kind C.int32_t, key C.int64_t) C.int64_t { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - runtimeCtx := instanceContext.Data().(*runtime.Context) - storageKey := asMemorySlice(instanceContext, key) - - var res []byte - var err error - - switch runtime.NodeStorageType(kind) { - case runtime.NodeStorageTypePersistent: - res, err = runtimeCtx.NodeStorage.PersistentStorage.Get(storageKey) - case runtime.NodeStorageTypeLocal: - res, err = runtimeCtx.NodeStorage.LocalStorage.Get(storageKey) - } - - if err != nil { - logger.Errorf("failed to get value from storage: %s", err) - } - // allocate memory for value and copy value to memory - ptr, err := toWasmMemoryOptional(instanceContext, res) - if err != nil { - logger.Errorf("failed to allocate memory: %s", err) - return 0 - } - return C.int64_t(ptr) -} - -//export ext_offchain_local_storage_set_version_1 -func ext_offchain_local_storage_set_version_1(context unsafe.Pointer, kind C.int32_t, key, value C.int64_t) { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - runtimeCtx := instanceContext.Data().(*runtime.Context) - storageKey := asMemorySlice(instanceContext, key) - newValue := asMemorySlice(instanceContext, value) - cp := make([]byte, len(newValue)) - copy(cp, newValue) - - var err error - switch runtime.NodeStorageType(kind) { - case runtime.NodeStorageTypePersistent: - err = runtimeCtx.NodeStorage.PersistentStorage.Put(storageKey, cp) - case runtime.NodeStorageTypeLocal: - err = runtimeCtx.NodeStorage.LocalStorage.Put(storageKey, cp) - } - - if err != nil { - logger.Errorf("failed to set value in storage: %s", err) - } -} - -//export ext_offchain_network_state_version_1 -func ext_offchain_network_state_version_1(context unsafe.Pointer) C.int64_t { - logger.Debug("executing...") - instanceContext := wasm.IntoInstanceContext(context) - runtimeCtx := instanceContext.Data().(*runtime.Context) - if runtimeCtx.Network == nil { - return 0 - } - - nsEnc, err := scale.Marshal(runtimeCtx.Network.NetworkState()) - if err != nil { - logger.Errorf("failed at encoding network state: %s", err) - return 0 - } - - // allocate memory for value and copy value to memory - ptr, err := toWasmMemorySized(instanceContext, nsEnc) - if err != nil { - logger.Errorf("failed to allocate memory: %s", err) - return 0 - } - - return C.int64_t(ptr) -} - -//export ext_offchain_random_seed_version_1 -func ext_offchain_random_seed_version_1(context unsafe.Pointer) C.int32_t { - logger.Debug("executing...") - instanceContext := wasm.IntoInstanceContext(context) - - seed := make([]byte, 32) - _, err := rand.Read(seed) //nolint:staticcheck - if err != nil { - logger.Errorf("failed to generate random seed: %s", err) - } - ptr, err := toWasmMemorySized(instanceContext, seed) - if err != nil { - logger.Errorf("failed to allocate memory: %s", err) - } - return C.int32_t(ptr) -} - -//export ext_offchain_submit_transaction_version_1 -func ext_offchain_submit_transaction_version_1(context unsafe.Pointer, data C.int64_t) C.int64_t { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - extBytes := asMemorySlice(instanceContext, data) - - var extrinsic []byte - err := scale.Unmarshal(extBytes, &extrinsic) - if err != nil { - logger.Errorf("failed to decode extrinsic data: %s", err) - } - - // validate the transaction - txv := transaction.NewValidity(0, [][]byte{{}}, [][]byte{{}}, 0, false) - vtx := transaction.NewValidTransaction(extrinsic, txv) - - runtimeCtx := instanceContext.Data().(*runtime.Context) - runtimeCtx.Transaction.AddToPool(vtx) - - ptr, err := toWasmMemoryOptionalNil(instanceContext) - if err != nil { - logger.Errorf("failed to allocate memory: %s", err) - } - return ptr -} - -//export ext_offchain_timestamp_version_1 -func ext_offchain_timestamp_version_1(_ unsafe.Pointer) C.int64_t { - logger.Trace("executing...") - - now := time.Now().Unix() - return C.int64_t(now) -} - -//export ext_offchain_sleep_until_version_1 -func ext_offchain_sleep_until_version_1(_ unsafe.Pointer, deadline C.int64_t) { - logger.Trace("executing...") - - dur := time.Until(time.UnixMilli(int64(deadline))) - if dur > 0 { - time.Sleep(dur) - } -} - -//export ext_offchain_http_request_start_version_1 -func ext_offchain_http_request_start_version_1(context unsafe.Pointer, - methodSpan, uriSpan, metaSpan C.int64_t) (pointerSize C.int64_t) { - logger.Debug("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - runtimeCtx := instanceContext.Data().(*runtime.Context) - - httpMethod := asMemorySlice(instanceContext, methodSpan) - uri := asMemorySlice(instanceContext, uriSpan) - - result := scale.NewResult(int16(0), nil) - - reqID, err := runtimeCtx.OffchainHTTPSet.StartRequest(string(httpMethod), string(uri)) - if err != nil { - // StartRequest error already was logged - logger.Errorf("failed to start request: %s", err) - err = result.Set(scale.Err, nil) - } else { - err = result.Set(scale.OK, reqID) - } - - // note: just check if an error occurs while setting the result data - if err != nil { - logger.Errorf("failed to set the result data: %s", err) - return C.int64_t(0) - } - - enc, err := scale.Marshal(result) - if err != nil { - logger.Errorf("failed to scale marshal the result: %s", err) - return C.int64_t(0) - } - - ptr, err := toWasmMemory(instanceContext, enc) - if err != nil { - logger.Errorf("failed to allocate result on memory: %s", err) - return C.int64_t(0) - } - - return C.int64_t(ptr) -} - -//export ext_offchain_http_request_add_header_version_1 -func ext_offchain_http_request_add_header_version_1(context unsafe.Pointer, - reqID C.int32_t, nameSpan, valueSpan C.int64_t) (pointerSize C.int64_t) { - logger.Debug("executing...") - instanceContext := wasm.IntoInstanceContext(context) - - name := asMemorySlice(instanceContext, nameSpan) - value := asMemorySlice(instanceContext, valueSpan) - - runtimeCtx := instanceContext.Data().(*runtime.Context) - offchainReq := runtimeCtx.OffchainHTTPSet.Get(int16(reqID)) - - result := scale.NewResult(nil, nil) - resultMode := scale.OK - - err := offchainReq.AddHeader(string(name), string(value)) - if err != nil { - logger.Errorf("failed to add request header: %s", err) - resultMode = scale.Err - } - - err = result.Set(resultMode, nil) - if err != nil { - logger.Errorf("failed to set the result data: %s", err) - return C.int64_t(0) - } - - enc, err := scale.Marshal(result) - if err != nil { - logger.Errorf("failed to scale marshal the result: %s", err) - return C.int64_t(0) - } - - ptr, err := toWasmMemory(instanceContext, enc) - if err != nil { - logger.Errorf("failed to allocate result on memory: %s", err) - return C.int64_t(0) - } - - return C.int64_t(ptr) -} - -//export ext_storage_append_version_1 -func ext_storage_append_version_1(context unsafe.Pointer, keySpan, valueSpan C.int64_t) { - logger.Trace("executing...") - instanceContext := wasm.IntoInstanceContext(context) - ctx := instanceContext.Data().(*runtime.Context) - storage := ctx.Storage - - key := asMemorySlice(instanceContext, keySpan) - valueAppend := asMemorySlice(instanceContext, valueSpan) - logger.Debugf( - "will append value 0x%x to values at key 0x%x", - valueAppend, key) - - cp := make([]byte, len(valueAppend)) - copy(cp, valueAppend) - - err := storageAppend(storage, key, cp, trie.V0) - if err != nil { - logger.Errorf("failed appending to storage: %s", err) - } -} - -//export ext_storage_changes_root_version_1 -func ext_storage_changes_root_version_1(context unsafe.Pointer, parentHashSpan C.int64_t) C.int64_t { - logger.Trace("executing...") - logger.Debug("returning None") - - instanceContext := wasm.IntoInstanceContext(context) - - rootSpan, err := toWasmMemoryOptionalNil(instanceContext) - if err != nil { - logger.Errorf("failed to allocate: %s", err) - return 0 - } - - return rootSpan -} - -//export ext_storage_clear_version_1 -func ext_storage_clear_version_1(context unsafe.Pointer, keySpan C.int64_t) { - logger.Trace("executing...") - instanceContext := wasm.IntoInstanceContext(context) - ctx := instanceContext.Data().(*runtime.Context) - storage := ctx.Storage - - key := asMemorySlice(instanceContext, keySpan) - - logger.Debugf("key: 0x%x", key) - err := storage.Delete(key) - panicOnError(err) -} - -//export ext_storage_clear_prefix_version_1 -func ext_storage_clear_prefix_version_1(context unsafe.Pointer, prefixSpan C.int64_t) { - logger.Trace("executing...") - instanceContext := wasm.IntoInstanceContext(context) - ctx := instanceContext.Data().(*runtime.Context) - storage := ctx.Storage - - prefix := asMemorySlice(instanceContext, prefixSpan) - logger.Debugf("prefix: 0x%x", prefix) - - err := storage.ClearPrefix(prefix) - panicOnError(err) -} - -//export ext_storage_clear_prefix_version_2 -func ext_storage_clear_prefix_version_2(context unsafe.Pointer, prefixSpan, lim C.int64_t) C.int64_t { - logger.Trace("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - ctx := instanceContext.Data().(*runtime.Context) - storage := ctx.Storage - - prefix := asMemorySlice(instanceContext, prefixSpan) - logger.Debugf("prefix: 0x%x", prefix) - - limitBytes := asMemorySlice(instanceContext, lim) - - var limit []byte - err := scale.Unmarshal(limitBytes, &limit) - if err != nil { - logger.Warnf("failed scale decoding limit: %s", err) - return mustToWasmMemoryNil(instanceContext) - } - - if len(limit) == 0 { - // limit is None, set limit to max - limit = []byte{0xff, 0xff, 0xff, 0xff} - } - - limitUint := binary.LittleEndian.Uint32(limit) - numRemoved, all, err := storage.ClearPrefixLimit(prefix, limitUint) - if err != nil { - logger.Errorf("failed to clear prefix limit: %s", err) - return mustToWasmMemoryNil(instanceContext) - } - - encBytes, err := toKillStorageResultEnum(all, numRemoved) - if err != nil { - logger.Errorf("failed to allocate memory: %s", err) - return mustToWasmMemoryNil(instanceContext) - } - - valueSpan, err := toWasmMemory(instanceContext, encBytes) - if err != nil { - logger.Errorf("failed to allocate: %s", err) - return mustToWasmMemoryNil(instanceContext) - } - - return C.int64_t(valueSpan) -} - -//export ext_storage_exists_version_1 -func ext_storage_exists_version_1(context unsafe.Pointer, keySpan C.int64_t) C.int32_t { - logger.Trace("executing...") - instanceContext := wasm.IntoInstanceContext(context) - storage := instanceContext.Data().(*runtime.Context).Storage - - key := asMemorySlice(instanceContext, keySpan) - logger.Debugf("key: 0x%x", key) - - value := storage.Get(key) - if value != nil { - return 1 - } - - return 0 -} - -//export ext_storage_get_version_1 -func ext_storage_get_version_1(context unsafe.Pointer, keySpan C.int64_t) C.int64_t { - logger.Trace("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - storage := instanceContext.Data().(*runtime.Context).Storage - - key := asMemorySlice(instanceContext, keySpan) - logger.Debugf("key: 0x%x", key) - - value := storage.Get(key) - logger.Debugf("value: 0x%x", value) - - valueSpan, err := toWasmMemoryOptional(instanceContext, value) - if err != nil { - logger.Errorf("failed to allocate: %s", err) - return mustToWasmMemoryOptionalNil(instanceContext) - } - - return C.int64_t(valueSpan) -} - -//export ext_storage_next_key_version_1 -func ext_storage_next_key_version_1(context unsafe.Pointer, keySpan C.int64_t) C.int64_t { - logger.Trace("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - storage := instanceContext.Data().(*runtime.Context).Storage - - key := asMemorySlice(instanceContext, keySpan) - - next := storage.NextKey(key) - logger.Debugf( - "key: 0x%x; next key 0x%x", - key, next) - - nextSpan, err := toWasmMemoryOptional(instanceContext, next) - if err != nil { - logger.Errorf("failed to allocate: %s", err) - return 0 - } - - return C.int64_t(nextSpan) -} - -//export ext_storage_read_version_1 -func ext_storage_read_version_1(context unsafe.Pointer, keySpan, valueOut C.int64_t, offset C.int32_t) C.int64_t { - logger.Trace("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - storage := instanceContext.Data().(*runtime.Context).Storage - memory := instanceContext.Memory().Data() - - key := asMemorySlice(instanceContext, keySpan) - value := storage.Get(key) - logger.Debugf( - "key 0x%x has value 0x%x", - key, value) - - if value == nil { - return mustToWasmMemoryOptionalNil(instanceContext) - } - - var size uint32 - if uint32(offset) <= uint32(len(value)) { - size = uint32(len(value[offset:])) - valueBuf, valueLen := splitPointerSize(int64(valueOut)) - copy(memory[valueBuf:valueBuf+valueLen], value[offset:]) - } - - sizeSpan, err := toWasmMemoryOptionalUint32(instanceContext, &size) - if err != nil { - logger.Errorf("failed to allocate: %s", err) - return 0 - } - - return C.int64_t(sizeSpan) -} - -//export ext_storage_root_version_1 -func ext_storage_root_version_1(context unsafe.Pointer) C.int64_t { - logger.Trace("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - storage := instanceContext.Data().(*runtime.Context).Storage - - root, err := storage.Root() - if err != nil { - logger.Errorf("failed to get storage root: %s", err) - return 0 - } - - logger.Debugf("root hash is: %s", root) - - rootSpan, err := toWasmMemory(instanceContext, root[:]) - if err != nil { - logger.Errorf("failed to allocate: %s", err) - return 0 - } - - return C.int64_t(rootSpan) -} - -//export ext_storage_root_version_2 -func ext_storage_root_version_2(context unsafe.Pointer, version C.int32_t) C.int64_t { - // TODO: update to use state trie version 1 (#2418) - return ext_storage_root_version_1(context) -} - -//export ext_storage_set_version_1 -func ext_storage_set_version_1(context unsafe.Pointer, keySpan, valueSpan C.int64_t) { - logger.Trace("executing...") - - instanceContext := wasm.IntoInstanceContext(context) - ctx := instanceContext.Data().(*runtime.Context) - storage := ctx.Storage - - key := asMemorySlice(instanceContext, keySpan) - value := asMemorySlice(instanceContext, valueSpan) - - cp := make([]byte, len(value)) - copy(cp, value) - - logger.Debugf( - "key 0x%x has value 0x%x", - key, value) - err := storage.Put(key, cp, trie.V0) - panicOnError(err) -} - -//export ext_storage_start_transaction_version_1 -func ext_storage_start_transaction_version_1(context unsafe.Pointer) { - logger.Debug("executing...") - instanceContext := wasm.IntoInstanceContext(context) - instanceContext.Data().(*runtime.Context).Storage.BeginStorageTransaction() -} - -//export ext_storage_rollback_transaction_version_1 -func ext_storage_rollback_transaction_version_1(context unsafe.Pointer) { - logger.Debug("executing...") - instanceContext := wasm.IntoInstanceContext(context) - instanceContext.Data().(*runtime.Context).Storage.RollbackStorageTransaction() -} - -//export ext_storage_commit_transaction_version_1 -func ext_storage_commit_transaction_version_1(context unsafe.Pointer) { - logger.Debug("executing...") - instanceContext := wasm.IntoInstanceContext(context) - instanceContext.Data().(*runtime.Context).Storage.CommitStorageTransaction() -} - -// importsNodeRuntime returns the WASM imports for the node runtime. -func importsNodeRuntime() (imports *wasm.Imports, err error) { - imports = wasm.NewImports() - // Note imports are closed by the call to wasm.Instance.Close() - - for _, toRegister := range []struct { - importName string - implementation interface{} - cgoPointer unsafe.Pointer - }{ - {"ext_allocator_free_version_1", ext_allocator_free_version_1, C.ext_allocator_free_version_1}, - {"ext_allocator_malloc_version_1", ext_allocator_malloc_version_1, C.ext_allocator_malloc_version_1}, - {"ext_crypto_ecdsa_verify_version_2", ext_crypto_ecdsa_verify_version_2, C.ext_crypto_ecdsa_verify_version_2}, - {"ext_crypto_ed25519_generate_version_1", ext_crypto_ed25519_generate_version_1, C.ext_crypto_ed25519_generate_version_1}, - {"ext_crypto_ed25519_public_keys_version_1", ext_crypto_ed25519_public_keys_version_1, C.ext_crypto_ed25519_public_keys_version_1}, - {"ext_crypto_ed25519_sign_version_1", ext_crypto_ed25519_sign_version_1, C.ext_crypto_ed25519_sign_version_1}, - {"ext_crypto_ed25519_verify_version_1", ext_crypto_ed25519_verify_version_1, C.ext_crypto_ed25519_verify_version_1}, - {"ext_crypto_finish_batch_verify_version_1", ext_crypto_finish_batch_verify_version_1, C.ext_crypto_finish_batch_verify_version_1}, - {"ext_crypto_secp256k1_ecdsa_recover_compressed_version_1", ext_crypto_secp256k1_ecdsa_recover_compressed_version_1, C.ext_crypto_secp256k1_ecdsa_recover_compressed_version_1}, - {"ext_crypto_secp256k1_ecdsa_recover_compressed_version_2", ext_crypto_secp256k1_ecdsa_recover_compressed_version_2, C.ext_crypto_secp256k1_ecdsa_recover_compressed_version_2}, - {"ext_crypto_secp256k1_ecdsa_recover_version_1", ext_crypto_secp256k1_ecdsa_recover_version_1, C.ext_crypto_secp256k1_ecdsa_recover_version_1}, - {"ext_crypto_secp256k1_ecdsa_recover_version_2", ext_crypto_secp256k1_ecdsa_recover_version_2, C.ext_crypto_secp256k1_ecdsa_recover_version_2}, - {"ext_crypto_sr25519_generate_version_1", ext_crypto_sr25519_generate_version_1, C.ext_crypto_sr25519_generate_version_1}, - {"ext_crypto_sr25519_public_keys_version_1", ext_crypto_sr25519_public_keys_version_1, C.ext_crypto_sr25519_public_keys_version_1}, - {"ext_crypto_sr25519_sign_version_1", ext_crypto_sr25519_sign_version_1, C.ext_crypto_sr25519_sign_version_1}, - {"ext_crypto_sr25519_verify_version_1", ext_crypto_sr25519_verify_version_1, C.ext_crypto_sr25519_verify_version_1}, - {"ext_crypto_sr25519_verify_version_2", ext_crypto_sr25519_verify_version_2, C.ext_crypto_sr25519_verify_version_2}, - {"ext_crypto_start_batch_verify_version_1", ext_crypto_start_batch_verify_version_1, C.ext_crypto_start_batch_verify_version_1}, - {"ext_default_child_storage_clear_prefix_version_1", ext_default_child_storage_clear_prefix_version_1, C.ext_default_child_storage_clear_prefix_version_1}, - {"ext_default_child_storage_clear_prefix_version_2", ext_default_child_storage_clear_prefix_version_2, C.ext_default_child_storage_clear_prefix_version_2}, - {"ext_default_child_storage_clear_version_1", ext_default_child_storage_clear_version_1, C.ext_default_child_storage_clear_version_1}, - {"ext_default_child_storage_exists_version_1", ext_default_child_storage_exists_version_1, C.ext_default_child_storage_exists_version_1}, - {"ext_default_child_storage_get_version_1", ext_default_child_storage_get_version_1, C.ext_default_child_storage_get_version_1}, - {"ext_default_child_storage_next_key_version_1", ext_default_child_storage_next_key_version_1, C.ext_default_child_storage_next_key_version_1}, - {"ext_default_child_storage_read_version_1", ext_default_child_storage_read_version_1, C.ext_default_child_storage_read_version_1}, - {"ext_default_child_storage_root_version_1", ext_default_child_storage_root_version_1, C.ext_default_child_storage_root_version_1}, - {"ext_default_child_storage_root_version_2", ext_default_child_storage_root_version_2, C.ext_default_child_storage_root_version_2}, - {"ext_default_child_storage_set_version_1", ext_default_child_storage_set_version_1, C.ext_default_child_storage_set_version_1}, - {"ext_default_child_storage_storage_kill_version_1", ext_default_child_storage_storage_kill_version_1, C.ext_default_child_storage_storage_kill_version_1}, - {"ext_default_child_storage_storage_kill_version_2", ext_default_child_storage_storage_kill_version_2, C.ext_default_child_storage_storage_kill_version_2}, - {"ext_default_child_storage_storage_kill_version_3", ext_default_child_storage_storage_kill_version_3, C.ext_default_child_storage_storage_kill_version_3}, - {"ext_hashing_blake2_128_version_1", ext_hashing_blake2_128_version_1, C.ext_hashing_blake2_128_version_1}, - {"ext_hashing_blake2_256_version_1", ext_hashing_blake2_256_version_1, C.ext_hashing_blake2_256_version_1}, - {"ext_hashing_keccak_256_version_1", ext_hashing_keccak_256_version_1, C.ext_hashing_keccak_256_version_1}, - {"ext_hashing_sha2_256_version_1", ext_hashing_sha2_256_version_1, C.ext_hashing_sha2_256_version_1}, - {"ext_hashing_twox_128_version_1", ext_hashing_twox_128_version_1, C.ext_hashing_twox_128_version_1}, - {"ext_hashing_twox_256_version_1", ext_hashing_twox_256_version_1, C.ext_hashing_twox_256_version_1}, - {"ext_hashing_twox_64_version_1", ext_hashing_twox_64_version_1, C.ext_hashing_twox_64_version_1}, - {"ext_logging_log_version_1", ext_logging_log_version_1, C.ext_logging_log_version_1}, - {"ext_logging_max_level_version_1", ext_logging_max_level_version_1, C.ext_logging_max_level_version_1}, - {"ext_misc_print_hex_version_1", ext_misc_print_hex_version_1, C.ext_misc_print_hex_version_1}, - {"ext_misc_print_num_version_1", ext_misc_print_num_version_1, C.ext_misc_print_num_version_1}, - {"ext_misc_print_utf8_version_1", ext_misc_print_utf8_version_1, C.ext_misc_print_utf8_version_1}, - {"ext_misc_runtime_version_version_1", ext_misc_runtime_version_version_1, C.ext_misc_runtime_version_version_1}, - {"ext_offchain_index_clear_version_1", ext_offchain_index_clear_version_1, C.ext_offchain_index_clear_version_1}, - {"ext_offchain_http_request_add_header_version_1", ext_offchain_http_request_add_header_version_1, C.ext_offchain_http_request_add_header_version_1}, - {"ext_offchain_http_request_start_version_1", ext_offchain_http_request_start_version_1, C.ext_offchain_http_request_start_version_1}, - {"ext_offchain_index_set_version_1", ext_offchain_index_set_version_1, C.ext_offchain_index_set_version_1}, - {"ext_offchain_is_validator_version_1", ext_offchain_is_validator_version_1, C.ext_offchain_is_validator_version_1}, - {"ext_offchain_local_storage_clear_version_1", ext_offchain_local_storage_clear_version_1, C.ext_offchain_local_storage_clear_version_1}, - {"ext_offchain_local_storage_compare_and_set_version_1", ext_offchain_local_storage_compare_and_set_version_1, C.ext_offchain_local_storage_compare_and_set_version_1}, - {"ext_offchain_local_storage_get_version_1", ext_offchain_local_storage_get_version_1, C.ext_offchain_local_storage_get_version_1}, - {"ext_offchain_local_storage_set_version_1", ext_offchain_local_storage_set_version_1, C.ext_offchain_local_storage_set_version_1}, - {"ext_offchain_network_state_version_1", ext_offchain_network_state_version_1, C.ext_offchain_network_state_version_1}, - {"ext_offchain_random_seed_version_1", ext_offchain_random_seed_version_1, C.ext_offchain_random_seed_version_1}, - {"ext_offchain_sleep_until_version_1", ext_offchain_sleep_until_version_1, C.ext_offchain_sleep_until_version_1}, - {"ext_offchain_submit_transaction_version_1", ext_offchain_submit_transaction_version_1, C.ext_offchain_submit_transaction_version_1}, - {"ext_offchain_timestamp_version_1", ext_offchain_timestamp_version_1, C.ext_offchain_timestamp_version_1}, - {"ext_sandbox_instance_teardown_version_1", ext_sandbox_instance_teardown_version_1, C.ext_sandbox_instance_teardown_version_1}, - {"ext_sandbox_instantiate_version_1", ext_sandbox_instantiate_version_1, C.ext_sandbox_instantiate_version_1}, - {"ext_sandbox_invoke_version_1", ext_sandbox_invoke_version_1, C.ext_sandbox_invoke_version_1}, - {"ext_sandbox_memory_get_version_1", ext_sandbox_memory_get_version_1, C.ext_sandbox_memory_get_version_1}, - {"ext_sandbox_memory_new_version_1", ext_sandbox_memory_new_version_1, C.ext_sandbox_memory_new_version_1}, - {"ext_sandbox_memory_set_version_1", ext_sandbox_memory_set_version_1, C.ext_sandbox_memory_set_version_1}, - {"ext_sandbox_memory_teardown_version_1", ext_sandbox_memory_teardown_version_1, C.ext_sandbox_memory_teardown_version_1}, - {"ext_storage_append_version_1", ext_storage_append_version_1, C.ext_storage_append_version_1}, - {"ext_storage_changes_root_version_1", ext_storage_changes_root_version_1, C.ext_storage_changes_root_version_1}, - {"ext_storage_clear_prefix_version_1", ext_storage_clear_prefix_version_1, C.ext_storage_clear_prefix_version_1}, - {"ext_storage_clear_prefix_version_2", ext_storage_clear_prefix_version_2, C.ext_storage_clear_prefix_version_2}, - {"ext_storage_clear_version_1", ext_storage_clear_version_1, C.ext_storage_clear_version_1}, - {"ext_storage_commit_transaction_version_1", ext_storage_commit_transaction_version_1, C.ext_storage_commit_transaction_version_1}, - {"ext_storage_exists_version_1", ext_storage_exists_version_1, C.ext_storage_exists_version_1}, - {"ext_storage_get_version_1", ext_storage_get_version_1, C.ext_storage_get_version_1}, - {"ext_storage_next_key_version_1", ext_storage_next_key_version_1, C.ext_storage_next_key_version_1}, - {"ext_storage_read_version_1", ext_storage_read_version_1, C.ext_storage_read_version_1}, - {"ext_storage_rollback_transaction_version_1", ext_storage_rollback_transaction_version_1, C.ext_storage_rollback_transaction_version_1}, - {"ext_storage_root_version_1", ext_storage_root_version_1, C.ext_storage_root_version_1}, - {"ext_storage_root_version_2", ext_storage_root_version_2, C.ext_storage_root_version_2}, - {"ext_storage_set_version_1", ext_storage_set_version_1, C.ext_storage_set_version_1}, - {"ext_storage_start_transaction_version_1", ext_storage_start_transaction_version_1, C.ext_storage_start_transaction_version_1}, - {"ext_transaction_index_index_version_1", ext_transaction_index_index_version_1, C.ext_transaction_index_index_version_1}, - {"ext_transaction_index_renew_version_1", ext_transaction_index_renew_version_1, C.ext_transaction_index_renew_version_1}, - {"ext_trie_blake2_256_ordered_root_version_1", ext_trie_blake2_256_ordered_root_version_1, C.ext_trie_blake2_256_ordered_root_version_1}, - {"ext_trie_blake2_256_ordered_root_version_2", ext_trie_blake2_256_ordered_root_version_2, C.ext_trie_blake2_256_ordered_root_version_2}, - {"ext_trie_blake2_256_root_version_1", ext_trie_blake2_256_root_version_1, C.ext_trie_blake2_256_root_version_1}, - {"ext_trie_blake2_256_verify_proof_version_1", ext_trie_blake2_256_verify_proof_version_1, C.ext_trie_blake2_256_verify_proof_version_1}, - } { - _, err = imports.AppendFunction(toRegister.importName, toRegister.implementation, toRegister.cgoPointer) - if err != nil { - return nil, fmt.Errorf("importing function: %w", err) - } - } - - return imports, nil -} diff --git a/lib/runtime/wasmer/imports_test.go b/lib/runtime/wasmer/imports_test.go deleted file mode 100644 index a546af3c1a..0000000000 --- a/lib/runtime/wasmer/imports_test.go +++ /dev/null @@ -1,1961 +0,0 @@ -// Copyright 2021 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package wasmer - -import ( - "bytes" - "encoding/binary" - "net/http" - "sort" - "testing" - "time" - - "github.com/ChainSafe/gossamer/internal/database" - "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/common/types" - "github.com/ChainSafe/gossamer/lib/crypto" - "github.com/ChainSafe/gossamer/lib/crypto/ed25519" - "github.com/ChainSafe/gossamer/lib/crypto/secp256k1" - "github.com/ChainSafe/gossamer/lib/crypto/sr25519" - "github.com/ChainSafe/gossamer/lib/keystore" - "github.com/ChainSafe/gossamer/lib/runtime" - "github.com/ChainSafe/gossamer/lib/runtime/storage" - "github.com/ChainSafe/gossamer/lib/trie" - "github.com/ChainSafe/gossamer/lib/trie/proof" - "github.com/ChainSafe/gossamer/pkg/scale" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/wasmerio/go-ext-wasm/wasmer" -) - -var testChildKey = []byte("childKey") -var testKey = []byte("key") -var testValue = []byte("value") - -func Test_ext_offchain_index_clear_version_1(t *testing.T) { - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - err := inst.ctx.NodeStorage.BaseDB.Put(testKey, testValue) - require.NoError(t, err) - - value, err := inst.ctx.NodeStorage.BaseDB.Get(testKey) - require.NoError(t, err) - require.Equal(t, testValue, value) - - encKey, err := scale.Marshal(testKey) - require.NoError(t, err) - - _, err = inst.Exec("rtm_ext_offchain_index_clear_version_1", encKey) - require.NoError(t, err) - - _, err = inst.ctx.NodeStorage.BaseDB.Get(testKey) - require.ErrorIs(t, err, database.ErrNotFound) -} - -func Test_ext_offchain_timestamp_version_1(t *testing.T) { - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - runtimeFunc, ok := inst.vm.Exports["rtm_ext_offchain_timestamp_version_1"] - require.True(t, ok) - - res, err := runtimeFunc(0, 0) - require.NoError(t, err) - - outputPtr, outputLength := splitPointerSize(res.ToI64()) - memory := inst.vm.Memory.Data() - data := memory[outputPtr : outputPtr+outputLength] - var timestamp int64 - err = scale.Unmarshal(data, ×tamp) - require.NoError(t, err) - - expected := time.Now().Unix() - require.GreaterOrEqual(t, expected, timestamp) -} - -func Test_ext_offchain_sleep_until_version_1(t *testing.T) { - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - input := time.Now().UnixMilli() - enc, err := scale.Marshal(input) - require.NoError(t, err) - - _, err = inst.Exec("rtm_ext_offchain_sleep_until_version_1", enc) //auto conversion to i64 - require.NoError(t, err) -} - -func Test_ext_hashing_blake2_128_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - data := []byte("helloworld") - enc, err := scale.Marshal(data) - require.NoError(t, err) - - ret, err := inst.Exec("rtm_ext_hashing_blake2_128_version_1", enc) - require.NoError(t, err) - - var hash []byte - err = scale.Unmarshal(ret, &hash) - require.NoError(t, err) - - expected, err := common.Blake2b128(data) - require.NoError(t, err) - require.Equal(t, expected[:], hash) -} - -func Test_ext_hashing_blake2_256_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - data := []byte("helloworld") - enc, err := scale.Marshal(data) - require.NoError(t, err) - - ret, err := inst.Exec("rtm_ext_hashing_blake2_256_version_1", enc) - require.NoError(t, err) - - var hash []byte - err = scale.Unmarshal(ret, &hash) - require.NoError(t, err) - - expected, err := common.Blake2bHash(data) - require.NoError(t, err) - require.Equal(t, expected[:], hash) -} - -func Test_ext_hashing_keccak_256_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - data := []byte("helloworld") - enc, err := scale.Marshal(data) - require.NoError(t, err) - - ret, err := inst.Exec("rtm_ext_hashing_keccak_256_version_1", enc) - require.NoError(t, err) - - var hash []byte - err = scale.Unmarshal(ret, &hash) - require.NoError(t, err) - - expected, err := common.Keccak256(data) - require.NoError(t, err) - require.Equal(t, expected[:], hash) -} - -func Test_ext_hashing_twox_128_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - data := []byte("helloworld") - enc, err := scale.Marshal(data) - require.NoError(t, err) - - ret, err := inst.Exec("rtm_ext_hashing_twox_128_version_1", enc) - require.NoError(t, err) - - var hash []byte - err = scale.Unmarshal(ret, &hash) - require.NoError(t, err) - - expected, err := common.Twox128Hash(data) - require.NoError(t, err) - require.Equal(t, expected[:], hash) -} - -func Test_ext_hashing_twox_64_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - data := []byte("helloworld") - enc, err := scale.Marshal(data) - require.NoError(t, err) - - ret, err := inst.Exec("rtm_ext_hashing_twox_64_version_1", enc) - require.NoError(t, err) - - var hash []byte - err = scale.Unmarshal(ret, &hash) - require.NoError(t, err) - - expected, err := common.Twox64(data) - require.NoError(t, err) - require.Equal(t, expected[:], hash) -} - -func Test_ext_hashing_sha2_256_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - data := []byte("helloworld") - enc, err := scale.Marshal(data) - require.NoError(t, err) - - ret, err := inst.Exec("rtm_ext_hashing_sha2_256_version_1", enc) - require.NoError(t, err) - - var hash []byte - err = scale.Unmarshal(ret, &hash) - require.NoError(t, err) - - expected := common.Sha256(data) - require.Equal(t, expected[:], hash) -} - -func Test_ext_storage_clear_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - testkey := []byte("noot") - inst.ctx.Storage.Put(testkey, []byte{1}, trie.V0) - - enc, err := scale.Marshal(testkey) - require.NoError(t, err) - - _, err = inst.Exec("rtm_ext_storage_clear_version_1", enc) - require.NoError(t, err) - - val := inst.ctx.Storage.Get(testkey) - require.Nil(t, val) -} - -func Test_ext_offchain_local_storage_clear_version_1_Persistent(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - testkey := []byte("key1") - err := inst.NodeStorage().PersistentStorage.Put(testkey, []byte{1}) - require.NoError(t, err) - - kind := int32(1) - encKind, err := scale.Marshal(kind) - require.NoError(t, err) - - encKey, err := scale.Marshal(testkey) - require.NoError(t, err) - - _, err = inst.Exec("rtm_ext_offchain_local_storage_clear_version_1", append(encKind, encKey...)) - require.NoError(t, err) - - val, err := inst.NodeStorage().PersistentStorage.Get(testkey) - require.EqualError(t, err, "pebble: not found") - require.Nil(t, val) -} - -func Test_ext_offchain_local_storage_clear_version_1_Local(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - testkey := []byte("key1") - err := inst.NodeStorage().LocalStorage.Put(testkey, []byte{1}) - require.NoError(t, err) - - kind := int32(2) - encKind, err := scale.Marshal(kind) - require.NoError(t, err) - - encKey, err := scale.Marshal(testkey) - require.NoError(t, err) - - _, err = inst.Exec("rtm_ext_offchain_local_storage_clear_version_1", append(encKind, encKey...)) - require.NoError(t, err) - - val, err := inst.NodeStorage().LocalStorage.Get(testkey) - require.EqualError(t, err, "pebble: not found") - require.Nil(t, val) -} - -func Test_ext_offchain_http_request_start_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - encMethod, err := scale.Marshal([]byte("GET")) - require.NoError(t, err) - - encURI, err := scale.Marshal([]byte("https://chainsafe.io")) - require.NoError(t, err) - - var optMeta *[]byte - encMeta, err := scale.Marshal(optMeta) - require.NoError(t, err) - - params := append([]byte{}, encMethod...) - params = append(params, encURI...) - params = append(params, encMeta...) - - resReqID := scale.NewResult(int16(0), nil) - - // start request number 0 - ret, err := inst.Exec("rtm_ext_offchain_http_request_start_version_1", params) - require.NoError(t, err) - - err = scale.Unmarshal(ret, &resReqID) - require.NoError(t, err) - - requestNumber, err := resReqID.Unwrap() - require.NoError(t, err) - require.Equal(t, int16(1), requestNumber) - - // start request number 1 - ret, err = inst.Exec("rtm_ext_offchain_http_request_start_version_1", params) - require.NoError(t, err) - - resReqID = scale.NewResult(int16(0), nil) - - err = scale.Unmarshal(ret, &resReqID) - require.NoError(t, err) - - requestNumber, err = resReqID.Unwrap() - require.NoError(t, err) - require.Equal(t, int16(2), requestNumber) - - // start request number 2 - resReqID = scale.NewResult(int16(0), nil) - ret, err = inst.Exec("rtm_ext_offchain_http_request_start_version_1", params) - require.NoError(t, err) - - err = scale.Unmarshal(ret, &resReqID) - require.NoError(t, err) - - requestNumber, err = resReqID.Unwrap() - require.NoError(t, err) - require.Equal(t, int16(3), requestNumber) -} - -func Test_ext_offchain_http_request_add_header(t *testing.T) { - t.Parallel() - - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - cases := map[string]struct { - key, value string - expectedErr bool - }{ - "should_add_headers_without_problems": { - key: "SOME_HEADER_KEY", - value: "SOME_HEADER_VALUE", - expectedErr: false, - }, - - "should_return_a_result_error": { - key: "", - value: "", - expectedErr: true, - }, - } - - for tname, tcase := range cases { - tcase := tcase - t.Run(tname, func(t *testing.T) { - t.Parallel() - - reqID, err := inst.ctx.OffchainHTTPSet.StartRequest(http.MethodGet, "http://uri.example") - require.NoError(t, err) - - encID, err := scale.Marshal(uint32(reqID)) - require.NoError(t, err) - - encHeaderKey, err := scale.Marshal(tcase.key) - require.NoError(t, err) - - encHeaderValue, err := scale.Marshal(tcase.value) - require.NoError(t, err) - - params := append([]byte{}, encID...) - params = append(params, encHeaderKey...) - params = append(params, encHeaderValue...) - - ret, err := inst.Exec("rtm_ext_offchain_http_request_add_header_version_1", params) - require.NoError(t, err) - - gotResult := scale.NewResult(nil, nil) - err = scale.Unmarshal(ret, &gotResult) - require.NoError(t, err) - - ok, err := gotResult.Unwrap() - if tcase.expectedErr { - require.Error(t, err) - } else { - require.NoError(t, err) - } - - offchainReq := inst.ctx.OffchainHTTPSet.Get(reqID) - gotValue := offchainReq.Request.Header.Get(tcase.key) - require.Equal(t, tcase.value, gotValue) - - require.Nil(t, ok) - }) - } -} - -func Test_ext_storage_clear_prefix_version_1_hostAPI(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - testkey := []byte("static") - inst.ctx.Storage.Put(testkey, []byte("Inverse"), trie.V0) - - testkey2 := []byte("even-keeled") - inst.ctx.Storage.Put(testkey2, []byte("Future-proofed"), trie.V0) - - enc, err := scale.Marshal(testkey[:3]) - require.NoError(t, err) - - _, err = inst.Exec("rtm_ext_storage_clear_prefix_version_1", enc) - require.NoError(t, err) - - val := inst.ctx.Storage.Get(testkey) - require.Nil(t, val) - - val = inst.ctx.Storage.Get(testkey2) - require.NotNil(t, val) -} - -func Test_ext_storage_clear_prefix_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - testkey := []byte("noot") - inst.ctx.Storage.Put(testkey, []byte{1}, trie.V0) - - testkey2 := []byte("spaghet") - inst.ctx.Storage.Put(testkey2, []byte{2}, trie.V0) - - enc, err := scale.Marshal(testkey[:3]) - require.NoError(t, err) - - _, err = inst.Exec("rtm_ext_storage_clear_prefix_version_1", enc) - require.NoError(t, err) - - val := inst.ctx.Storage.Get(testkey) - require.Nil(t, val) - - val = inst.ctx.Storage.Get(testkey2) - require.NotNil(t, val) -} - -func Test_ext_storage_clear_prefix_version_2(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - testkey := []byte("noot") - inst.ctx.Storage.Put(testkey, []byte{1}, trie.V0) - - testkey2 := []byte("noot1") - inst.ctx.Storage.Put(testkey2, []byte{1}, trie.V0) - - testkey3 := []byte("noot2") - inst.ctx.Storage.Put(testkey3, []byte{1}, trie.V0) - - testkey4 := []byte("noot3") - inst.ctx.Storage.Put(testkey4, []byte{1}, trie.V0) - - testkey5 := []byte("spaghet") - testValue5 := []byte{2} - inst.ctx.Storage.Put(testkey5, testValue5, trie.V0) - - enc, err := scale.Marshal(testkey[:3]) - require.NoError(t, err) - - testLimit := uint32(2) - testLimitBytes := make([]byte, 4) - binary.LittleEndian.PutUint32(testLimitBytes, testLimit) - - optLimit, err := scale.Marshal(&testLimitBytes) - require.NoError(t, err) - - // clearing prefix for "noo" prefix with limit 2 - encValue, err := inst.Exec("rtm_ext_storage_clear_prefix_version_2", append(enc, optLimit...)) - require.NoError(t, err) - - var decVal []byte - scale.Unmarshal(encValue, &decVal) - - var numDeleted uint32 - // numDeleted represents no. of actual keys deleted - scale.Unmarshal(decVal[1:], &numDeleted) - require.Equal(t, uint32(2), numDeleted) - - var expectedAllDeleted byte - // expectedAllDeleted value 0 represents all keys deleted, 1 represents keys are pending with prefix in trie - expectedAllDeleted = 1 - require.Equal(t, expectedAllDeleted, decVal[0]) - - val := inst.ctx.Storage.Get(testkey) - require.NotNil(t, val) - - val = inst.ctx.Storage.Get(testkey5) - require.NotNil(t, val) - require.Equal(t, testValue5, val) - - // clearing prefix again for "noo" prefix with limit 2 - encValue, err = inst.Exec("rtm_ext_storage_clear_prefix_version_2", append(enc, optLimit...)) - require.NoError(t, err) - - scale.Unmarshal(encValue, &decVal) - scale.Unmarshal(decVal[1:], &numDeleted) - require.Equal(t, uint32(2), numDeleted) - - expectedAllDeleted = 0 - require.Equal(t, expectedAllDeleted, decVal[0]) - - val = inst.ctx.Storage.Get(testkey) - require.Nil(t, val) - - val = inst.ctx.Storage.Get(testkey5) - require.NotNil(t, val) - require.Equal(t, testValue5, val) -} - -func Test_ext_storage_get_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - testkey := []byte("noot") - testvalue := []byte{1, 2} - inst.ctx.Storage.Put(testkey, testvalue, trie.V0) - - enc, err := scale.Marshal(testkey) - require.NoError(t, err) - - ret, err := inst.Exec("rtm_ext_storage_get_version_1", enc) - require.NoError(t, err) - - var value *[]byte - err = scale.Unmarshal(ret, &value) - require.NoError(t, err) - require.NotNil(t, value) - require.Equal(t, testvalue, *value) -} - -func Test_ext_storage_exists_version_1(t *testing.T) { - t.Parallel() - - testCases := map[string]struct { - key []byte - value []byte // leave to nil to not insert pair - result byte - }{ - "value_does_not_exist": { - key: []byte{1}, - result: 0, - }, - "empty_value_exists": { - key: []byte{1}, - value: []byte{}, - result: 1, - }, - "value_exist": { - key: []byte{1}, - value: []byte{2}, - result: 1, - }, - } - - for name, testCase := range testCases { - testCase := testCase - t.Run(name, func(t *testing.T) { - t.Parallel() - instance := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - if testCase.value != nil { - instance.ctx.Storage.Put(testCase.key, testCase.value, trie.V0) - } - - encodedKey, err := scale.Marshal(testCase.key) - require.NoError(t, err) - - encodedResult, err := instance.Exec("rtm_ext_storage_exists_version_1", encodedKey) - require.NoError(t, err) - - var result byte - err = scale.Unmarshal(encodedResult, &result) - require.NoError(t, err) - - assert.Equal(t, testCase.result, result) - }) - } -} - -func Test_ext_storage_next_key_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - testkey := []byte("noot") - inst.ctx.Storage.Put(testkey, []byte{1}, trie.V0) - - nextkey := []byte("oot") - inst.ctx.Storage.Put(nextkey, []byte{1}, trie.V0) - - enc, err := scale.Marshal(testkey) - require.NoError(t, err) - - ret, err := inst.Exec("rtm_ext_storage_next_key_version_1", enc) - require.NoError(t, err) - - var next *[]byte - err = scale.Unmarshal(ret, &next) - require.NoError(t, err) - require.NotNil(t, next) - require.Equal(t, nextkey, *next) -} - -func Test_ext_storage_read_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - testkey := []byte("noot") - testvalue := []byte("washere") - inst.ctx.Storage.Put(testkey, testvalue, trie.V0) - - testoffset := uint32(2) - testBufferSize := uint32(100) - - encKey, err := scale.Marshal(testkey) - require.NoError(t, err) - encOffset, err := scale.Marshal(testoffset) - require.NoError(t, err) - encBufferSize, err := scale.Marshal(testBufferSize) - require.NoError(t, err) - - ret, err := inst.Exec("rtm_ext_storage_read_version_1", append(append(encKey, encOffset...), encBufferSize...)) - require.NoError(t, err) - - var read *[]byte - err = scale.Unmarshal(ret, &read) - require.NoError(t, err) - require.NotNil(t, read) - val := *read - require.Equal(t, testvalue[testoffset:], val[:len(testvalue)-int(testoffset)]) -} - -func Test_ext_storage_read_version_1_again(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - testkey := []byte("noot") - testvalue := []byte("_was_here_") - inst.ctx.Storage.Put(testkey, testvalue, trie.V0) - - testoffset := uint32(8) - testBufferSize := uint32(5) - - encKey, err := scale.Marshal(testkey) - require.NoError(t, err) - encOffset, err := scale.Marshal(testoffset) - require.NoError(t, err) - encBufferSize, err := scale.Marshal(testBufferSize) - require.NoError(t, err) - - ret, err := inst.Exec("rtm_ext_storage_read_version_1", append(append(encKey, encOffset...), encBufferSize...)) - require.NoError(t, err) - - var read *[]byte - err = scale.Unmarshal(ret, &read) - require.NoError(t, err) - - val := *read - require.Equal(t, len(testvalue)-int(testoffset), len(val)) - require.Equal(t, testvalue[testoffset:], val[:len(testvalue)-int(testoffset)]) -} - -func Test_ext_storage_read_version_1_OffsetLargerThanValue(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - testkey := []byte("noot") - testvalue := []byte("washere") - inst.ctx.Storage.Put(testkey, testvalue, trie.V0) - - testoffset := uint32(len(testvalue)) - testBufferSize := uint32(8) - - encKey, err := scale.Marshal(testkey) - require.NoError(t, err) - encOffset, err := scale.Marshal(testoffset) - require.NoError(t, err) - encBufferSize, err := scale.Marshal(testBufferSize) - require.NoError(t, err) - - ret, err := inst.Exec("rtm_ext_storage_read_version_1", append(append(encKey, encOffset...), encBufferSize...)) - require.NoError(t, err) - - var read *[]byte - err = scale.Unmarshal(ret, &read) - require.NoError(t, err) - require.NotNil(t, read) - val := *read - require.Equal(t, []byte{}, val) -} - -func Test_ext_storage_root_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - ret, err := inst.Exec("rtm_ext_storage_root_version_1", []byte{}) - require.NoError(t, err) - - var hash []byte - err = scale.Unmarshal(ret, &hash) - require.NoError(t, err) - - expected := trie.EmptyHash - require.Equal(t, expected[:], hash) -} - -func Test_ext_storage_set_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - testkey := []byte("noot") - testvalue := []byte("washere") - - encKey, err := scale.Marshal(testkey) - require.NoError(t, err) - encValue, err := scale.Marshal(testvalue) - require.NoError(t, err) - - _, err = inst.Exec("rtm_ext_storage_set_version_1", append(encKey, encValue...)) - require.NoError(t, err) - - val := inst.ctx.Storage.Get(testkey) - require.Equal(t, testvalue, val) -} - -func Test_ext_offline_index_set_version_1(t *testing.T) { - t.Parallel() - // TODO this currently fails with error could not find exported function, add rtm_ func to tester wasm (#1026) - t.Skip() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - testkey := []byte("noot") - testvalue := []byte("washere") - - encKey, err := scale.Marshal(testkey) - require.NoError(t, err) - encValue, err := scale.Marshal(testvalue) - require.NoError(t, err) - - _, err = inst.Exec("rtm_ext_offline_index_set_version_1", append(encKey, encValue...)) - require.NoError(t, err) - - val, err := inst.ctx.NodeStorage.PersistentStorage.Get(testkey) - require.NoError(t, err) - require.Equal(t, testvalue, val) -} - -func Test_ext_crypto_ed25519_generate_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - idData := []byte(keystore.AccoName) - ks, _ := inst.ctx.Keystore.GetKeystore(idData) - require.Equal(t, 0, ks.Size()) - - mnemonic, err := crypto.NewBIP39Mnemonic() - require.NoError(t, err) - - mnemonicBytes := []byte(mnemonic) - var data = &mnemonicBytes - seedData, err := scale.Marshal(data) - require.NoError(t, err) - - params := append(idData, seedData...) - - // we manually store and call the runtime function here since inst.exec assumes - // the data returned from the function is a pointer-size, but for ext_crypto_ed25519_generate_version_1, - // it's just a pointer - ptr, err := inst.ctx.Allocator.Allocate(uint32(len(params))) - require.NoError(t, err) - - memory := inst.vm.Memory.Data() - copy(memory[ptr:ptr+uint32(len(params))], params) - - dataLen := int32(len(params)) - - runtimeFunc, ok := inst.vm.Exports["rtm_ext_crypto_ed25519_generate_version_1"] - require.True(t, ok) - - ret, err := runtimeFunc(int32(ptr), dataLen) - require.NoError(t, err) - - mem := inst.vm.Memory.Data() - // this SCALE encoded, but it should just be a 32 byte buffer. may be due to way test runtime is written. - pubKeyBytes := mem[ret.ToI32()+1 : ret.ToI32()+1+32] - pubKey, err := ed25519.NewPublicKey(pubKeyBytes) - require.NoError(t, err) - - require.Equal(t, 1, ks.Size()) - kp := ks.GetKeypair(pubKey) - require.NotNil(t, kp) -} - -func Test_ext_crypto_ed25519_public_keys_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - idData := []byte(keystore.DumyName) - ks, _ := inst.ctx.Keystore.GetKeystore(idData) - require.Equal(t, 0, ks.Size()) - - size := 5 - pubKeys := make([][32]byte, size) - for i := range pubKeys { - kp, err := ed25519.GenerateKeypair() - require.NoError(t, err) - - ks.Insert(kp) - copy(pubKeys[i][:], kp.Public().Encode()) - } - - sort.Slice(pubKeys, func(i int, j int) bool { - return bytes.Compare(pubKeys[i][:], pubKeys[j][:]) < 0 - }) - - res, err := inst.Exec("rtm_ext_crypto_ed25519_public_keys_version_1", idData) - require.NoError(t, err) - - var out []byte - err = scale.Unmarshal(res, &out) - require.NoError(t, err) - - var ret [][32]byte - err = scale.Unmarshal(out, &ret) - require.NoError(t, err) - - sort.Slice(ret, func(i int, j int) bool { - return bytes.Compare(ret[i][:], ret[j][:]) < 0 - }) - - require.Equal(t, pubKeys, ret) -} - -func Test_ext_crypto_ed25519_sign_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - kp, err := ed25519.GenerateKeypair() - require.NoError(t, err) - - idData := []byte(keystore.AccoName) - ks, _ := inst.ctx.Keystore.GetKeystore(idData) - ks.Insert(kp) - - pubKeyData := kp.Public().Encode() - encPubKey, err := scale.Marshal(pubKeyData) - require.NoError(t, err) - - msgData := []byte("Hello world!") - encMsg, err := scale.Marshal(msgData) - require.NoError(t, err) - - res, err := inst.Exec("rtm_ext_crypto_ed25519_sign_version_1", append(append(idData, encPubKey...), encMsg...)) - require.NoError(t, err) - - var out []byte - err = scale.Unmarshal(res, &out) - require.NoError(t, err) - - var val *[64]byte - err = scale.Unmarshal(out, &val) - require.NoError(t, err) - require.NotNil(t, val) - - value := make([]byte, 64) - copy(value[:], val[:]) - - ok, err := kp.Public().Verify(msgData, value) - require.NoError(t, err) - require.True(t, ok) -} - -func Test_ext_crypto_ed25519_verify_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - kp, err := ed25519.GenerateKeypair() - require.NoError(t, err) - - idData := []byte(keystore.AccoName) - ks, _ := inst.ctx.Keystore.GetKeystore(idData) - ks.Insert(kp) - - pubKeyData := kp.Public().Encode() - encPubKey, err := scale.Marshal(pubKeyData) - require.NoError(t, err) - - msgData := []byte("Hello world!") - encMsg, err := scale.Marshal(msgData) - require.NoError(t, err) - - sign, err := kp.Private().Sign(msgData) - require.NoError(t, err) - encSign, err := scale.Marshal(sign) - require.NoError(t, err) - - ret, err := inst.Exec("rtm_ext_crypto_ed25519_verify_version_1", append(append(encSign, encMsg...), encPubKey...)) - require.NoError(t, err) - - var read *[]byte - err = scale.Unmarshal(ret, &read) - require.NoError(t, err) - require.NotNil(t, read) -} - -func Test_ext_crypto_ecdsa_verify_version_2(t *testing.T) { - t.Parallel() - - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - kp, err := secp256k1.GenerateKeypair() - require.NoError(t, err) - - pubKeyData := kp.Public().Encode() - encPubKey, err := scale.Marshal(pubKeyData) - require.NoError(t, err) - - msgData := []byte("Hello world!") - encMsg, err := scale.Marshal(msgData) - require.NoError(t, err) - - msgHash, err := common.Blake2bHash(msgData) - require.NoError(t, err) - - sig, err := kp.Private().Sign(msgHash[:]) - require.NoError(t, err) - - encSig, err := scale.Marshal(sig) - require.NoError(t, err) - - ret, err := inst.Exec("rtm_ext_crypto_ecdsa_verify_version_2", append(append(encSig, encMsg...), encPubKey...)) - require.NoError(t, err) - - var read *[]byte - err = scale.Unmarshal(ret, &read) - require.NoError(t, err) - - require.NotNil(t, read) -} - -func Test_ext_crypto_ecdsa_verify_version_2_Table(t *testing.T) { - t.Parallel() - testCases := map[string]struct { - sig []byte - msg []byte - key []byte - expected []byte - err error - }{ - "valid_signature": { - sig: []byte{5, 1, 187, 179, 88, 183, 46, 115, 242, 32, 9, 54, 141, 207, 44, 15, 238, 42, 217, 196, 111, 173, 239, 204, 128, 93, 49, 179, 137, 150, 162, 125, 226, 225, 28, 145, 122, 127, 15, 154, 185, 11, 3, 66, 27, 187, 204, 242, 107, 68, 26, 111, 245, 30, 115, 141, 85, 74, 158, 211, 161, 217, 43, 151, 120, 125, 1}, //nolint:lll - msg: []byte{48, 72, 101, 108, 108, 111, 32, 119, 111, 114, 108, 100, 33}, - key: []byte{132, 2, 39, 206, 55, 134, 131, 142, 43, 100, 63, 134, 96, 14, 253, 15, 222, 119, 154, 110, 188, 20, 159, 62, 125, 42, 59, 127, 19, 16, 0, 161, 236, 109}, //nolint:lll - expected: []byte{1, 0, 0, 0}, - }, - "invalid_signature": { - sig: []byte{5, 1, 187, 0, 0, 183, 46, 115, 242, 32, 9, 54, 141, 207, 44, 15, 238, 42, 217, 196, 111, 173, 239, 204, 128, 93, 49, 179, 137, 150, 162, 125, 226, 225, 28, 145, 122, 127, 15, 154, 185, 11, 3, 66, 27, 187, 204, 242, 107, 68, 26, 111, 245, 30, 115, 141, 85, 74, 158, 211, 161, 217, 43, 151, 120, 125, 1}, //nolint:lll - msg: []byte{48, 72, 101, 108, 108, 111, 32, 119, 111, 114, 108, 100, 33}, - key: []byte{132, 2, 39, 206, 55, 134, 131, 142, 43, 100, 63, 134, 96, 14, 253, 15, 222, 119, 154, 110, 188, 20, 159, 62, 125, 42, 59, 127, 19, 16, 0, 161, 236, 109}, //nolint:lll - expected: []byte{0, 0, 0, 0}, - }, - "wrong_key": { - sig: []byte{5, 1, 187, 0, 0, 183, 46, 115, 242, 32, 9, 54, 141, 207, 44, 15, 238, 42, 217, 196, 111, 173, 239, 204, 128, 93, 49, 179, 137, 150, 162, 125, 226, 225, 28, 145, 122, 127, 15, 154, 185, 11, 3, 66, 27, 187, 204, 242, 107, 68, 26, 111, 245, 30, 115, 141, 85, 74, 158, 211, 161, 217, 43, 151, 120, 125, 1}, //nolint:lll - msg: []byte{48, 72, 101, 108, 108, 111, 32, 119, 111, 114, 108, 100, 33}, - key: []byte{132, 2, 39, 0, 55, 134, 131, 142, 43, 100, 63, 134, 96, 14, 253, 15, 222, 119, 154, 110, 188, 20, 159, 62, 125, 42, 59, 127, 19, 16, 0, 161, 236, 109}, //nolint:lll - expected: []byte{0, 0, 0, 0}, - }, - "invalid_key": { - sig: []byte{5, 1, 187, 0, 0, 183, 46, 115, 242, 32, 9, 54, 141, 207, 44, 15, 238, 42, 217, 196, 111, 173, 239, 204, 128, 93, 49, 179, 137, 150, 162, 125, 226, 225, 28, 145, 122, 127, 15, 154, 185, 11, 3, 66, 27, 187, 204, 242, 107, 68, 26, 111, 245, 30, 115, 141, 85, 74, 158, 211, 161, 217, 43, 151, 120, 125, 1}, //nolint:lll - msg: []byte{48, 72, 101, 108, 108, 111, 32, 119, 111, 114, 108, 100, 33}, - key: []byte{132, 2, 39, 55, 134, 131, 142, 43, 100, 63, 134, 96, 14, 253, 15, 222, 119, 154, 110, 188, 20, 159, 62, 125, 42, 59, 127, 19, 16, 0, 161, 236, 109}, //nolint:lll - err: wasmer.NewExportedFunctionError( - "rtm_ext_crypto_ecdsa_verify_version_2", - "running runtime function: Failed to call the `%s` exported function."), - }, - "invalid_message": { - sig: []byte{5, 1, 187, 179, 88, 183, 46, 115, 242, 32, 9, 54, 141, 207, 44, 15, 238, 42, 217, 196, 111, 173, 239, 204, 128, 93, 49, 179, 137, 150, 162, 125, 226, 225, 28, 145, 122, 127, 15, 154, 185, 11, 3, 66, 27, 187, 204, 242, 107, 68, 26, 111, 245, 30, 115, 141, 85, 74, 158, 211, 161, 217, 43, 151, 120, 125, 1}, //nolint:lll - msg: []byte{48, 72, 101, 108, 108, 111, 32, 119, 111, 114, 108, 100}, - key: []byte{132, 2, 39, 206, 55, 134, 131, 142, 43, 100, 63, 134, 96, 14, 253, 15, 222, 119, 154, 110, 188, 20, 159, 62, 125, 42, 59, 127, 19, 16, 0, 161, 236, 109}, //nolint:lll - err: wasmer.NewExportedFunctionError( - "rtm_ext_crypto_ecdsa_verify_version_2", - "running runtime function: Failed to call the `%s` exported function."), - }, - } - for name, tc := range testCases { - tc := tc - t.Run(name, func(t *testing.T) { - t.Parallel() - - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - ret, err := inst.Exec("rtm_ext_crypto_ecdsa_verify_version_2", append(append(tc.sig, tc.msg...), tc.key...)) - assert.Equal(t, tc.expected, ret) - if tc.err != nil { - assert.EqualError(t, err, tc.err.Error()) - return - } - assert.NoError(t, err) - }) - } -} - -func Test_ext_crypto_sr25519_generate_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - idData := []byte(keystore.AccoName) - ks, _ := inst.ctx.Keystore.GetKeystore(idData) - require.Equal(t, 0, ks.Size()) - - mnemonic, err := crypto.NewBIP39Mnemonic() - require.NoError(t, err) - - mnemonicBytes := []byte(mnemonic) - var data = &mnemonicBytes - seedData, err := scale.Marshal(data) - require.NoError(t, err) - - params := append(idData, seedData...) - - ret, err := inst.Exec("rtm_ext_crypto_sr25519_generate_version_1", params) - require.NoError(t, err) - - var out []byte - err = scale.Unmarshal(ret, &out) - require.NoError(t, err) - - pubKey, err := ed25519.NewPublicKey(out) - require.NoError(t, err) - require.Equal(t, 1, ks.Size()) - - kp := ks.GetKeypair(pubKey) - require.NotNil(t, kp) -} - -func Test_ext_crypto_secp256k1_ecdsa_recover_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - msgData := []byte("Hello world!") - blakeHash, err := common.Blake2bHash(msgData) - require.NoError(t, err) - - kp, err := secp256k1.GenerateKeypair() - require.NoError(t, err) - - sigData, err := kp.Private().Sign(blakeHash.ToBytes()) - require.NoError(t, err) - - expectedPubKey := kp.Public().Encode() - - encSign, err := scale.Marshal(sigData) - require.NoError(t, err) - encMsg, err := scale.Marshal(blakeHash.ToBytes()) - require.NoError(t, err) - - ret, err := inst.Exec("rtm_ext_crypto_secp256k1_ecdsa_recover_version_1", append(encSign, encMsg...)) - require.NoError(t, err) - - var out []byte - err = scale.Unmarshal(ret, &out) - require.NoError(t, err) - - buf := &bytes.Buffer{} - buf.Write(out) - - uncomPubKey, err := new(types.Result).Decode(buf) - require.NoError(t, err) - rawPub := uncomPubKey.Value() - require.Equal(t, 64, len(rawPub)) - - publicKey := new(secp256k1.PublicKey) - - // Generates [33]byte compressed key from uncompressed [65]byte public key. - err = publicKey.UnmarshalPubkey(append([]byte{4}, rawPub...)) - require.NoError(t, err) - require.Equal(t, expectedPubKey, publicKey.Encode()) -} - -func Test_ext_crypto_secp256k1_ecdsa_recover_compressed_version_1(t *testing.T) { - t.Parallel() - t.Skip("host API tester does not yet contain rtm_ext_crypto_secp256k1_ecdsa_recover_compressed_version_1") - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - msgData := []byte("Hello world!") - blakeHash, err := common.Blake2bHash(msgData) - require.NoError(t, err) - - kp, err := secp256k1.GenerateKeypair() - require.NoError(t, err) - - sigData, err := kp.Private().Sign(blakeHash.ToBytes()) - require.NoError(t, err) - - expectedPubKey := kp.Public().Encode() - - encSign, err := scale.Marshal(sigData) - require.NoError(t, err) - encMsg, err := scale.Marshal(blakeHash.ToBytes()) - require.NoError(t, err) - - ret, err := inst.Exec("rtm_ext_crypto_secp256k1_ecdsa_recover_compressed_version_1", append(encSign, encMsg...)) - require.NoError(t, err) - - var out []byte - err = scale.Unmarshal(ret, &out) - require.NoError(t, err) - - buf := &bytes.Buffer{} - buf.Write(out) - - uncomPubKey, err := new(types.Result).Decode(buf) - require.NoError(t, err) - rawPub := uncomPubKey.Value() - require.Equal(t, 33, len(rawPub)) - - publicKey := new(secp256k1.PublicKey) - - err = publicKey.Decode(rawPub) - require.NoError(t, err) - require.Equal(t, expectedPubKey, publicKey.Encode()) -} - -func Test_ext_crypto_sr25519_public_keys_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - idData := []byte(keystore.DumyName) - ks, _ := inst.ctx.Keystore.GetKeystore(idData) - require.Equal(t, 0, ks.Size()) - - const size = 5 - pubKeys := make([][32]byte, size) - for i := range pubKeys { - kp, err := sr25519.GenerateKeypair() - require.NoError(t, err) - - ks.Insert(kp) - copy(pubKeys[i][:], kp.Public().Encode()) - } - - sort.Slice(pubKeys, func(i int, j int) bool { - return bytes.Compare(pubKeys[i][:], pubKeys[j][:]) < 0 - }) - - res, err := inst.Exec("rtm_ext_crypto_sr25519_public_keys_version_1", idData) - require.NoError(t, err) - - var out []byte - err = scale.Unmarshal(res, &out) - require.NoError(t, err) - - var ret [][32]byte - err = scale.Unmarshal(out, &ret) - require.NoError(t, err) - - sort.Slice(ret, func(i int, j int) bool { - return bytes.Compare(ret[i][:], ret[j][:]) < 0 - }) - - require.Equal(t, pubKeys, ret) -} - -func Test_ext_crypto_sr25519_sign_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - kp, err := sr25519.GenerateKeypair() - require.NoError(t, err) - - idData := []byte(keystore.AccoName) - ks, _ := inst.ctx.Keystore.GetKeystore(idData) - require.Equal(t, 0, ks.Size()) - - ks.Insert(kp) - - pubKeyData := kp.Public().Encode() - encPubKey, err := scale.Marshal(pubKeyData) - require.NoError(t, err) - - msgData := []byte("Hello world!") - encMsg, err := scale.Marshal(msgData) - require.NoError(t, err) - - res, err := inst.Exec("rtm_ext_crypto_sr25519_sign_version_1", append(append(idData, encPubKey...), encMsg...)) - require.NoError(t, err) - - var out []byte - err = scale.Unmarshal(res, &out) - require.NoError(t, err) - - var val *[64]byte - err = scale.Unmarshal(out, &val) - require.NoError(t, err) - require.NotNil(t, val) - - value := make([]byte, 64) - copy(value[:], val[:]) - - ok, err := kp.Public().Verify(msgData, value) - require.NoError(t, err) - require.True(t, ok) -} - -func Test_ext_crypto_sr25519_verify_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - kp, err := sr25519.GenerateKeypair() - require.NoError(t, err) - - idData := []byte(keystore.AccoName) - ks, _ := inst.ctx.Keystore.GetKeystore(idData) - require.Equal(t, 0, ks.Size()) - - pubKeyData := kp.Public().Encode() - encPubKey, err := scale.Marshal(pubKeyData) - require.NoError(t, err) - - msgData := []byte("Hello world!") - encMsg, err := scale.Marshal(msgData) - require.NoError(t, err) - - sign, err := kp.Private().Sign(msgData) - require.NoError(t, err) - encSign, err := scale.Marshal(sign) - require.NoError(t, err) - - ret, err := inst.Exec("rtm_ext_crypto_sr25519_verify_version_1", append(append(encSign, encMsg...), encPubKey...)) - require.NoError(t, err) - - var read *[]byte - err = scale.Unmarshal(ret, &read) - require.NoError(t, err) - require.NotNil(t, read) -} - -func Test_ext_default_child_storage_read_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - err := inst.ctx.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) - require.NoError(t, err) - - err = inst.ctx.Storage.SetChildStorage(testChildKey, testKey, testValue) - require.NoError(t, err) - - testOffset := uint32(2) - testBufferSize := uint32(100) - - encChildKey, err := scale.Marshal(testChildKey) - require.NoError(t, err) - - encKey, err := scale.Marshal(testKey) - require.NoError(t, err) - - encBufferSize, err := scale.Marshal(testBufferSize) - require.NoError(t, err) - - encOffset, err := scale.Marshal(testOffset) - require.NoError(t, err) - - ret, err := inst.Exec( - "rtm_ext_default_child_storage_read_version_1", - append(append(encChildKey, encKey...), - append(encOffset, encBufferSize...)...)) - require.NoError(t, err) - - var read *[]byte - err = scale.Unmarshal(ret, &read) - require.NoError(t, err) - require.NotNil(t, read) - - val := *read - require.Equal(t, testValue[testOffset:], val[:len(testValue)-int(testOffset)]) -} - -func Test_ext_default_child_storage_clear_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - err := inst.ctx.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) - require.NoError(t, err) - - err = inst.ctx.Storage.SetChildStorage(testChildKey, testKey, testValue) - require.NoError(t, err) - - // Confirm if value is set - val, err := inst.ctx.Storage.GetChildStorage(testChildKey, testKey) - require.NoError(t, err) - require.Equal(t, testValue, val) - - encChildKey, err := scale.Marshal(testChildKey) - require.NoError(t, err) - - encKey, err := scale.Marshal(testKey) - require.NoError(t, err) - - _, err = inst.Exec("rtm_ext_default_child_storage_clear_version_1", append(encChildKey, encKey...)) - require.NoError(t, err) - - val, err = inst.ctx.Storage.GetChildStorage(testChildKey, testKey) - require.NoError(t, err) - require.Nil(t, val) -} - -func Test_ext_default_child_storage_clear_prefix_version_2(t *testing.T) { - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - prefix := []byte("key") - - testKeyValuePair := []struct { - key []byte - value []byte - }{ - {[]byte("keyOne"), []byte("value1")}, - {[]byte("keyTwo"), []byte("value2")}, - {[]byte("keyThree"), []byte("value3")}, - } - - err := inst.ctx.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) - require.NoError(t, err) - - for _, kv := range testKeyValuePair { - err = inst.ctx.Storage.SetChildStorage(testChildKey, kv.key, kv.value) - require.NoError(t, err) - } - - // Confirm if value is set - keys, err := inst.ctx.Storage.(*storage.TrieState).GetKeysWithPrefixFromChild(testChildKey, prefix) - require.NoError(t, err) - require.Equal(t, 3, len(keys)) - - encChildKey, err := scale.Marshal(testChildKey) - require.NoError(t, err) - - encPrefix, err := scale.Marshal(prefix) - require.NoError(t, err) - - testLimit := uint32(1) - testLimitBytes := make([]byte, 4) - binary.LittleEndian.PutUint32(testLimitBytes, testLimit) - - encLimit, err := scale.Marshal(&testLimitBytes) - require.NoError(t, err) - - data := append(encChildKey, encPrefix...) - data = append(data, encLimit...) - - _, err = inst.Exec("rtm_ext_default_child_storage_clear_prefix_version_2", data) - require.NoError(t, err) - - keys, err = inst.ctx.Storage.(*storage.TrieState).GetKeysWithPrefixFromChild(testChildKey, prefix) - require.NoError(t, err) - // since one key is removed, there will be two remaining. - require.Equal(t, 2, len(keys)) -} - -func Test_ext_default_child_storage_clear_prefix_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - prefix := []byte("key") - - testKeyValuePair := []struct { - key []byte - value []byte - }{ - {[]byte("keyOne"), []byte("value1")}, - {[]byte("keyTwo"), []byte("value2")}, - {[]byte("keyThree"), []byte("value3")}, - } - - err := inst.ctx.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) - require.NoError(t, err) - - for _, kv := range testKeyValuePair { - err = inst.ctx.Storage.SetChildStorage(testChildKey, kv.key, kv.value) - require.NoError(t, err) - } - - // Confirm if value is set - keys, err := inst.ctx.Storage.(*storage.TrieState).GetKeysWithPrefixFromChild(testChildKey, prefix) - require.NoError(t, err) - require.Equal(t, 3, len(keys)) - - encChildKey, err := scale.Marshal(testChildKey) - require.NoError(t, err) - - encPrefix, err := scale.Marshal(prefix) - require.NoError(t, err) - - _, err = inst.Exec("rtm_ext_default_child_storage_clear_prefix_version_1", append(encChildKey, encPrefix...)) - require.NoError(t, err) - - keys, err = inst.ctx.Storage.(*storage.TrieState).GetKeysWithPrefixFromChild(testChildKey, prefix) - require.NoError(t, err) - require.Equal(t, 0, len(keys)) -} - -func Test_ext_default_child_storage_exists_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - err := inst.ctx.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) - require.NoError(t, err) - - err = inst.ctx.Storage.SetChildStorage(testChildKey, testKey, testValue) - require.NoError(t, err) - - encChildKey, err := scale.Marshal(testChildKey) - require.NoError(t, err) - - encKey, err := scale.Marshal(testKey) - require.NoError(t, err) - - ret, err := inst.Exec("rtm_ext_default_child_storage_exists_version_1", append(encChildKey, encKey...)) - require.NoError(t, err) - - var read *[]byte - err = scale.Unmarshal(ret, &read) - require.NoError(t, err) - require.NotNil(t, read) -} - -func Test_ext_default_child_storage_get_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - err := inst.ctx.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) - require.NoError(t, err) - - err = inst.ctx.Storage.SetChildStorage(testChildKey, testKey, testValue) - require.NoError(t, err) - - encChildKey, err := scale.Marshal(testChildKey) - require.NoError(t, err) - - encKey, err := scale.Marshal(testKey) - require.NoError(t, err) - - ret, err := inst.Exec("rtm_ext_default_child_storage_get_version_1", append(encChildKey, encKey...)) - require.NoError(t, err) - - var read *[]byte - err = scale.Unmarshal(ret, &read) - require.NoError(t, err) - require.NotNil(t, read) -} - -func Test_ext_default_child_storage_next_key_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - testKeyValuePair := []struct { - key []byte - value []byte - }{ - {[]byte("apple"), []byte("value1")}, - {[]byte("key"), []byte("value2")}, - } - - key := testKeyValuePair[0].key - - err := inst.ctx.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) - require.NoError(t, err) - - for _, kv := range testKeyValuePair { - err = inst.ctx.Storage.SetChildStorage(testChildKey, kv.key, kv.value) - require.NoError(t, err) - } - - encChildKey, err := scale.Marshal(testChildKey) - require.NoError(t, err) - - encKey, err := scale.Marshal(key) - require.NoError(t, err) - - ret, err := inst.Exec("rtm_ext_default_child_storage_next_key_version_1", append(encChildKey, encKey...)) - require.NoError(t, err) - - var read *[]byte - err = scale.Unmarshal(ret, &read) - require.NoError(t, err) - require.NotNil(t, read) - require.Equal(t, testKeyValuePair[1].key, *read) -} - -func Test_ext_default_child_storage_root_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - err := inst.ctx.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) - require.NoError(t, err) - - err = inst.ctx.Storage.SetChildStorage(testChildKey, testKey, testValue) - require.NoError(t, err) - - child, err := inst.ctx.Storage.GetChild(testChildKey) - require.NoError(t, err) - - rootHash, err := child.Hash() - require.NoError(t, err) - - encChildKey, err := scale.Marshal(testChildKey) - require.NoError(t, err) - encKey, err := scale.Marshal(testKey) - require.NoError(t, err) - - ret, err := inst.Exec("rtm_ext_default_child_storage_root_version_1", append(encChildKey, encKey...)) - require.NoError(t, err) - - var hash []byte - err = scale.Unmarshal(ret, &hash) - require.NoError(t, err) - - // Convert decoded interface to common Hash - actualValue := common.BytesToHash(hash) - require.Equal(t, rootHash, actualValue) -} - -func Test_ext_default_child_storage_set_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - err := inst.ctx.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) - require.NoError(t, err) - - // Check if value is not set - val, err := inst.ctx.Storage.GetChildStorage(testChildKey, testKey) - require.NoError(t, err) - require.Nil(t, val) - - encChildKey, err := scale.Marshal(testChildKey) - require.NoError(t, err) - - encKey, err := scale.Marshal(testKey) - require.NoError(t, err) - - encVal, err := scale.Marshal(testValue) - require.NoError(t, err) - - _, err = inst.Exec("rtm_ext_default_child_storage_set_version_1", append(append(encChildKey, encKey...), encVal...)) - require.NoError(t, err) - - val, err = inst.ctx.Storage.GetChildStorage(testChildKey, testKey) - require.NoError(t, err) - require.Equal(t, testValue, val) -} - -func Test_ext_default_child_storage_storage_kill_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - err := inst.ctx.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) - require.NoError(t, err) - - // Confirm if value is set - child, err := inst.ctx.Storage.GetChild(testChildKey) - require.NoError(t, err) - require.NotNil(t, child) - - encChildKey, err := scale.Marshal(testChildKey) - require.NoError(t, err) - - _, err = inst.Exec("rtm_ext_default_child_storage_storage_kill_version_1", encChildKey) - require.NoError(t, err) - - child, _ = inst.ctx.Storage.GetChild(testChildKey) - require.Nil(t, child) -} - -func Test_ext_default_child_storage_storage_kill_version_2_limit_all(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) - tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) - err := inst.ctx.Storage.SetChild(testChildKey, tr) - require.NoError(t, err) - - // Confirm if value is set - child, err := inst.ctx.Storage.GetChild(testChildKey) - require.NoError(t, err) - require.NotNil(t, child) - - encChildKey, err := scale.Marshal(testChildKey) - require.NoError(t, err) - - testLimit := uint32(2) - testLimitBytes := make([]byte, 4) - binary.LittleEndian.PutUint32(testLimitBytes, testLimit) - - optLimit, err := scale.Marshal(&testLimitBytes) - require.NoError(t, err) - - res, err := inst.Exec("rtm_ext_default_child_storage_storage_kill_version_2", append(encChildKey, optLimit...)) - require.NoError(t, err) - require.Equal(t, []byte{1, 0, 0, 0}, res) - - child, err = inst.ctx.Storage.GetChild(testChildKey) - require.NoError(t, err) - require.Empty(t, child.Entries()) -} - -func Test_ext_default_child_storage_storage_kill_version_2_limit_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) - tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) - err := inst.ctx.Storage.SetChild(testChildKey, tr) - require.NoError(t, err) - - // Confirm if value is set - child, err := inst.ctx.Storage.GetChild(testChildKey) - require.NoError(t, err) - require.NotNil(t, child) - - encChildKey, err := scale.Marshal(testChildKey) - require.NoError(t, err) - - testLimit := uint32(1) - testLimitBytes := make([]byte, 4) - binary.LittleEndian.PutUint32(testLimitBytes, testLimit) - - optLimit, err := scale.Marshal(&testLimitBytes) - require.NoError(t, err) - - res, err := inst.Exec("rtm_ext_default_child_storage_storage_kill_version_2", append(encChildKey, optLimit...)) - require.NoError(t, err) - require.Equal(t, []byte{0, 0, 0, 0}, res) - - child, err = inst.ctx.Storage.GetChild(testChildKey) - require.NoError(t, err) - require.Equal(t, 1, len(child.Entries())) -} - -func Test_ext_default_child_storage_storage_kill_version_2_limit_none(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) - tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) - err := inst.ctx.Storage.SetChild(testChildKey, tr) - require.NoError(t, err) - - // Confirm if value is set - child, err := inst.ctx.Storage.GetChild(testChildKey) - require.NoError(t, err) - require.NotNil(t, child) - - encChildKey, err := scale.Marshal(testChildKey) - require.NoError(t, err) - - var val *[]byte - optLimit, err := scale.Marshal(val) - require.NoError(t, err) - - res, err := inst.Exec("rtm_ext_default_child_storage_storage_kill_version_2", append(encChildKey, optLimit...)) - require.NoError(t, err) - require.Equal(t, []byte{1, 0, 0, 0}, res) - - child, err = inst.ctx.Storage.GetChild(testChildKey) - require.Error(t, err) - require.Nil(t, child) -} - -func Test_ext_default_child_storage_storage_kill_version_3(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) - tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) - tr.Put([]byte(`key3`), []byte(`value3`), trie.V0) - err := inst.ctx.Storage.SetChild(testChildKey, tr) - require.NoError(t, err) - - testLimitBytes := make([]byte, 4) - binary.LittleEndian.PutUint32(testLimitBytes, uint32(2)) - optLimit2 := &testLimitBytes - - testCases := []struct { - key []byte - limit *[]byte - expected []byte - errMsg string - }{ - { - key: []byte(`fakekey`), - limit: optLimit2, - expected: []byte{0, 0, 0, 0, 0}, - errMsg: "running runtime function: " + - "Failed to call the `rtm_ext_default_child_storage_storage_kill_version_3` exported function.", - }, - {key: testChildKey, limit: optLimit2, expected: []byte{1, 2, 0, 0, 0}}, - {key: testChildKey, limit: nil, expected: []byte{0, 1, 0, 0, 0}}, - } - - for _, test := range testCases { - encChildKey, err := scale.Marshal(test.key) - require.NoError(t, err) - encOptLimit, err := scale.Marshal(test.limit) - require.NoError(t, err) - res, err := inst.Exec("rtm_ext_default_child_storage_storage_kill_version_3", append(encChildKey, encOptLimit...)) - if test.errMsg != "" { - require.Error(t, err) - require.EqualError(t, err, test.errMsg) - continue - } - require.NoError(t, err) - - var read *[]byte - err = scale.Unmarshal(res, &read) - require.NoError(t, err) - require.NotNil(t, read) - require.Equal(t, test.expected, *read) - } -} - -func Test_ext_storage_append_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - testkey := []byte("noot") - testvalue := []byte("was") - testvalueAppend := []byte("here") - - encKey, err := scale.Marshal(testkey) - require.NoError(t, err) - encVal, err := scale.Marshal(testvalue) - require.NoError(t, err) - doubleEncVal, err := scale.Marshal(encVal) - require.NoError(t, err) - - encArr, err := scale.Marshal([][]byte{testvalue}) - require.NoError(t, err) - - // place SCALE encoded value in storage - _, err = inst.Exec("rtm_ext_storage_append_version_1", append(encKey, doubleEncVal...)) - require.NoError(t, err) - - val := inst.ctx.Storage.Get(testkey) - require.Equal(t, encArr, val) - - encValueAppend, err := scale.Marshal(testvalueAppend) - require.NoError(t, err) - doubleEncValueAppend, err := scale.Marshal(encValueAppend) - require.NoError(t, err) - - _, err = inst.Exec("rtm_ext_storage_append_version_1", append(encKey, doubleEncValueAppend...)) - require.NoError(t, err) - - ret := inst.ctx.Storage.Get(testkey) - require.NotNil(t, ret) - - var res [][]byte - err = scale.Unmarshal(ret, &res) - require.NoError(t, err) - - require.Equal(t, 2, len(res)) - require.Equal(t, testvalue, res[0]) - require.Equal(t, testvalueAppend, res[1]) - - expected, err := scale.Marshal([][]byte{testvalue, testvalueAppend}) - require.NoError(t, err) - require.Equal(t, expected, ret) -} - -func Test_ext_storage_append_version_1_again(t *testing.T) { - t.Parallel() - DefaultTestLogLvl = 5 - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - testkey := []byte("noot") - testvalue := []byte("abc") - testvalueAppend := []byte("def") - - encKey, err := scale.Marshal(testkey) - require.NoError(t, err) - encVal, err := scale.Marshal(testvalue) - require.NoError(t, err) - doubleEncVal, err := scale.Marshal(encVal) - require.NoError(t, err) - - encArr, err := scale.Marshal([][]byte{testvalue}) - require.NoError(t, err) - - // place SCALE encoded value in storage - _, err = inst.Exec("rtm_ext_storage_append_version_1", append(encKey, doubleEncVal...)) - require.NoError(t, err) - - val := inst.ctx.Storage.Get(testkey) - require.Equal(t, encArr, val) - - encValueAppend, err := scale.Marshal(testvalueAppend) - require.NoError(t, err) - doubleEncValueAppend, err := scale.Marshal(encValueAppend) - require.NoError(t, err) - - _, err = inst.Exec("rtm_ext_storage_append_version_1", append(encKey, doubleEncValueAppend...)) - require.NoError(t, err) - - ret := inst.ctx.Storage.Get(testkey) - require.NotNil(t, ret) - - var res [][]byte - err = scale.Unmarshal(ret, &res) - require.NoError(t, err) - - require.Equal(t, 2, len(res)) - require.Equal(t, testvalue, res[0]) - require.Equal(t, testvalueAppend, res[1]) - - expected, err := scale.Marshal([][]byte{testvalue, testvalueAppend}) - require.NoError(t, err) - require.Equal(t, expected, ret) -} - -func Test_ext_trie_blake2_256_ordered_root_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - testvalues := []string{"static", "even-keeled", "Future-proofed"} - encValues, err := scale.Marshal(testvalues) - require.NoError(t, err) - - res, err := inst.Exec("rtm_ext_trie_blake2_256_ordered_root_version_1", encValues) - require.NoError(t, err) - - var hash []byte - err = scale.Unmarshal(res, &hash) - require.NoError(t, err) - - expected := common.MustHexToHash("0xd847b86d0219a384d11458e829e9f4f4cce7e3cc2e6dcd0e8a6ad6f12c64a737") - require.Equal(t, expected[:], hash) -} - -func Test_ext_trie_blake2_256_root_version_1(t *testing.T) { - t.Parallel() - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - testinput := []string{"noot", "was", "here", "??"} - encInput, err := scale.Marshal(testinput) - require.NoError(t, err) - encInput[0] = encInput[0] >> 1 - - res, err := inst.Exec("rtm_ext_trie_blake2_256_root_version_1", encInput) - require.NoError(t, err) - - var hash []byte - err = scale.Unmarshal(res, &hash) - require.NoError(t, err) - - tt := trie.NewEmptyTrie() - tt.Put([]byte("noot"), []byte("was"), trie.V0) - tt.Put([]byte("here"), []byte("??"), trie.V0) - - expected := tt.MustHash() - require.Equal(t, expected[:], hash) -} - -func Test_ext_trie_blake2_256_verify_proof_version_1(t *testing.T) { - t.Parallel() - - tmp := t.TempDir() - - memdb, err := database.NewPebble(tmp, true) - require.NoError(t, err) - - otherTrie := trie.NewEmptyTrie() - otherTrie.Put([]byte("simple"), []byte("cat"), trie.V0) - - otherHash, err := otherTrie.Hash() - require.NoError(t, err) - - tr := trie.NewEmptyTrie() - tr.Put([]byte("do"), []byte("verb"), trie.V0) - tr.Put([]byte("domain"), []byte("website"), trie.V0) - tr.Put([]byte("other"), []byte("random"), trie.V0) - tr.Put([]byte("otherwise"), []byte("randomstuff"), trie.V0) - tr.Put([]byte("cat"), []byte("another animal"), trie.V0) - - err = tr.WriteDirty(memdb) - require.NoError(t, err) - - hash, err := tr.Hash() - require.NoError(t, err) - - keys := [][]byte{ - []byte("do"), - []byte("domain"), - []byte("other"), - []byte("otherwise"), - []byte("cat"), - } - - root := hash.ToBytes() - otherRoot := otherHash.ToBytes() - - allProofs, err := proof.Generate(root, keys, memdb) - require.NoError(t, err) - - testcases := map[string]struct { - root, key, value []byte - proof [][]byte - expect bool - }{ - "Proof_should_be_true": { - root: root, key: []byte("do"), proof: allProofs, value: []byte("verb"), expect: true}, - "Root_empty,_proof_should_be_false": { - root: []byte{}, key: []byte("do"), proof: allProofs, value: []byte("verb"), expect: false}, - "Other_root,_proof_should_be_false": { - root: otherRoot, key: []byte("do"), proof: allProofs, value: []byte("verb"), expect: false}, - "Value_empty,_proof_should_be_true": { - root: root, key: []byte("do"), proof: allProofs, value: nil, expect: true}, - "Unknow_key,_proof_should_be_false": { - root: root, key: []byte("unknow"), proof: allProofs, value: nil, expect: false}, - "Key_and_value_unknow,_proof_should_be_false": { - root: root, key: []byte("unknow"), proof: allProofs, value: []byte("unknow"), expect: false}, - "Empty_proof,_should_be_false": { - root: root, key: []byte("do"), proof: [][]byte{}, value: nil, expect: false}, - } - - inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - - for name, testcase := range testcases { - testcase := testcase - t.Run(name, func(t *testing.T) { - t.Parallel() - - hashEnc, err := scale.Marshal(testcase.root) - require.NoError(t, err) - - args := hashEnc - - encProof, err := scale.Marshal(testcase.proof) - require.NoError(t, err) - args = append(args, encProof...) - - keyEnc, err := scale.Marshal(testcase.key) - require.NoError(t, err) - args = append(args, keyEnc...) - - valueEnc, err := scale.Marshal(testcase.value) - require.NoError(t, err) - args = append(args, valueEnc...) - - res, err := inst.Exec("rtm_ext_trie_blake2_256_verify_proof_version_1", args) - require.NoError(t, err) - - var got bool - err = scale.Unmarshal(res, &got) - require.NoError(t, err) - require.Equal(t, testcase.expect, got) - }) - } -} From 644dc20a601b71a230c1763b389fd09dce1a82c3 Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 17 Aug 2023 13:15:34 -0300 Subject: [PATCH 045/128] Version child tries functions --- .../modules/childstate_integration_test.go | 2 +- dot/rpc/modules/childstate_test.go | 2 +- dot/rpc/modules/state_integration_test.go | 2 +- dot/state/storage_test.go | 2 +- lib/runtime/interfaces.go | 4 +- lib/runtime/storage/trie.go | 8 ++-- lib/runtime/storage/trie_test.go | 4 +- lib/runtime/wazero/imports.go | 2 +- lib/runtime/wazero/imports_test.go | 44 +++++++++---------- lib/trie/child_storage.go | 10 ++--- lib/trie/child_storage_test.go | 8 ++-- lib/trie/database_test.go | 2 +- 12 files changed, 45 insertions(+), 45 deletions(-) diff --git a/dot/rpc/modules/childstate_integration_test.go b/dot/rpc/modules/childstate_integration_test.go index 119e20df53..c7bc986adf 100644 --- a/dot/rpc/modules/childstate_integration_test.go +++ b/dot/rpc/modules/childstate_integration_test.go @@ -252,7 +252,7 @@ func setupChildStateStorage(t *testing.T) (*ChildStateModule, common.Hash) { childTr.Put([]byte(":child_second"), []byte(":child_second_value"), trie.V0) childTr.Put([]byte(":another_child"), []byte("value"), trie.V0) - err = tr.SetChild([]byte(":child_storage_key"), childTr) + err = tr.SetChild([]byte(":child_storage_key"), childTr, trie.V0) require.NoError(t, err) stateRoot, err := tr.Root() diff --git a/dot/rpc/modules/childstate_test.go b/dot/rpc/modules/childstate_test.go index 533e6cb57e..8cbae5cf8b 100644 --- a/dot/rpc/modules/childstate_test.go +++ b/dot/rpc/modules/childstate_test.go @@ -32,7 +32,7 @@ func createTestTrieState(t *testing.T) (*trie.Trie, common.Hash) { childTr.Put([]byte(":child_second"), []byte(":child_second_value"), trie.V0) childTr.Put([]byte(":another_child"), []byte("value"), trie.V0) - err := tr.SetChild([]byte(":child_storage_key"), childTr) + err := tr.SetChild([]byte(":child_storage_key"), childTr, trie.V0) require.NoError(t, err) stateRoot, err := tr.Root() diff --git a/dot/rpc/modules/state_integration_test.go b/dot/rpc/modules/state_integration_test.go index 7dff2167cc..20a92ad0c1 100644 --- a/dot/rpc/modules/state_integration_test.go +++ b/dot/rpc/modules/state_integration_test.go @@ -554,7 +554,7 @@ func setupStateModule(t *testing.T) (*StateModule, *common.Hash, *common.Hash) { ts.Put([]byte(`:key2`), []byte(`value2`), trie.V0) ts.Put([]byte(`:key1`), []byte(`value1`), trie.V0) - ts.SetChildStorage([]byte(`:child1`), []byte(`:key1`), []byte(`:childValue1`)) + ts.SetChildStorage([]byte(`:child1`), []byte(`:key1`), []byte(`:childValue1`), trie.V0) sr1, err := ts.Root() require.NoError(t, err) diff --git a/dot/state/storage_test.go b/dot/state/storage_test.go index 82ae0ea7fb..5cf60b81e3 100644 --- a/dot/state/storage_test.go +++ b/dot/state/storage_test.go @@ -190,7 +190,7 @@ func TestGetStorageChildAndGetStorageFromChild(t *testing.T) { testChildTrie.Put([]byte("keyInsidechild"), []byte("voila"), trie.V0) - err = genTrie.SetChild([]byte("keyToChild"), testChildTrie) + err = genTrie.SetChild([]byte("keyToChild"), testChildTrie, trie.V0) require.NoError(t, err) tries := newTriesEmpty() diff --git a/lib/runtime/interfaces.go b/lib/runtime/interfaces.go index 9a46e5a650..6693e52c06 100644 --- a/lib/runtime/interfaces.go +++ b/lib/runtime/interfaces.go @@ -14,8 +14,8 @@ type Storage interface { Put(key []byte, value []byte, version trie.Version) (err error) Get(key []byte) []byte Root() (common.Hash, error) - SetChild(keyToChild []byte, child *trie.Trie) error - SetChildStorage(keyToChild, key, value []byte) error + SetChild(keyToChild []byte, child *trie.Trie, version trie.Version) error + SetChildStorage(keyToChild, key, value []byte, version trie.Version) error GetChildStorage(keyToChild, key []byte) ([]byte, error) Delete(key []byte) (err error) DeleteChild(keyToChild []byte) (err error) diff --git a/lib/runtime/storage/trie.go b/lib/runtime/storage/trie.go index 0c6f1a9619..35690efc2c 100644 --- a/lib/runtime/storage/trie.go +++ b/lib/runtime/storage/trie.go @@ -146,17 +146,17 @@ func (s *TrieState) TrieEntries() map[string][]byte { } // SetChild sets the child trie at the given key -func (s *TrieState) SetChild(keyToChild []byte, child *trie.Trie) error { +func (s *TrieState) SetChild(keyToChild []byte, child *trie.Trie, version trie.Version) error { s.lock.Lock() defer s.lock.Unlock() - return s.t.SetChild(keyToChild, child) + return s.t.SetChild(keyToChild, child, version) } // SetChildStorage sets a key-value pair in a child trie -func (s *TrieState) SetChildStorage(keyToChild, key, value []byte) error { +func (s *TrieState) SetChildStorage(keyToChild, key, value []byte, version trie.Version) error { s.lock.Lock() defer s.lock.Unlock() - return s.t.PutIntoChild(keyToChild, key, value) + return s.t.PutIntoChild(keyToChild, key, value, version) } // GetChild returns the child trie at the given key diff --git a/lib/runtime/storage/trie_test.go b/lib/runtime/storage/trie_test.go index 73e528db89..97af37f168 100644 --- a/lib/runtime/storage/trie_test.go +++ b/lib/runtime/storage/trie_test.go @@ -110,7 +110,7 @@ func TestTrieState_ClearPrefixInChild(t *testing.T) { keyToChild := []byte("keytochild") - err := ts.SetChild(keyToChild, child) + err := ts.SetChild(keyToChild, child, trie.V0) require.NoError(t, err) err = ts.ClearPrefixInChild(keyToChild, []byte("noo")) @@ -196,7 +196,7 @@ func TestTrieState_DeleteChildLimit(t *testing.T) { keyToChild := []byte("keytochild") - err := ts.SetChild(keyToChild, child) + err := ts.SetChild(keyToChild, child, trie.V0) require.NoError(t, err) testLimitBytes := make([]byte, 4) diff --git a/lib/runtime/wazero/imports.go b/lib/runtime/wazero/imports.go index 0de05b4cdf..0ef7bfe652 100644 --- a/lib/runtime/wazero/imports.go +++ b/lib/runtime/wazero/imports.go @@ -1071,7 +1071,7 @@ func ext_default_child_storage_set_version_1( cp := make([]byte, len(value)) copy(cp, value) - err := storage.SetChildStorage(childStorageKey, key, cp) + err := storage.SetChildStorage(childStorageKey, key, cp, trie.V0) if err != nil { logger.Errorf("failed to set value in child storage: %s", err) panic(err) diff --git a/lib/runtime/wazero/imports_test.go b/lib/runtime/wazero/imports_test.go index 6f7f1207a2..ab87e07d29 100644 --- a/lib/runtime/wazero/imports_test.go +++ b/lib/runtime/wazero/imports_test.go @@ -744,10 +744,10 @@ func Test_ext_default_child_storage_read_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie(), trie.V0) require.NoError(t, err) - err = inst.Context.Storage.SetChildStorage(testChildKey, testKey, testValue) + err = inst.Context.Storage.SetChildStorage(testChildKey, testKey, testValue, trie.V0) require.NoError(t, err) testOffset := uint32(2) @@ -783,7 +783,7 @@ func Test_ext_default_child_storage_read_version_1(t *testing.T) { func Test_ext_default_child_storage_set_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie(), trie.V0) require.NoError(t, err) // Check if value is not set @@ -811,10 +811,10 @@ func Test_ext_default_child_storage_set_version_1(t *testing.T) { func Test_ext_default_child_storage_clear_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie(), trie.V0) require.NoError(t, err) - err = inst.Context.Storage.SetChildStorage(testChildKey, testKey, testValue) + err = inst.Context.Storage.SetChildStorage(testChildKey, testKey, testValue, trie.V0) require.NoError(t, err) // Confirm if value is set @@ -850,11 +850,11 @@ func Test_ext_default_child_storage_clear_prefix_version_1(t *testing.T) { {[]byte("keyThree"), []byte("value3")}, } - err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie(), trie.V0) require.NoError(t, err) for _, kv := range testKeyValuePair { - err = inst.Context.Storage.SetChildStorage(testChildKey, kv.key, kv.value) + err = inst.Context.Storage.SetChildStorage(testChildKey, kv.key, kv.value, trie.V0) require.NoError(t, err) } @@ -880,10 +880,10 @@ func Test_ext_default_child_storage_clear_prefix_version_1(t *testing.T) { func Test_ext_default_child_storage_exists_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie(), trie.V0) require.NoError(t, err) - err = inst.Context.Storage.SetChildStorage(testChildKey, testKey, testValue) + err = inst.Context.Storage.SetChildStorage(testChildKey, testKey, testValue, trie.V0) require.NoError(t, err) encChildKey, err := scale.Marshal(testChildKey) @@ -904,10 +904,10 @@ func Test_ext_default_child_storage_exists_version_1(t *testing.T) { func Test_ext_default_child_storage_get_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie(), trie.V0) require.NoError(t, err) - err = inst.Context.Storage.SetChildStorage(testChildKey, testKey, testValue) + err = inst.Context.Storage.SetChildStorage(testChildKey, testKey, testValue, trie.V0) require.NoError(t, err) encChildKey, err := scale.Marshal(testChildKey) @@ -938,11 +938,11 @@ func Test_ext_default_child_storage_next_key_version_1(t *testing.T) { key := testKeyValuePair[0].key - err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie(), trie.V0) require.NoError(t, err) for _, kv := range testKeyValuePair { - err = inst.Context.Storage.SetChildStorage(testChildKey, kv.key, kv.value) + err = inst.Context.Storage.SetChildStorage(testChildKey, kv.key, kv.value, trie.V0) require.NoError(t, err) } @@ -965,10 +965,10 @@ func Test_ext_default_child_storage_next_key_version_1(t *testing.T) { func Test_ext_default_child_storage_root_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie(), trie.V0) require.NoError(t, err) - err = inst.Context.Storage.SetChildStorage(testChildKey, testKey, testValue) + err = inst.Context.Storage.SetChildStorage(testChildKey, testKey, testValue, trie.V0) require.NoError(t, err) child, err := inst.Context.Storage.GetChild(testChildKey) @@ -997,7 +997,7 @@ func Test_ext_default_child_storage_root_version_1(t *testing.T) { func Test_ext_default_child_storage_storage_kill_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie(), trie.V0) require.NoError(t, err) // Confirm if value is set @@ -1021,7 +1021,7 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_all(t *testing. tr := trie.NewEmptyTrie() tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) - err := inst.Context.Storage.SetChild(testChildKey, tr) + err := inst.Context.Storage.SetChild(testChildKey, tr, trie.V0) require.NoError(t, err) // Confirm if value is set @@ -1054,7 +1054,7 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_1(t *testing.T) tr := trie.NewEmptyTrie() tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) - err := inst.Context.Storage.SetChild(testChildKey, tr) + err := inst.Context.Storage.SetChild(testChildKey, tr, trie.V0) require.NoError(t, err) // Confirm if value is set @@ -1087,7 +1087,7 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_none(t *testing tr := trie.NewEmptyTrie() tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) - err := inst.Context.Storage.SetChild(testChildKey, tr) + err := inst.Context.Storage.SetChild(testChildKey, tr, trie.V0) require.NoError(t, err) // Confirm if value is set @@ -1118,7 +1118,7 @@ func Test_ext_default_child_storage_storage_kill_version_3(t *testing.T) { tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) tr.Put([]byte(`key3`), []byte(`value3`), trie.V0) - err := inst.Context.Storage.SetChild(testChildKey, tr) + err := inst.Context.Storage.SetChild(testChildKey, tr, trie.V0) require.NoError(t, err) testLimitBytes := make([]byte, 4) @@ -1316,11 +1316,11 @@ func Test_ext_default_child_storage_clear_prefix_version_2(t *testing.T) { {[]byte("keyThree"), []byte("value3")}, } - err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie(), trie.V0) require.NoError(t, err) for _, kv := range testKeyValuePair { - err = inst.Context.Storage.SetChildStorage(testChildKey, kv.key, kv.value) + err = inst.Context.Storage.SetChildStorage(testChildKey, kv.key, kv.value, trie.V0) require.NoError(t, err) } diff --git a/lib/trie/child_storage.go b/lib/trie/child_storage.go index 444c713da0..0b19b37e0a 100644 --- a/lib/trie/child_storage.go +++ b/lib/trie/child_storage.go @@ -18,7 +18,7 @@ var ErrChildTrieDoesNotExist = errors.New("child trie does not exist") // SetChild inserts a child trie into the main trie at key :child_storage:[keyToChild] // A child trie is added as a node (K, V) in the main trie. K is the child storage key // associated to the child trie, and V is the root hash of the child trie. -func (t *Trie) SetChild(keyToChild []byte, child *Trie) error { +func (t *Trie) SetChild(keyToChild []byte, child *Trie, version Version) error { childHash, err := child.Hash() if err != nil { return err @@ -28,7 +28,7 @@ func (t *Trie) SetChild(keyToChild []byte, child *Trie) error { copy(key, ChildStorageKeyPrefix) copy(key[len(ChildStorageKeyPrefix):], keyToChild) - err = t.Put(key, childHash.ToBytes(), V0) + err = t.Put(key, childHash.ToBytes(), version) if err != nil { return fmt.Errorf("putting child trie root hash %s in trie: %w", childHash, err) } @@ -52,7 +52,7 @@ func (t *Trie) GetChild(keyToChild []byte) (*Trie, error) { } // PutIntoChild puts a key-value pair into the child trie located in the main trie at key :child_storage:[keyToChild] -func (t *Trie) PutIntoChild(keyToChild, key, value []byte) error { +func (t *Trie) PutIntoChild(keyToChild, key, value []byte, version Version) error { child, err := t.GetChild(keyToChild) if err != nil { return err @@ -63,7 +63,7 @@ func (t *Trie) PutIntoChild(keyToChild, key, value []byte) error { return err } - err = child.Put(key, value, V0) + err = child.Put(key, value, version) if err != nil { return fmt.Errorf("putting into child trie located at key 0x%x: %w", keyToChild, err) } @@ -76,7 +76,7 @@ func (t *Trie) PutIntoChild(keyToChild, key, value []byte) error { delete(t.childTries, origChildHash) t.childTries[childHash] = child - return t.SetChild(keyToChild, child) + return t.SetChild(keyToChild, child, V0) } // GetFromChild retrieves a key-value pair from the child trie located diff --git a/lib/trie/child_storage_test.go b/lib/trie/child_storage_test.go index f8a53a82d3..325501bea5 100644 --- a/lib/trie/child_storage_test.go +++ b/lib/trie/child_storage_test.go @@ -14,7 +14,7 @@ func TestPutAndGetChild(t *testing.T) { childTrie := buildSmallTrie() parentTrie := NewEmptyTrie() - err := parentTrie.SetChild(childKey, childTrie) + err := parentTrie.SetChild(childKey, childTrie, V0) if err != nil { t.Fatal(err) } @@ -34,14 +34,14 @@ func TestPutAndGetFromChild(t *testing.T) { childTrie := buildSmallTrie() parentTrie := NewEmptyTrie() - err := parentTrie.SetChild(childKey, childTrie) + err := parentTrie.SetChild(childKey, childTrie, V0) if err != nil { t.Fatal(err) } testKey := []byte("child_key") testValue := []byte("child_value") - err = parentTrie.PutIntoChild(childKey, testKey, testValue) + err = parentTrie.PutIntoChild(childKey, testKey, testValue, V0) if err != nil { t.Fatal(err) } @@ -57,7 +57,7 @@ func TestPutAndGetFromChild(t *testing.T) { testKey = []byte("child_key_again") testValue = []byte("child_value_again") - err = parentTrie.PutIntoChild(childKey, testKey, testValue) + err = parentTrie.PutIntoChild(childKey, testKey, testValue, V0) if err != nil { t.Fatal(err) } diff --git a/lib/trie/database_test.go b/lib/trie/database_test.go index 01d79387cf..b05310729e 100644 --- a/lib/trie/database_test.go +++ b/lib/trie/database_test.go @@ -299,7 +299,7 @@ func Test_Trie_PutChild_Store_Load(t *testing.T) { } for _, keyToChildTrie := range keysToChildTries { - err := trie.SetChild(keyToChildTrie, childTrie) + err := trie.SetChild(keyToChildTrie, childTrie, V0) require.NoError(t, err) err = trie.WriteDirty(db) From 99626a3d6a83cbc70a79dcb64b66c958aeda9bae Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 22 Aug 2023 11:41:23 -0300 Subject: [PATCH 046/128] Improve version test cover --- lib/trie/version_test.go | 61 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/lib/trie/version_test.go b/lib/trie/version_test.go index 0000a43099..93fe755b76 100644 --- a/lib/trie/version_test.go +++ b/lib/trie/version_test.go @@ -62,6 +62,14 @@ func Test_ParseVersion(t *testing.T) { s: "V0", version: V0, }, + "v1": { + s: "v1", + version: V1, + }, + "V1": { + s: "V1", + version: V1, + }, "invalid": { s: "xyz", errWrapped: ErrParseVersion, @@ -84,3 +92,56 @@ func Test_ParseVersion(t *testing.T) { }) } } + +func Test_ShouldHashValue(t *testing.T) { + t.Parallel() + + testCases := map[string]struct { + version Version + value []byte + shouldHash bool + panicMessage string + }{ + "v0_small_value": { + version: V0, + value: []byte("smallvalue"), + shouldHash: false, + }, + "v0_large_value": { + version: V0, + value: []byte("newvaluewithmorethan32byteslength"), + shouldHash: false, + }, + "v1_small_value": { + version: V1, + value: []byte("smallvalue"), + shouldHash: false, + }, + "v1_large_value": { + version: V1, + value: []byte("newvaluewithmorethan32byteslength"), + shouldHash: true, + }, + "invalid": { + version: Version(99), + panicMessage: "unknown version 99", + }, + } + + for name, testCase := range testCases { + testCase := testCase + t.Run(name, func(t *testing.T) { + t.Parallel() + + if testCase.panicMessage != "" { + assert.PanicsWithValue(t, testCase.panicMessage, func() { + _ = testCase.version.ShouldHashValue(testCase.value) + }) + return + } + + shouldHash := testCase.version.ShouldHashValue(testCase.value) + assert.Equal(t, testCase.shouldHash, shouldHash) + }) + } +} From 376a97e72d06d06992565d21ec7056ffbea1f013 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 22 Aug 2023 12:20:28 -0300 Subject: [PATCH 047/128] Remove unused function --- lib/trie/trie.go | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/lib/trie/trie.go b/lib/trie/trie.go index 9316b3a059..360f2bb3e9 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -381,10 +381,7 @@ func (t *Trie) insertKeyLE(keyLE, value []byte, isValueHashed := version.ShouldHashValue(value) if isValueHashed { - hashedValue, err := common.Blake2bHash(value) - if err != nil { - return err - } + hashedValue := common.MustBlake2bHash(value) // Add original value in db using the hashed value as key err = t.db.Put(hashedValue.ToBytes(), value) @@ -674,29 +671,6 @@ func LoadFromMap(data map[string]string, version Version) (trie Trie, err error) return trie, nil } -// LoadFromEntries loads the given slice of key values into a new empty trie. -// The keys are in hexadecimal little Endian encoding and the values -// are hexadecimal encoded. -func LoadFromEntries(entries [][2][]byte, version Version) (trie *Trie, err error) { - trie = NewEmptyTrie() - - pendingDeltas := tracking.New() - defer func() { - trie.handleTrackedDeltas(err == nil, pendingDeltas) - }() - - for _, keyValue := range entries { - keyLE := keyValue[0] - value := keyValue[1] - err := trie.insertKeyLE(keyLE, value, pendingDeltas, version) - if err != nil { - return nil, err - } - } - - return trie, nil -} - // GetKeysWithPrefix returns all keys in little Endian // format from nodes in the trie that have the given little // Endian formatted prefix in their key. From 130e9bb59145c5192779d109b008fc3f8cf08785 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 22 Aug 2023 13:18:00 -0300 Subject: [PATCH 048/128] Refactor tests --- dot/import_integration_test.go | 42 --------------------------------- dot/import_test.go | 43 ++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 42 deletions(-) diff --git a/dot/import_integration_test.go b/dot/import_integration_test.go index 8dcc0efc0c..e0e9714244 100644 --- a/dot/import_integration_test.go +++ b/dot/import_integration_test.go @@ -19,48 +19,6 @@ import ( "github.com/stretchr/testify/require" ) -func Test_newTrieFromPairs(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - filename string - version trie.Version - want common.Hash - err error - }{ - { - name: "no_arguments", - err: errors.New("read .: is a directory"), - want: common.Hash{}, - }, - { - name: "working example", - filename: setupStateFile(t), - version: trie.V0, - want: common.MustHexToHash("0x09f9ca28df0560c2291aa16b56e15e07d1e1927088f51356d522722aa90ca7cb"), - }, - } - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - got, err := newTrieFromPairs(tt.filename, tt.version) - if tt.err != nil { - assert.EqualError(t, err, tt.err.Error()) - } else { - assert.NoError(t, err) - } - if tt.want.IsEmpty() { - assert.Nil(t, got) - } else { - assert.Equal(t, tt.want, got.MustHash()) - } - }) - } -} - func TestNewHeaderFromFile(t *testing.T) { fp := setupHeaderFile(t) header, err := newHeaderFromFile(fp) diff --git a/dot/import_test.go b/dot/import_test.go index 38b54f8db7..6229c97f9e 100644 --- a/dot/import_test.go +++ b/dot/import_test.go @@ -12,6 +12,7 @@ import ( "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -139,3 +140,45 @@ func Test_newHeaderFromFile(t *testing.T) { }) } } + +func Test_newTrieFromPairs(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + filename string + version trie.Version + want common.Hash + err error + }{ + { + name: "no_arguments", + err: errors.New("read .: is a directory"), + want: common.Hash{}, + }, + { + name: "working example", + filename: setupStateFile(t), + version: trie.V0, + want: common.MustHexToHash("0x09f9ca28df0560c2291aa16b56e15e07d1e1927088f51356d522722aa90ca7cb"), + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := newTrieFromPairs(tt.filename, tt.version) + if tt.err != nil { + assert.EqualError(t, err, tt.err.Error()) + } else { + assert.NoError(t, err) + } + if tt.want.IsEmpty() { + assert.Nil(t, got) + } else { + assert.Equal(t, tt.want, got.MustHash()) + } + }) + } +} From 15770b4831be34e7b4636de45d154259c8e530ba Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 22 Aug 2023 13:43:38 -0300 Subject: [PATCH 049/128] Add child storage tests --- lib/runtime/storage/trie_test.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lib/runtime/storage/trie_test.go b/lib/runtime/storage/trie_test.go index 97af37f168..3dca5122c6 100644 --- a/lib/runtime/storage/trie_test.go +++ b/lib/runtime/storage/trie_test.go @@ -40,6 +40,28 @@ func TestTrieState_SetGet(t *testing.T) { testFunc(ts) } +func TestTrieState_SetGetChildStorage(t *testing.T) { + testFunc := func(ts *TrieState) { + for _, tc := range testCases { + childTrie := trie.NewEmptyTrie() + err := ts.SetChild([]byte(tc), childTrie, trie.V0) + require.NoError(t, err) + + err = ts.SetChildStorage([]byte(tc), []byte(tc), []byte(tc), trie.V0) + require.NoError(t, err) + } + + for _, tc := range testCases { + res, err := ts.GetChildStorage([]byte(tc), []byte(tc)) + require.NoError(t, err) + require.Equal(t, []byte(tc), res) + } + } + + ts := &TrieState{t: trie.NewEmptyTrie()} + testFunc(ts) +} + func TestTrieState_Delete(t *testing.T) { testFunc := func(ts *TrieState) { for _, tc := range testCases { From f00678c9106968f3110dad7ff0a9776ada7b46b6 Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 23 Aug 2023 00:14:48 -0300 Subject: [PATCH 050/128] Add more tests to increase coverage --- cmd/gossamer/commands/import_state.go | 19 +++--- cmd/gossamer/commands/import_state_test.go | 68 ++++++++++++++++++++++ lib/runtime/storage/trie_test.go | 25 ++++++++ 3 files changed, 102 insertions(+), 10 deletions(-) create mode 100644 cmd/gossamer/commands/import_state_test.go diff --git a/cmd/gossamer/commands/import_state.go b/cmd/gossamer/commands/import_state.go index 0141271af7..5394dcc71f 100644 --- a/cmd/gossamer/commands/import_state.go +++ b/cmd/gossamer/commands/import_state.go @@ -13,7 +13,6 @@ import ( ) func init() { - ImportStateCmd.Flags().String("chain", "", "Chain id used to load default configuration for specified chain") ImportStateCmd.Flags().String("state-file", "", "Path to JSON file consisting of key-value pairs") ImportStateCmd.Flags().String("state-version", trie.DefaultStateVersion.String(), "State version v0 or v1") ImportStateCmd.Flags().String("header-file", "", "Path to JSON file of block header corresponding to the given state") @@ -28,7 +27,7 @@ var ImportStateCmd = &cobra.Command{ in the form of key-value pairs to be imported. Input can be generated by using the RPC function state_getPairs. Example: - gossamer import-state --state-file state.json --header-file header.json --first-slot `, + gossamer import-state --state-file state.json --state-version v1 --header-file header.json --first-slot `, RunE: func(cmd *cobra.Command, args []string) error { return execImportState(cmd) }, @@ -56,6 +55,14 @@ func execImportState(cmd *cobra.Command) error { return fmt.Errorf("state-file must be specified") } + headerFile, err := cmd.Flags().GetString("header-file") + if err != nil { + return fmt.Errorf("failed to get header-file: %s", err) + } + if headerFile == "" { + return fmt.Errorf("header-file must be specified") + } + stateVersionFlag, err := cmd.Flags().GetString("state-version") if err != nil { return fmt.Errorf("failed to get state-version: %s", err) @@ -65,14 +72,6 @@ func execImportState(cmd *cobra.Command) error { return fmt.Errorf("failed to parse state-version: %s", err) } - headerFile, err := cmd.Flags().GetString("header-file") - if err != nil { - return fmt.Errorf("failed to get header-file: %s", err) - } - if headerFile == "" { - return fmt.Errorf("header-file must be specified") - } - basePath = utils.ExpandDir(basePath) return dot.ImportState(basePath, stateFile, headerFile, firstSlot, stateVersion) diff --git a/cmd/gossamer/commands/import_state_test.go b/cmd/gossamer/commands/import_state_test.go new file mode 100644 index 0000000000..32a24e215c --- /dev/null +++ b/cmd/gossamer/commands/import_state_test.go @@ -0,0 +1,68 @@ +package commands + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestImportStateMissingStateFile(t *testing.T) { + rootCmd, err := NewRootCommand() + require.NoError(t, err) + rootCmd.AddCommand(ImportStateCmd) + + rootCmd.SetArgs([]string{ImportStateCmd.Name()}) + err = rootCmd.Execute() + assert.ErrorContains(t, err, "state-file must be specified") +} + +func TestImportStateInvalidFirstSlot(t *testing.T) { + rootCmd, err := NewRootCommand() + require.NoError(t, err) + rootCmd.AddCommand(ImportStateCmd) + + rootCmd.SetArgs([]string{ImportStateCmd.Name(), "--first-slot", "wrong"}) + err = rootCmd.Execute() + assert.ErrorContains(t, err, "invalid argument \"wrong\"") +} + +func TestImportStateEmptyStateFile(t *testing.T) { + rootCmd, err := NewRootCommand() + require.NoError(t, err) + rootCmd.AddCommand(ImportStateCmd) + + rootCmd.SetArgs([]string{ImportStateCmd.Name(), "--state-version", "v0", "--state-file", ""}) + err = rootCmd.Execute() + assert.ErrorContains(t, err, "state-file must be specified") +} + +func TestImportStateEmptyHeaderFile(t *testing.T) { + rootCmd, err := NewRootCommand() + require.NoError(t, err) + rootCmd.AddCommand(ImportStateCmd) + + rootCmd.SetArgs([]string{ImportStateCmd.Name(), "--state-version", "v0", "--state-file", "test", "--header-file", ""}) + err = rootCmd.Execute() + assert.ErrorContains(t, err, "header-file must be specified") +} + +func TestImportStateInvalidStateVersion(t *testing.T) { + rootCmd, err := NewRootCommand() + require.NoError(t, err) + rootCmd.AddCommand(ImportStateCmd) + + rootCmd.SetArgs([]string{ImportStateCmd.Name(), "--state-version", "v999", "--state-file", "test", "--header-file", "test"}) + err = rootCmd.Execute() + assert.ErrorContains(t, err, "failed to parse state-version: parsing version failed: \"v999\" must be one of [v0, v1]") +} + +func TestImportStateErrorImportingState(t *testing.T) { + rootCmd, err := NewRootCommand() + require.NoError(t, err) + rootCmd.AddCommand(ImportStateCmd) + + rootCmd.SetArgs([]string{ImportStateCmd.Name(), "--state-version", "v0", "--state-file", "test", "--header-file", "test"}) + err = rootCmd.Execute() + assert.ErrorContains(t, err, "no such file or directory") +} diff --git a/lib/runtime/storage/trie_test.go b/lib/runtime/storage/trie_test.go index 3dca5122c6..aafd0099a2 100644 --- a/lib/runtime/storage/trie_test.go +++ b/lib/runtime/storage/trie_test.go @@ -62,6 +62,31 @@ func TestTrieState_SetGetChildStorage(t *testing.T) { testFunc(ts) } +func TestTrieState_SetAndClearFromChild(t *testing.T) { + testFunc := func(ts *TrieState) { + for _, tc := range testCases { + childTrie := trie.NewEmptyTrie() + err := ts.SetChild([]byte(tc), childTrie, trie.V0) + require.NoError(t, err) + + err = ts.SetChildStorage([]byte(tc), []byte(tc), []byte(tc), trie.V0) + require.NoError(t, err) + } + + for _, tc := range testCases { + err := ts.ClearChildStorage([]byte(tc), []byte(tc)) + require.NoError(t, err) + + res, err := ts.GetChildStorage([]byte(tc), []byte(tc)) + require.NoError(t, err) + require.Equal(t, []uint8(nil), res) + } + } + + ts := &TrieState{t: trie.NewEmptyTrie()} + testFunc(ts) +} + func TestTrieState_Delete(t *testing.T) { testFunc := func(ts *TrieState) { for _, tc := range testCases { From 8834add7856b8bc29fef494c31ea6b91f1cdbe52 Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 23 Aug 2023 11:12:47 -0300 Subject: [PATCH 051/128] Fix linters --- cmd/gossamer/commands/import_state.go | 2 +- cmd/gossamer/commands/import_state_test.go | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cmd/gossamer/commands/import_state.go b/cmd/gossamer/commands/import_state.go index 5394dcc71f..6ee5ac4671 100644 --- a/cmd/gossamer/commands/import_state.go +++ b/cmd/gossamer/commands/import_state.go @@ -27,7 +27,7 @@ var ImportStateCmd = &cobra.Command{ in the form of key-value pairs to be imported. Input can be generated by using the RPC function state_getPairs. Example: - gossamer import-state --state-file state.json --state-version v1 --header-file header.json --first-slot `, + gossamer import-state --state-file state.json --state-version v1 --header-file header.json --first-slot `, //nolint:lll RunE: func(cmd *cobra.Command, args []string) error { return execImportState(cmd) }, diff --git a/cmd/gossamer/commands/import_state_test.go b/cmd/gossamer/commands/import_state_test.go index 32a24e215c..265b43c863 100644 --- a/cmd/gossamer/commands/import_state_test.go +++ b/cmd/gossamer/commands/import_state_test.go @@ -1,3 +1,6 @@ +// Copyright 2023 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + package commands import ( @@ -42,7 +45,7 @@ func TestImportStateEmptyHeaderFile(t *testing.T) { require.NoError(t, err) rootCmd.AddCommand(ImportStateCmd) - rootCmd.SetArgs([]string{ImportStateCmd.Name(), "--state-version", "v0", "--state-file", "test", "--header-file", ""}) + rootCmd.SetArgs([]string{ImportStateCmd.Name(), "--state-version", "v0", "--state-file", "test", "--header-file", ""}) //nolint:lll err = rootCmd.Execute() assert.ErrorContains(t, err, "header-file must be specified") } @@ -52,7 +55,7 @@ func TestImportStateInvalidStateVersion(t *testing.T) { require.NoError(t, err) rootCmd.AddCommand(ImportStateCmd) - rootCmd.SetArgs([]string{ImportStateCmd.Name(), "--state-version", "v999", "--state-file", "test", "--header-file", "test"}) + rootCmd.SetArgs([]string{ImportStateCmd.Name(), "--state-version", "v999", "--state-file", "test", "--header-file", "test"}) //nolint:lll err = rootCmd.Execute() assert.ErrorContains(t, err, "failed to parse state-version: parsing version failed: \"v999\" must be one of [v0, v1]") } @@ -62,7 +65,7 @@ func TestImportStateErrorImportingState(t *testing.T) { require.NoError(t, err) rootCmd.AddCommand(ImportStateCmd) - rootCmd.SetArgs([]string{ImportStateCmd.Name(), "--state-version", "v0", "--state-file", "test", "--header-file", "test"}) + rootCmd.SetArgs([]string{ImportStateCmd.Name(), "--state-version", "v0", "--state-file", "test", "--header-file", "test"}) //nolint:lll err = rootCmd.Execute() assert.ErrorContains(t, err, "no such file or directory") } From d4e2ac6b3af83d698d0ee08fcefbb86efec30b6b Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 23 Aug 2023 11:38:33 -0300 Subject: [PATCH 052/128] PR comments --- dot/state/interfaces.go | 2 +- dot/state/storage.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dot/state/interfaces.go b/dot/state/interfaces.go index 3ebd3f86b5..8e53101645 100644 --- a/dot/state/interfaces.go +++ b/dot/state/interfaces.go @@ -31,7 +31,7 @@ type GetPutter interface { // Batcher has methods to get values and create a // new batch. -type Batcher interface { +type GetterPutterNewBatcher interface { Getter Putter NewBatcher diff --git a/dot/state/storage.go b/dot/state/storage.go index 0844bc0b8d..8b068ee877 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -33,7 +33,7 @@ type StorageState struct { blockState *BlockState tries *Tries - db Batcher + db GetterPutterNewBatcher sync.RWMutex // change notifiers From 024965a8ed89087b7226f2c747091fd7bf7b21bd Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 23 Aug 2023 12:15:17 -0300 Subject: [PATCH 053/128] Increase coverage in child storage tests --- lib/trie/child_storage_test.go | 78 ++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 32 deletions(-) diff --git a/lib/trie/child_storage_test.go b/lib/trie/child_storage_test.go index 325501bea5..dcb00d0082 100644 --- a/lib/trie/child_storage_test.go +++ b/lib/trie/child_storage_test.go @@ -4,9 +4,9 @@ package trie import ( - "bytes" - "reflect" "testing" + + "github.com/stretchr/testify/assert" ) func TestPutAndGetChild(t *testing.T) { @@ -15,18 +15,46 @@ func TestPutAndGetChild(t *testing.T) { parentTrie := NewEmptyTrie() err := parentTrie.SetChild(childKey, childTrie, V0) - if err != nil { - t.Fatal(err) - } + assert.NoError(t, err) childTrieRes, err := parentTrie.GetChild(childKey) - if err != nil { - t.Fatal(err) - } + assert.NoError(t, err) + + assert.Equal(t, childTrie, childTrieRes) +} + +func TestPutAndDeleteChild(t *testing.T) { + childKey := []byte("default") + childTrie := buildSmallTrie() + parentTrie := NewEmptyTrie() + + err := parentTrie.SetChild(childKey, childTrie, V0) + assert.NoError(t, err) + + err = parentTrie.DeleteChild(childKey) + assert.NoError(t, err) + + _, err = parentTrie.GetChild(childKey) + assert.ErrorContains(t, err, "child trie does not exist at key") +} + +func TestPutAndClearFromChild(t *testing.T) { + childKey := []byte("default") + keyInChild := []byte{0x01, 0x35} + childTrie := buildSmallTrie() + parentTrie := NewEmptyTrie() + + err := parentTrie.SetChild(childKey, childTrie, V0) + assert.NoError(t, err) + + err = parentTrie.ClearFromChild(childKey, keyInChild) + assert.NoError(t, err) + + childTrie, err = parentTrie.GetChild(childKey) + assert.NoError(t, err) - if !reflect.DeepEqual(childTrie, childTrieRes) { - t.Fatalf("Fail: got %v expected %v", childTrieRes, childTrie) - } + value := childTrie.Get(keyInChild) + assert.Equal(t, []uint8(nil), value) } func TestPutAndGetFromChild(t *testing.T) { @@ -35,39 +63,25 @@ func TestPutAndGetFromChild(t *testing.T) { parentTrie := NewEmptyTrie() err := parentTrie.SetChild(childKey, childTrie, V0) - if err != nil { - t.Fatal(err) - } + assert.NoError(t, err) testKey := []byte("child_key") testValue := []byte("child_value") err = parentTrie.PutIntoChild(childKey, testKey, testValue, V0) - if err != nil { - t.Fatal(err) - } + assert.NoError(t, err) valueRes, err := parentTrie.GetFromChild(childKey, testKey) - if err != nil { - t.Fatal(err) - } + assert.NoError(t, err) - if !bytes.Equal(valueRes, testValue) { - t.Fatalf("Fail: got %x expected %x", valueRes, testValue) - } + assert.Equal(t, valueRes, testValue) testKey = []byte("child_key_again") testValue = []byte("child_value_again") err = parentTrie.PutIntoChild(childKey, testKey, testValue, V0) - if err != nil { - t.Fatal(err) - } + assert.NoError(t, err) valueRes, err = parentTrie.GetFromChild(childKey, testKey) - if err != nil { - t.Fatal(err) - } + assert.NoError(t, err) - if !bytes.Equal(valueRes, testValue) { - t.Fatalf("Fail: got %x expected %x", valueRes, testValue) - } + assert.Equal(t, valueRes, testValue) } From 4f8780179f7f8e6cc4b1eae224c4c581a93bdb0c Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 23 Aug 2023 13:06:34 -0300 Subject: [PATCH 054/128] Remove unused functions --- lib/trie/trie.go | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/lib/trie/trie.go b/lib/trie/trie.go index 360f2bb3e9..8ebe4c1da6 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -200,40 +200,6 @@ func (t *Trie) Hash() (rootHash common.Hash, err error) { return rootHash, nil } -// EntriesList returns all the key-value pairs in the trie as a slice of key value -// where the keys are encoded in Little Endian. The slice starts with root node. -func (t *Trie) EntriesList() [][2][]byte { - list := make([][2][]byte, 0) - entriesList(t.root, nil, &list) - return list -} - -func entriesList(parent *Node, prefix []byte, list *[][2][]byte) { - if parent == nil { - return - } - - if parent.Kind() == node.Leaf { - parentKey := parent.PartialKey - fullKeyNibbles := concatenateSlices(prefix, parentKey) - keyLE := codec.NibblesToKeyLE(fullKeyNibbles) - *list = append(*list, [2][]byte{keyLE, parent.StorageValue}) - return - } - - branch := parent - if branch.StorageValue != nil { - fullKeyNibbles := concatenateSlices(prefix, branch.PartialKey) - keyLE := codec.NibblesToKeyLE(fullKeyNibbles) - *list = append(*list, [2][]byte{keyLE, parent.StorageValue}) - } - - for i, child := range branch.Children { - childPrefix := concatenateSlices(prefix, branch.PartialKey, intToByteSlice(i)) - entriesList(child, childPrefix, list) - } -} - // Entries returns all the key-value pairs in the trie as a map of keys to values // where the keys are encoded in Little Endian. func (t *Trie) Entries() (keyValueMap map[string][]byte) { From 84541d84eaa7fad93ac837db809964329f522051 Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 24 Aug 2023 11:08:39 -0300 Subject: [PATCH 055/128] Removes todos and skip validations for unused params --- lib/runtime/wazero/imports.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/lib/runtime/wazero/imports.go b/lib/runtime/wazero/imports.go index c5ddfe9037..fe99a03bfd 100644 --- a/lib/runtime/wazero/imports.go +++ b/lib/runtime/wazero/imports.go @@ -866,7 +866,6 @@ func ext_trie_blake2_256_root_version_2(ctx context.Context, m api.Module, dataS } stateVersion := uint8(version) - trie.EnforceValidVersion(stateVersion) data := read(m, dataSpan) @@ -885,7 +884,6 @@ func ext_trie_blake2_256_root_version_2(ctx context.Context, m api.Module, dataS } for _, kv := range kvs { - //TODO: use version parameter here err := t.Put(kv.Key, kv.Value, trie.Version(stateVersion)) if err != nil { logger.Errorf("failed putting key 0x%x and value 0x%x into trie: %s", @@ -901,7 +899,6 @@ func ext_trie_blake2_256_root_version_2(ctx context.Context, m api.Module, dataS return 0 } - //TODO: use version parameter here hash, err := t.Hash() if err != nil { logger.Errorf("failed computing trie Merkle root hash: %s", err) @@ -995,7 +992,6 @@ func ext_trie_blake2_256_ordered_root_version_2( "put key=0x%x and value=0x%x", key, value) - //TODO: use version parameter here err = t.Put(key, value, trie.Version(stateVersion)) if err != nil { logger.Errorf("failed putting key 0x%x and value 0x%x into trie: %s", @@ -1011,7 +1007,6 @@ func ext_trie_blake2_256_ordered_root_version_2( return 0 } - //TODO: use version parameter here hash, err := t.Hash() if err != nil { logger.Errorf("failed computing trie Merkle root hash: %s", err) @@ -1056,7 +1051,7 @@ func ext_trie_blake2_256_verify_proof_version_1( } func ext_trie_blake2_256_verify_proof_version_2( - ctx context.Context, m api.Module, rootSpan uint32, proofSpan, keySpan, valueSpan uint64, version uint32) uint32 { + ctx context.Context, m api.Module, rootSpan uint32, proofSpan, keySpan, valueSpan uint64, version uint32) uint32 { //skipcq: RVV-B0012 rtCtx := ctx.Value(runtimeContextKey).(*runtime.Context) if rtCtx == nil { panic("nil runtime context") @@ -1078,7 +1073,6 @@ func ext_trie_blake2_256_verify_proof_version_2( panic("read overflow") } - //TODO: use trie version parameter here err = proof.Verify(encodedProofNodes, trieRoot, key, value) if err != nil { logger.Errorf("failed proof verification: %s", err) @@ -1411,7 +1405,7 @@ func ext_default_child_storage_root_version_1( //export ext_default_child_storage_root_version_2 func ext_default_child_storage_root_version_2(ctx context.Context, m api.Module, childStorageKey uint64, - stateVersion uint32) (ptrSize uint64) { + stateVersion uint32) (ptrSize uint64) { //skipcq: RVV-B0012 rtCtx := ctx.Value(runtimeContextKey).(*runtime.Context) if rtCtx == nil { panic("nil runtime context") @@ -1423,7 +1417,6 @@ func ext_default_child_storage_root_version_2(ctx context.Context, m api.Module, return 0 } - //TODO: version this call childRoot, err := child.Hash() if err != nil { logger.Errorf("failed to encode child root: %s", err) @@ -2434,7 +2427,6 @@ func ext_storage_root_version_2(ctx context.Context, m api.Module, version uint3 } storage := rtCtx.Storage - //TODO: user version parameter root, err := storage.Root() if err != nil { logger.Errorf("failed to get storage root: %s", err) From 793d80cc8eddb4730fc4da34a1193dfa2bc81a72 Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 28 Aug 2023 16:56:40 +0200 Subject: [PATCH 056/128] Add new runtime functions tests --- lib/runtime/constants.go | 7 +- lib/runtime/wazero/imports_test.go | 228 ++++++++++++++++++++++++++++- 2 files changed, 231 insertions(+), 4 deletions(-) diff --git a/lib/runtime/constants.go b/lib/runtime/constants.go index 26d0ccd267..c6c5346ccf 100644 --- a/lib/runtime/constants.go +++ b/lib/runtime/constants.go @@ -6,9 +6,10 @@ package runtime const ( // v0.9 test API wasm // This wasm is generated using https://github.com/ChainSafe/polkadot-spec. - HOST_API_TEST_RUNTIME = "hostapi_runtime" - HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm" - HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/raw/master/test/" + + HOST_API_TEST_RUNTIME = "hostapi_runtime" + HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm" + // TODO: change this when https://github.com/ChainSafe/polkadot-spec/pull/17 is merged + HOST_API_TEST_RUNTIME_URL = "https://github.com/dimartiro/polkadot-spec/raw/add-triev1-functions/test/" + "runtimes/hostapi/hostapi_runtime.compact.wasm" // v0.9.29 polkadot diff --git a/lib/runtime/wazero/imports_test.go b/lib/runtime/wazero/imports_test.go index ab87e07d29..499d7083a1 100644 --- a/lib/runtime/wazero/imports_test.go +++ b/lib/runtime/wazero/imports_test.go @@ -554,7 +554,9 @@ func Test_ext_trie_blake2_256_root_version_1(t *testing.T) { require.NoError(t, err) encInput[0] = encInput[0] >> 1 - res, err := inst.Exec("rtm_ext_trie_blake2_256_root_version_1", encInput) + data := encInput + + res, err := inst.Exec("rtm_ext_trie_blake2_256_root_version_1", data) require.NoError(t, err) var hash []byte @@ -569,6 +571,39 @@ func Test_ext_trie_blake2_256_root_version_1(t *testing.T) { require.Equal(t, expected[:], hash) } +// TODO: revisit it +func Test_ext_trie_blake2_256_root_version_2(t *testing.T) { + inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) + + testinput := []string{"dimartiro", "was", "here", "??"} + encInput, err := scale.Marshal(testinput) + require.NoError(t, err) + encInput[0] = encInput[0] >> 1 + + stateVersion := uint32(trie.V1) + stateVersionBytes := make([]byte, 4) + binary.LittleEndian.PutUint32(stateVersionBytes, stateVersion) + + encVersion, err := scale.Marshal(&stateVersionBytes) + require.NoError(t, err) + + data := append(encInput, encVersion...) + + res, err := inst.Exec("rtm_ext_trie_blake2_256_root_version_2", data) + require.NoError(t, err) + + var hash []byte + err = scale.Unmarshal(res, &hash) + require.NoError(t, err) + + tt := trie.NewEmptyTrie() + tt.Put([]byte("dimartiro"), []byte("was"), trie.V1) + tt.Put([]byte("here"), []byte("??"), trie.V1) + + expected := tt.MustHash() + require.Equal(t, expected[:], hash) +} + func Test_ext_trie_blake2_256_ordered_root_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) @@ -587,6 +622,33 @@ func Test_ext_trie_blake2_256_ordered_root_version_1(t *testing.T) { require.Equal(t, expected[:], hash) } +func Test_ext_trie_blake2_256_ordered_root_version_2(t *testing.T) { + inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) + + testvalues := []string{"static", "even-keeled", "Future-proofed"} + encValues, err := scale.Marshal(testvalues) + require.NoError(t, err) + + stateVersion := uint32(trie.V1) + stateVersionBytes := make([]byte, 4) + binary.LittleEndian.PutUint32(stateVersionBytes, stateVersion) + + encVersion, err := scale.Marshal(&stateVersionBytes) + require.NoError(t, err) + + data := append(encValues, encVersion...) + + res, err := inst.Exec("rtm_ext_trie_blake2_256_ordered_root_version_2", data) + require.NoError(t, err) + + var hash []byte + err = scale.Unmarshal(res, &hash) + require.NoError(t, err) + + expected := common.MustHexToHash("0xd847b86d0219a384d11458e829e9f4f4cce7e3cc2e6dcd0e8a6ad6f12c64a737") + require.Equal(t, expected[:], hash) +} + func Test_ext_trie_blake2_256_verify_proof_version_1(t *testing.T) { tmp := t.TempDir() memdb, err := database.NewPebble(tmp, true) @@ -679,6 +741,107 @@ func Test_ext_trie_blake2_256_verify_proof_version_1(t *testing.T) { } } +func Test_ext_trie_blake2_256_verify_proof_version_2(t *testing.T) { + tmp := t.TempDir() + memdb, err := database.NewPebble(tmp, true) + require.NoError(t, err) + + otherTrie := trie.NewEmptyTrie() + otherTrie.Put([]byte("simple"), []byte("cat"), trie.V1) + + otherHash, err := otherTrie.Hash() + require.NoError(t, err) + + tr := trie.NewEmptyTrie() + tr.Put([]byte("do"), []byte("verb"), trie.V1) + tr.Put([]byte("domain"), []byte("website"), trie.V1) + tr.Put([]byte("other"), []byte("random"), trie.V1) + tr.Put([]byte("otherwise"), []byte("randomstuff"), trie.V1) + tr.Put([]byte("cat"), []byte("another animal"), trie.V1) + + err = tr.WriteDirty(memdb) + require.NoError(t, err) + + hash, err := tr.Hash() + require.NoError(t, err) + + keys := [][]byte{ + []byte("do"), + []byte("domain"), + []byte("other"), + []byte("otherwise"), + []byte("cat"), + } + + root := hash.ToBytes() + otherRoot := otherHash.ToBytes() + + allProofs, err := proof.Generate(root, keys, memdb) + require.NoError(t, err) + + testcases := map[string]struct { + root, key, value []byte + proof [][]byte + expect bool + }{ + "Proof_should_be_true": { + root: root, key: []byte("do"), proof: allProofs, value: []byte("verb"), expect: true}, + "Root_empty,_proof_should_be_false": { + root: []byte{}, key: []byte("do"), proof: allProofs, value: []byte("verb"), expect: false}, + "Other_root,_proof_should_be_false": { + root: otherRoot, key: []byte("do"), proof: allProofs, value: []byte("verb"), expect: false}, + "Value_empty,_proof_should_be_true": { + root: root, key: []byte("do"), proof: allProofs, value: nil, expect: true}, + "Unknow_key,_proof_should_be_false": { + root: root, key: []byte("unknow"), proof: allProofs, value: nil, expect: false}, + "Key_and_value_unknow,_proof_should_be_false": { + root: root, key: []byte("unknow"), proof: allProofs, value: []byte("unknow"), expect: false}, + "Empty_proof,_should_be_false": { + root: root, key: []byte("do"), proof: [][]byte{}, value: nil, expect: false}, + } + + inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) + + for name, testcase := range testcases { + testcase := testcase + t.Run(name, func(t *testing.T) { + hashEnc, err := scale.Marshal(testcase.root) + require.NoError(t, err) + + args := hashEnc + + encProof, err := scale.Marshal(testcase.proof) + require.NoError(t, err) + args = append(args, encProof...) + + keyEnc, err := scale.Marshal(testcase.key) + require.NoError(t, err) + args = append(args, keyEnc...) + + valueEnc, err := scale.Marshal(testcase.value) + require.NoError(t, err) + args = append(args, valueEnc...) + + stateVersion := uint32(trie.V1) + stateVersionBytes := make([]byte, 4) + binary.LittleEndian.PutUint32(stateVersionBytes, stateVersion) + + encVersion, err := scale.Marshal(&stateVersionBytes) + require.NoError(t, err) + + args = append(args, encVersion...) + + res, err := inst.Exec("rtm_ext_trie_blake2_256_verify_proof_version_2", args) + require.NoError(t, err) + + var got bool + err = scale.Unmarshal(res, &got) + require.NoError(t, err) + require.Equal(t, testcase.expect, got) + }) + } +} + func Test_ext_misc_runtime_version_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) @@ -994,6 +1157,48 @@ func Test_ext_default_child_storage_root_version_1(t *testing.T) { require.Equal(t, rootHash, actualValue) } +func Test_ext_default_child_storage_root_version_2(t *testing.T) { + inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) + + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie(), trie.V1) + require.NoError(t, err) + + err = inst.Context.Storage.SetChildStorage(testChildKey, testKey, testValue, trie.V1) + require.NoError(t, err) + + child, err := inst.Context.Storage.GetChild(testChildKey) + require.NoError(t, err) + + rootHash, err := child.Hash() + require.NoError(t, err) + + encChildKey, err := scale.Marshal(testChildKey) + require.NoError(t, err) + encKey, err := scale.Marshal(testKey) + require.NoError(t, err) + + stateVersion := uint32(trie.V1) + stateVersionBytes := make([]byte, 4) + binary.LittleEndian.PutUint32(stateVersionBytes, stateVersion) + + encVersion, err := scale.Marshal(&stateVersionBytes) + require.NoError(t, err) + + data := append(encChildKey, encKey...) + data = append(data, encVersion...) + + ret, err := inst.Exec("rtm_ext_default_child_storage_root_version_2", data) + require.NoError(t, err) + + var hash []byte + err = scale.Unmarshal(ret, &hash) + require.NoError(t, err) + + // Convert decoded interface to common Hash + actualValue := common.BytesToHash(hash) + require.Equal(t, rootHash, actualValue) +} + func Test_ext_default_child_storage_storage_kill_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) @@ -1835,6 +2040,27 @@ func Test_ext_storage_root_version_1(t *testing.T) { require.Equal(t, expected[:], hash) } +func Test_ext_storage_root_version_2(t *testing.T) { + inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) + + stateVersion := uint32(trie.V1) + stateVersionBytes := make([]byte, 4) + binary.LittleEndian.PutUint32(stateVersionBytes, stateVersion) + + encVersion, err := scale.Marshal(&stateVersionBytes) + require.NoError(t, err) + + ret, err := inst.Exec("rtm_ext_storage_root_version_2", encVersion) + require.NoError(t, err) + + var hash []byte + err = scale.Unmarshal(ret, &hash) + require.NoError(t, err) + + expected := trie.EmptyHash + require.Equal(t, expected[:], hash) +} + func Test_ext_storage_set_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) From 413916032fd2e0da21d0a4b16a211d9ba1c7fa6f Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 30 Aug 2023 00:23:08 +0200 Subject: [PATCH 057/128] Improve largest lines to not ignore linter --- cmd/gossamer/commands/import_state_test.go | 18 +++++++++++++++--- dot/import.go | 1 - lib/runtime/wazero/imports_test.go | 2 +- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/cmd/gossamer/commands/import_state_test.go b/cmd/gossamer/commands/import_state_test.go index 265b43c863..e7a35ed3d4 100644 --- a/cmd/gossamer/commands/import_state_test.go +++ b/cmd/gossamer/commands/import_state_test.go @@ -45,7 +45,11 @@ func TestImportStateEmptyHeaderFile(t *testing.T) { require.NoError(t, err) rootCmd.AddCommand(ImportStateCmd) - rootCmd.SetArgs([]string{ImportStateCmd.Name(), "--state-version", "v0", "--state-file", "test", "--header-file", ""}) //nolint:lll + rootCmd.SetArgs([]string{ImportStateCmd.Name(), + "--state-version", "v0", + "--state-file", "test", + "--header-file", "", + }) err = rootCmd.Execute() assert.ErrorContains(t, err, "header-file must be specified") } @@ -55,7 +59,11 @@ func TestImportStateInvalidStateVersion(t *testing.T) { require.NoError(t, err) rootCmd.AddCommand(ImportStateCmd) - rootCmd.SetArgs([]string{ImportStateCmd.Name(), "--state-version", "v999", "--state-file", "test", "--header-file", "test"}) //nolint:lll + rootCmd.SetArgs([]string{ImportStateCmd.Name(), + "--state-version", "v999", + "--state-file", "test", + "--header-file", "test", + }) err = rootCmd.Execute() assert.ErrorContains(t, err, "failed to parse state-version: parsing version failed: \"v999\" must be one of [v0, v1]") } @@ -65,7 +73,11 @@ func TestImportStateErrorImportingState(t *testing.T) { require.NoError(t, err) rootCmd.AddCommand(ImportStateCmd) - rootCmd.SetArgs([]string{ImportStateCmd.Name(), "--state-version", "v0", "--state-file", "test", "--header-file", "test"}) //nolint:lll + rootCmd.SetArgs([]string{ImportStateCmd.Name(), + "--state-version", "v0", + "--state-file", "test", + "--header-file", "test", + }) err = rootCmd.Execute() assert.ErrorContains(t, err, "no such file or directory") } diff --git a/dot/import.go b/dot/import.go index 5e6687274e..177ba3edda 100644 --- a/dot/import.go +++ b/dot/import.go @@ -62,7 +62,6 @@ func newTrieFromPairs(filename string, stateVersion trie.Version) (*trie.Trie, e entries[pairArr[0].(string)] = pairArr[1].(string) } - //TODO: revisit this to use the right trie version tr, err := trie.LoadFromMap(entries, stateVersion) if err != nil { return nil, err diff --git a/lib/runtime/wazero/imports_test.go b/lib/runtime/wazero/imports_test.go index ab87e07d29..f7caa06a0c 100644 --- a/lib/runtime/wazero/imports_test.go +++ b/lib/runtime/wazero/imports_test.go @@ -1190,7 +1190,7 @@ func Test_ext_hashing_blake2_256_version_1(t *testing.T) { enc, err := scale.Marshal(data) require.NoError(t, err) - ret, err := inst.Exec("rtm_ext_hashing_blake2_256_version_1", enc) +ret, err := inst.Exec("rtm_ext_hashing_blake2_256_version_1", enc) require.NoError(t, err) var hash []byte From cfcb324065a53515e84e863b3f98a3cb87f75a78 Mon Sep 17 00:00:00 2001 From: Diego Date: Fri, 1 Sep 2023 16:38:13 +0200 Subject: [PATCH 058/128] Fix trie snapshot creation copying internal db --- dot/state/db_mocks_test.go | 17 ++++++++++- dot/state/mocks_generate_test.go | 2 +- dot/state/storage_test.go | 52 ++++++++++++++++++++++++++++++-- lib/runtime/genesis.go | 1 - lib/trie/database.go | 21 ++++--------- lib/trie/db/db.go | 33 ++++++++++++++++++++ lib/trie/proof/generate.go | 5 +-- lib/trie/proof/verify.go | 2 +- lib/trie/proof/verify_test.go | 2 +- lib/trie/trie.go | 30 ++++++++++++------ lib/trie/trie_test.go | 6 ++-- 11 files changed, 134 insertions(+), 37 deletions(-) diff --git a/dot/state/db_mocks_test.go b/dot/state/db_mocks_test.go index 6df73ac4d8..0c7fdd4069 100644 --- a/dot/state/db_mocks_test.go +++ b/dot/state/db_mocks_test.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/ChainSafe/gossamer/lib/trie (interfaces: Database) +// Source: github.com/ChainSafe/gossamer/lib/trie/db (interfaces: Database) // Package state is a generated GoMock package. package state @@ -7,6 +7,7 @@ package state import ( reflect "reflect" + db "github.com/ChainSafe/gossamer/lib/trie/db" gomock "github.com/golang/mock/gomock" ) @@ -33,6 +34,20 @@ func (m *MockDatabase) EXPECT() *MockDatabaseMockRecorder { return m.recorder } +// Copy mocks base method. +func (m *MockDatabase) Copy() db.Database { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Copy") + ret0, _ := ret[0].(db.Database) + return ret0 +} + +// Copy indicates an expected call of Copy. +func (mr *MockDatabaseMockRecorder) Copy() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Copy", reflect.TypeOf((*MockDatabase)(nil).Copy)) +} + // Get mocks base method. func (m *MockDatabase) Get(arg0 []byte) ([]byte, error) { m.ctrl.T.Helper() diff --git a/dot/state/mocks_generate_test.go b/dot/state/mocks_generate_test.go index fdac9b006f..c4de67b207 100644 --- a/dot/state/mocks_generate_test.go +++ b/dot/state/mocks_generate_test.go @@ -7,4 +7,4 @@ package state //go:generate mockgen -destination=mocks_runtime_test.go -package $GOPACKAGE github.com/ChainSafe/gossamer/lib/runtime Instance //go:generate mockgen -destination=mock_gauge_test.go -package $GOPACKAGE github.com/prometheus/client_golang/prometheus Gauge //go:generate mockgen -destination=mock_counter_test.go -package $GOPACKAGE github.com/prometheus/client_golang/prometheus Counter -//go:generate mockgen -destination=db_mocks_test.go -package=$GOPACKAGE github.com/ChainSafe/gossamer/lib/trie Database +//go:generate mockgen -destination=db_mocks_test.go -package=$GOPACKAGE github.com/ChainSafe/gossamer/lib/trie/db Database diff --git a/dot/state/storage_test.go b/dot/state/storage_test.go index 5cf60b81e3..97471147b4 100644 --- a/dot/state/storage_test.go +++ b/dot/state/storage_test.go @@ -104,7 +104,7 @@ func TestStorage_TrieState(t *testing.T) { require.Equal(t, ts.Trie().MustHash(), ts3.Trie().MustHash()) } -func TestStorage_LoadFromDB(t *testing.T) { +func TestStorage_LoadFromDB_v0(t *testing.T) { storage := newTestStorageState(t) ts, err := storage.TrieState(&trie.EmptyHash) require.NoError(t, err) @@ -117,6 +117,7 @@ func TestStorage_LoadFromDB(t *testing.T) { {[]byte("key1"), []byte("value1")}, {[]byte("key2"), []byte("value2")}, {[]byte("xyzKey1"), []byte("xyzValue1")}, + {[]byte("long"), []byte("newvaluewithmorethan32byteslength")}, } for _, kv := range trieKV { @@ -147,7 +148,54 @@ func TestStorage_LoadFromDB(t *testing.T) { entries, err := storage.Entries(&root) require.NoError(t, err) - require.Equal(t, 4, len(entries)) + require.Equal(t, 5, len(entries)) +} + +func TestStorage_LoadFromDB_v1(t *testing.T) { + storage := newTestStorageState(t) + ts, err := storage.TrieState(&trie.EmptyHash) + require.NoError(t, err) + + trieKV := []struct { + key []byte + value []byte + }{ + {value: []byte{}}, + {[]byte("key1"), []byte("value1")}, + {[]byte("key2"), []byte("value2")}, + {[]byte("xyzKey1"), []byte("xyzValue1")}, + {[]byte("long"), []byte("newvaluewithmorethan32byteslength")}, + } + + for _, kv := range trieKV { + ts.Put(kv.key, kv.value, trie.V1) + } + + root, err := ts.Root() + require.NoError(t, err) + + // Write trie to disk. + err = storage.StoreTrie(ts, nil) + require.NoError(t, err) + + // Clear trie from cache and fetch data from disk. + storage.blockState.tries.delete(root) + + data, err := storage.GetStorage(&root, trieKV[0].key) + require.NoError(t, err) + require.Equal(t, trieKV[0].value, data) + + storage.blockState.tries.delete(root) + + prefixKeys, err := storage.GetKeysWithPrefix(&root, []byte("ke")) + require.NoError(t, err) + require.Equal(t, 2, len(prefixKeys)) + + storage.blockState.tries.delete(root) + + entries, err := storage.Entries(&root) + require.NoError(t, err) + require.Equal(t, 5, len(entries)) } func TestStorage_StoreTrie_NotSyncing(t *testing.T) { diff --git a/lib/runtime/genesis.go b/lib/runtime/genesis.go index b1547c3611..c1c5cb4d94 100644 --- a/lib/runtime/genesis.go +++ b/lib/runtime/genesis.go @@ -26,7 +26,6 @@ func NewTrieFromGenesis(gen genesis.Genesis) (tr trie.Trie, err error) { ErrGenesisTopNotFound, gen.Name) } - //TODO: check if we could get the trie version from genesis tr, err = trie.LoadFromMap(keyValues, trie.V0) if err != nil { return tr, fmt.Errorf("loading genesis top key values into trie: %w", err) diff --git a/lib/trie/database.go b/lib/trie/database.go index 7e5c07bef4..e4335dd2cd 100644 --- a/lib/trie/database.go +++ b/lib/trie/database.go @@ -11,18 +11,9 @@ import ( "github.com/ChainSafe/gossamer/internal/trie/codec" "github.com/ChainSafe/gossamer/internal/trie/node" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/trie/db" ) -// DBGetter gets a value corresponding to the given key. -type DBGetter interface { - Get(key []byte) (value []byte, err error) -} - -// DBPutter puts a value at the given key and returns an error. -type DBPutter interface { - Put(key []byte, value []byte) error -} - // NewBatcher creates a new database batch. type NewBatcher interface { NewBatch() database.Batch @@ -30,7 +21,7 @@ type NewBatcher interface { // Load reconstructs the trie from the database from the given root hash. // It is used when restarting the node to load the current state trie. -func (t *Trie) Load(db DBGetter, rootHash common.Hash) error { +func (t *Trie) Load(db db.DBGetter, rootHash common.Hash) error { if rootHash == EmptyHash { t.root = nil return nil @@ -54,7 +45,7 @@ func (t *Trie) Load(db DBGetter, rootHash common.Hash) error { return t.loadNode(db, t.root) } -func (t *Trie) loadNode(db DBGetter, n *Node) error { +func (t *Trie) loadNode(db db.DBGetter, n *Node) error { if n.Kind() != node.Branch { return nil } @@ -196,7 +187,7 @@ func recordAllDeleted(n *Node, recorder DeltaRecorder) { // It recursively descends into the trie using the database starting // from the root node until it reaches the node with the given key. // It then reads the value from the database. -func GetFromDB(db DBGetter, rootHash common.Hash, key []byte) ( +func GetFromDB(db db.DBGetter, rootHash common.Hash, key []byte) ( value []byte, err error) { if rootHash == EmptyHash { return nil, nil @@ -222,7 +213,7 @@ func GetFromDB(db DBGetter, rootHash common.Hash, key []byte) ( // for the value corresponding to a key. // Note it does not copy the value so modifying the value bytes // slice will modify the value of the node in the trie. -func getFromDBAtNode(db DBGetter, n *Node, key []byte) ( +func getFromDBAtNode(db db.DBGetter, n *Node, key []byte) ( value []byte, err error) { if n.Kind() == node.Leaf { if bytes.Equal(n.PartialKey, key) { @@ -289,7 +280,7 @@ func (t *Trie) WriteDirty(db NewBatcher) error { return batch.Flush() } -func (t *Trie) writeDirtyNode(db DBPutter, n *Node) (err error) { +func (t *Trie) writeDirtyNode(db db.DBPutter, n *Node) (err error) { if n == nil || !n.Dirty { return nil } diff --git a/lib/trie/db/db.go b/lib/trie/db/db.go index e5d38ddc2c..779380cca0 100644 --- a/lib/trie/db/db.go +++ b/lib/trie/db/db.go @@ -9,6 +9,27 @@ import ( "github.com/ChainSafe/gossamer/lib/common" ) +type Database interface { + DBGetter + DBPutter + Copier +} + +// DBGetter gets a value corresponding to the given key. +type DBGetter interface { + Get(key []byte) (value []byte, err error) +} + +// DBPutter puts a value at the given key and returns an error. +type DBPutter interface { + Put(key []byte, value []byte) error +} + +// DBCopy creates a deep copy from a source DB +type Copier interface { + Copy() Database +} + type MemoryDB struct { data map[common.Hash][]byte l sync.RWMutex @@ -38,6 +59,18 @@ func NewMemoryDBFromProof(encodedNodes [][]byte) (*MemoryDB, error) { } +func (mdb *MemoryDB) Copy() Database { + newDB := NewEmptyMemoryDB() + copyData := make(map[common.Hash][]byte, len(mdb.data)) + + for k, v := range mdb.data { + copyData[k] = v + } + + newDB.data = copyData + return newDB +} + func (mdb *MemoryDB) Get(key []byte) ([]byte, error) { if len(key) != common.HashLength { return nil, fmt.Errorf("expected %d bytes length key, given %d (%x)", common.HashLength, len(key), key) diff --git a/lib/trie/proof/generate.go b/lib/trie/proof/generate.go index 6155ebac1c..861a8787d6 100644 --- a/lib/trie/proof/generate.go +++ b/lib/trie/proof/generate.go @@ -13,6 +13,7 @@ import ( "github.com/ChainSafe/gossamer/internal/trie/pools" "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/lib/trie/db" ) var ( @@ -22,8 +23,8 @@ var ( // Database defines a key value Get method used // for proof generation. type Database interface { - trie.DBGetter - trie.DBPutter + db.DBGetter + db.DBPutter } // Generate generates and deduplicates the encoded proof nodes diff --git a/lib/trie/proof/verify.go b/lib/trie/proof/verify.go index 1e8457f04f..97e439fdec 100644 --- a/lib/trie/proof/verify.go +++ b/lib/trie/proof/verify.go @@ -62,7 +62,7 @@ var ( ) // buildTrie sets a partial trie based on the proof slice of encoded nodes. -func buildTrie(encodedProofNodes [][]byte, rootHash []byte, db Database) (t *trie.Trie, err error) { +func buildTrie(encodedProofNodes [][]byte, rootHash []byte, db db.Database) (t *trie.Trie, err error) { if len(encodedProofNodes) == 0 { return nil, fmt.Errorf("%w: for Merkle root hash 0x%x", ErrEmptyProof, rootHash) diff --git a/lib/trie/proof/verify_test.go b/lib/trie/proof/verify_test.go index db969e1619..78e0e5dd50 100644 --- a/lib/trie/proof/verify_test.go +++ b/lib/trie/proof/verify_test.go @@ -141,7 +141,7 @@ func Test_buildTrie(t *testing.T) { encodedProofNodes [][]byte rootHash []byte expectedTrie *trie.Trie - db Database + db db.Database errWrapped error errMessage string } diff --git a/lib/trie/trie.go b/lib/trie/trie.go index 8ebe4c1da6..a32d104a04 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -17,17 +17,12 @@ import ( // EmptyHash is the empty trie hash. var EmptyHash = common.MustBlake2bHash([]byte{0}) -type Database interface { - DBGetter - DBPutter -} - // Trie is a base 16 modified Merkle Patricia trie. type Trie struct { generation uint64 root *Node childTries map[common.Hash]*Trie - db Database + db db.Database // deltas stores trie deltas since the last trie snapshot. // For example node hashes that were deleted since // the last snapshot. These are used by the online @@ -42,7 +37,7 @@ func NewEmptyTrie() *Trie { } // NewTrie creates a trie with an existing root node -func NewTrie(root *Node, db Database) *Trie { +func NewTrie(root *Node, db db.Database) *Trie { return &Trie{ root: root, childTries: make(map[common.Hash]*Trie), @@ -69,9 +64,15 @@ func (t *Trie) Snapshot() (newTrie *Trie) { } } + var dbCopy db.Database + if t.db != nil { + dbCopy = t.db.Copy() + } + return &Trie{ generation: t.generation + 1, root: t.root, + db: dbCopy, childTries: childTries, deltas: tracking.New(), } @@ -165,6 +166,10 @@ func (t *Trie) DeepCopy() (trieCopy *Trie) { trieCopy.root = t.root.Copy(copySettings) } + if t.db != nil { + trieCopy.db = t.db.Copy() + } + return trieCopy } @@ -213,6 +218,7 @@ func (t *Trie) buildEntriesMap(currentNode *Node, prefix []byte, kv map[string][ return } + // Leaf if currentNode.Kind() == node.Leaf { key := currentNode.PartialKey fullKeyNibbles := concatenateSlices(prefix, key) @@ -221,6 +227,7 @@ func (t *Trie) buildEntriesMap(currentNode *Node, prefix []byte, kv map[string][ return } + // Branch branch := currentNode if branch.StorageValue != nil { fullKeyNibbles := concatenateSlices(prefix, branch.PartialKey) @@ -618,11 +625,14 @@ func LoadFromMap(data map[string]string, version Version) (trie Trie, err error) }() for key, value := range data { + //"0x1cb6f36e027abb2091cfb5110ab5087f5e0621c4869aa60c02be9adcc98a0d1d" + //"\x1c\xb6\xf3n\x02z\xbb \x91ϵ\x11\n\xb5\b\x7f^\x06!Ć\x9a\xa6\f\x02\xbe\x9a\xdcɊ\r\x1d" keyLEBytes, err := common.HexToBytes(key) if err != nil { return Trie{}, fmt.Errorf("cannot convert key hex to bytes: %w", err) } + //"0x04d43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d0100000000000000" valueBytes, err := common.HexToBytes(value) if err != nil { return Trie{}, fmt.Errorf("cannot convert value hex to bytes: %w", err) @@ -747,7 +757,7 @@ func (t *Trie) Get(keyLE []byte) (value []byte) { return retrieve(t.db, t.root, keyNibbles) } -func retrieve(db DBGetter, parent *Node, key []byte) (value []byte) { +func retrieve(db db.DBGetter, parent *Node, key []byte) (value []byte) { if parent == nil { return nil } @@ -758,7 +768,7 @@ func retrieve(db DBGetter, parent *Node, key []byte) (value []byte) { return retrieveFromBranch(db, parent, key) } -func retrieveFromLeaf(db DBGetter, leaf *Node, key []byte) (value []byte) { +func retrieveFromLeaf(db db.DBGetter, leaf *Node, key []byte) (value []byte) { if bytes.Equal(leaf.PartialKey, key) { if leaf.HashedValue { // We get the node @@ -773,7 +783,7 @@ func retrieveFromLeaf(db DBGetter, leaf *Node, key []byte) (value []byte) { return nil } -func retrieveFromBranch(db DBGetter, branch *Node, key []byte) (value []byte) { +func retrieveFromBranch(db db.DBGetter, branch *Node, key []byte) (value []byte) { if len(key) == 0 || bytes.Equal(branch.PartialKey, key) { return branch.StorageValue } diff --git a/lib/trie/trie_test.go b/lib/trie/trie_test.go index 6ca47ce25a..d62a4138c2 100644 --- a/lib/trie/trie_test.go +++ b/lib/trie/trie_test.go @@ -1161,7 +1161,7 @@ func Test_Trie_Put(t *testing.T) { }, }), }, - db: func() Database { + db: func() db.Database { db := db.NewEmptyMemoryDB() db.Put(longValueHash, longValue) return db @@ -2171,7 +2171,7 @@ func Test_retrieve(t *testing.T) { parent *Node key []byte value []byte - db DBGetter + db db.DBGetter }{ "nil_parent": { key: []byte{1}, @@ -2280,7 +2280,7 @@ func Test_retrieve(t *testing.T) { }, key: []byte{1, 2, 3, 4, 5}, value: hashedValueResult, - db: func() DBGetter { + db: func() db.DBGetter { defaultDBGetterMock := NewMockDBGetter(ctrl) defaultDBGetterMock.EXPECT().Get(gomock.Any()).Return(hashedValueResult, nil).Times(1) From 3da0d325331f50308934efc7f473453d24a55631 Mon Sep 17 00:00:00 2001 From: Diego Date: Fri, 1 Sep 2023 17:16:48 +0200 Subject: [PATCH 059/128] Fix DBGetter mock --- lib/trie/db_getter_mocks_test.go | 2 +- lib/trie/mocks_generate_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/trie/db_getter_mocks_test.go b/lib/trie/db_getter_mocks_test.go index e5a31ea71d..ecf5a52dbb 100644 --- a/lib/trie/db_getter_mocks_test.go +++ b/lib/trie/db_getter_mocks_test.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/ChainSafe/gossamer/lib/trie (interfaces: DBGetter) +// Source: github.com/ChainSafe/gossamer/lib/trie/db (interfaces: DBGetter) // Package trie is a generated GoMock package. package trie diff --git a/lib/trie/mocks_generate_test.go b/lib/trie/mocks_generate_test.go index 767f1a8aed..f101c2239c 100644 --- a/lib/trie/mocks_generate_test.go +++ b/lib/trie/mocks_generate_test.go @@ -3,4 +3,4 @@ package trie -//go:generate mockgen -destination=db_getter_mocks_test.go -package=$GOPACKAGE . DBGetter +//go:generate mockgen -destination=db_getter_mocks_test.go -package=$GOPACKAGE github.com/ChainSafe/gossamer/lib/trie/db DBGetter From 919468c9f84390837160c53e2e5ef6cb0274ae6d Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 4 Sep 2023 15:38:14 +0200 Subject: [PATCH 060/128] Remove comments --- lib/trie/trie.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/trie/trie.go b/lib/trie/trie.go index a32d104a04..4c33f0a1f1 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -625,14 +625,11 @@ func LoadFromMap(data map[string]string, version Version) (trie Trie, err error) }() for key, value := range data { - //"0x1cb6f36e027abb2091cfb5110ab5087f5e0621c4869aa60c02be9adcc98a0d1d" - //"\x1c\xb6\xf3n\x02z\xbb \x91ϵ\x11\n\xb5\b\x7f^\x06!Ć\x9a\xa6\f\x02\xbe\x9a\xdcɊ\r\x1d" keyLEBytes, err := common.HexToBytes(key) if err != nil { return Trie{}, fmt.Errorf("cannot convert key hex to bytes: %w", err) } - //"0x04d43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d0100000000000000" valueBytes, err := common.HexToBytes(value) if err != nil { return Trie{}, fmt.Errorf("cannot convert value hex to bytes: %w", err) From b94c622d891f762011fb548d7357791cdf41b1c3 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 5 Sep 2023 23:12:18 +0200 Subject: [PATCH 061/128] Fix get version in runtime functions --- lib/runtime/wazero/imports.go | 19 ++++++++++++++----- lib/runtime/wazero/imports_test.go | 27 +++++++-------------------- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/lib/runtime/wazero/imports.go b/lib/runtime/wazero/imports.go index fe99a03bfd..dde47bb55f 100644 --- a/lib/runtime/wazero/imports.go +++ b/lib/runtime/wazero/imports.go @@ -865,7 +865,8 @@ func ext_trie_blake2_256_root_version_2(ctx context.Context, m api.Module, dataS panic("nil runtime context") } - stateVersion := uint8(version) + stateVersionBytes, _ := m.Memory().Read(version, 4) + stateVersion := uint8(binary.LittleEndian.Uint32(stateVersionBytes)) trie.EnforceValidVersion(stateVersion) data := read(m, dataSpan) @@ -969,11 +970,11 @@ func ext_trie_blake2_256_ordered_root_version_2( panic("nil runtime context") } - stateVersion := uint8(version) - trie.EnforceValidVersion(stateVersion) - data := read(m, dataSpan) + stateVersionBytes, _ := m.Memory().Read(version, 4) + stateVersion := binary.LittleEndian.Uint32(stateVersionBytes) + t := trie.NewEmptyTrie() var values [][]byte err := scale.Unmarshal(data, &values) @@ -1051,12 +1052,16 @@ func ext_trie_blake2_256_verify_proof_version_1( } func ext_trie_blake2_256_verify_proof_version_2( - ctx context.Context, m api.Module, rootSpan uint32, proofSpan, keySpan, valueSpan uint64, version uint32) uint32 { //skipcq: RVV-B0012 + ctx context.Context, m api.Module, rootSpan uint32, proofSpan, keySpan, valueSpan uint64, version uint32) uint32 { rtCtx := ctx.Value(runtimeContextKey).(*runtime.Context) if rtCtx == nil { panic("nil runtime context") } + stateVersionBytes, _ := m.Memory().Read(version, 4) + stateVersion := uint8(binary.LittleEndian.Uint32(stateVersionBytes)) + trie.EnforceValidVersion(stateVersion) + toDecProofs := read(m, proofSpan) var encodedProofNodes [][]byte err := scale.Unmarshal(toDecProofs, &encodedProofNodes) @@ -2427,6 +2432,10 @@ func ext_storage_root_version_2(ctx context.Context, m api.Module, version uint3 } storage := rtCtx.Storage + stateVersionBytes, _ := m.Memory().Read(version, 4) + stateVersion := uint8(binary.LittleEndian.Uint32(stateVersionBytes)) + trie.EnforceValidVersion(stateVersion) + root, err := storage.Root() if err != nil { logger.Errorf("failed to get storage root: %s", err) diff --git a/lib/runtime/wazero/imports_test.go b/lib/runtime/wazero/imports_test.go index 499d7083a1..a8dd94d493 100644 --- a/lib/runtime/wazero/imports_test.go +++ b/lib/runtime/wazero/imports_test.go @@ -571,7 +571,6 @@ func Test_ext_trie_blake2_256_root_version_1(t *testing.T) { require.Equal(t, expected[:], hash) } -// TODO: revisit it func Test_ext_trie_blake2_256_root_version_2(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) @@ -584,10 +583,7 @@ func Test_ext_trie_blake2_256_root_version_2(t *testing.T) { stateVersionBytes := make([]byte, 4) binary.LittleEndian.PutUint32(stateVersionBytes, stateVersion) - encVersion, err := scale.Marshal(&stateVersionBytes) - require.NoError(t, err) - - data := append(encInput, encVersion...) + data := append(encInput, stateVersionBytes...) res, err := inst.Exec("rtm_ext_trie_blake2_256_root_version_2", data) require.NoError(t, err) @@ -633,10 +629,7 @@ func Test_ext_trie_blake2_256_ordered_root_version_2(t *testing.T) { stateVersionBytes := make([]byte, 4) binary.LittleEndian.PutUint32(stateVersionBytes, stateVersion) - encVersion, err := scale.Marshal(&stateVersionBytes) - require.NoError(t, err) - - data := append(encValues, encVersion...) + data := append(encValues, stateVersionBytes...) res, err := inst.Exec("rtm_ext_trie_blake2_256_ordered_root_version_2", data) require.NoError(t, err) @@ -826,10 +819,7 @@ func Test_ext_trie_blake2_256_verify_proof_version_2(t *testing.T) { stateVersionBytes := make([]byte, 4) binary.LittleEndian.PutUint32(stateVersionBytes, stateVersion) - encVersion, err := scale.Marshal(&stateVersionBytes) - require.NoError(t, err) - - args = append(args, encVersion...) + args = append(args, stateVersionBytes...) res, err := inst.Exec("rtm_ext_trie_blake2_256_verify_proof_version_2", args) require.NoError(t, err) @@ -1178,10 +1168,10 @@ func Test_ext_default_child_storage_root_version_2(t *testing.T) { require.NoError(t, err) stateVersion := uint32(trie.V1) - stateVersionBytes := make([]byte, 4) - binary.LittleEndian.PutUint32(stateVersionBytes, stateVersion) + //stateVersionBytes := make([]byte, 4) + //binary.LittleEndian.PutUint32(stateVersionBytes, stateVersion) - encVersion, err := scale.Marshal(&stateVersionBytes) + encVersion, err := scale.Marshal(&stateVersion) require.NoError(t, err) data := append(encChildKey, encKey...) @@ -2047,10 +2037,7 @@ func Test_ext_storage_root_version_2(t *testing.T) { stateVersionBytes := make([]byte, 4) binary.LittleEndian.PutUint32(stateVersionBytes, stateVersion) - encVersion, err := scale.Marshal(&stateVersionBytes) - require.NoError(t, err) - - ret, err := inst.Exec("rtm_ext_storage_root_version_2", encVersion) + ret, err := inst.Exec("rtm_ext_storage_root_version_2", stateVersionBytes) require.NoError(t, err) var hash []byte From 48ccf915027f7045f539e4d5217f88bfc531e7a8 Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 7 Sep 2023 17:11:39 +0200 Subject: [PATCH 062/128] Fix merge --- lib/runtime/wazero/instance.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/runtime/wazero/instance.go b/lib/runtime/wazero/instance.go index 693e15fa74..de0ba4bf41 100644 --- a/lib/runtime/wazero/instance.go +++ b/lib/runtime/wazero/instance.go @@ -207,13 +207,7 @@ func NewInstance(code []byte, cfg Config) (instance *Instance, err error) { WithFunc(ext_trie_blake2_256_root_version_1). Export("ext_trie_blake2_256_root_version_1"). NewFunctionBuilder(). -<<<<<<< HEAD WithFunc(ext_trie_blake2_256_root_version_2). -======= - WithFunc(func(a int64, v int32) int32 { - panic("ext_trie_blake2_256_root_version_2 unimplemented") - }). ->>>>>>> 919468c9f84390837160c53e2e5ef6cb0274ae6d Export("ext_trie_blake2_256_root_version_2"). NewFunctionBuilder(). WithFunc(ext_trie_blake2_256_ordered_root_version_1). From f69553005bdc4c69975e158859ab4d20e5479e74 Mon Sep 17 00:00:00 2001 From: Diego Date: Fri, 8 Sep 2023 11:30:25 +0200 Subject: [PATCH 063/128] Improve version parsing and simplify checks --- lib/runtime/wazero/imports.go | 35 ++++++++++++++++++++++---------- lib/trie/version.go | 23 ++++++++++----------- lib/trie/version_test.go | 38 ++++++++++++++++++++++++++++------- 3 files changed, 66 insertions(+), 30 deletions(-) diff --git a/lib/runtime/wazero/imports.go b/lib/runtime/wazero/imports.go index 8ef1ecba04..8f87deb1b3 100644 --- a/lib/runtime/wazero/imports.go +++ b/lib/runtime/wazero/imports.go @@ -823,8 +823,11 @@ func ext_trie_blake2_256_root_version_2(ctx context.Context, m api.Module, dataS } stateVersionBytes, _ := m.Memory().Read(version, 4) - stateVersion := uint8(binary.LittleEndian.Uint32(stateVersionBytes)) - trie.EnforceValidVersion(stateVersion) + stateVersion, err := trie.ParseVersion(binary.LittleEndian.Uint32(stateVersionBytes)) + if err != nil { + logger.Errorf("failed parsing state version: %s", err) + return 0 + } data := read(m, dataSpan) @@ -842,7 +845,7 @@ func ext_trie_blake2_256_root_version_2(ctx context.Context, m api.Module, dataS } for _, kv := range kvs { - err := t.Put(kv.Key, kv.Value, trie.Version(stateVersion)) + err := t.Put(kv.Key, kv.Value, stateVersion) if err != nil { logger.Errorf("failed putting key 0x%x and value 0x%x into trie: %s", kv.Key, kv.Value, err) @@ -930,11 +933,15 @@ func ext_trie_blake2_256_ordered_root_version_2( data := read(m, dataSpan) stateVersionBytes, _ := m.Memory().Read(version, 4) - stateVersion := binary.LittleEndian.Uint32(stateVersionBytes) + stateVersion, err := trie.ParseVersion(binary.LittleEndian.Uint32(stateVersionBytes)) + if err != nil { + logger.Errorf("failed parsing state version: %s", err) + return 0 + } t := trie.NewEmptyTrie() var values [][]byte - err := scale.Unmarshal(data, &values) + err = scale.Unmarshal(data, &values) if err != nil { logger.Errorf("failed scale decoding data: %s", err) return 0 @@ -950,7 +957,7 @@ func ext_trie_blake2_256_ordered_root_version_2( "put key=0x%x and value=0x%x", key, value) - err = t.Put(key, value, trie.Version(stateVersion)) + err = t.Put(key, value, stateVersion) if err != nil { logger.Errorf("failed putting key 0x%x and value 0x%x into trie: %s", key, value, err) @@ -1016,12 +1023,15 @@ func ext_trie_blake2_256_verify_proof_version_2( } stateVersionBytes, _ := m.Memory().Read(version, 4) - stateVersion := uint8(binary.LittleEndian.Uint32(stateVersionBytes)) - trie.EnforceValidVersion(stateVersion) + _, err := trie.ParseVersion(binary.LittleEndian.Uint32(stateVersionBytes)) + if err != nil { + logger.Errorf("failed parsing state version: %s", err) + return 0 + } toDecProofs := read(m, proofSpan) var encodedProofNodes [][]byte - err := scale.Unmarshal(toDecProofs, &encodedProofNodes) + err = scale.Unmarshal(toDecProofs, &encodedProofNodes) if err != nil { logger.Errorf("failed scale decoding proof data: %s", err) return uint32(0) @@ -2354,8 +2364,11 @@ func ext_storage_root_version_2(ctx context.Context, m api.Module, version uint3 storage := rtCtx.Storage stateVersionBytes, _ := m.Memory().Read(version, 4) - stateVersion := uint8(binary.LittleEndian.Uint32(stateVersionBytes)) - trie.EnforceValidVersion(stateVersion) + _, err := trie.ParseVersion(binary.LittleEndian.Uint32(stateVersionBytes)) + if err != nil { + logger.Errorf("failed parsing state version: %s", err) + return 0 + } root, err := storage.Root() if err != nil { diff --git a/lib/trie/version.go b/lib/trie/version.go index 84e1105a80..c4049c749a 100644 --- a/lib/trie/version.go +++ b/lib/trie/version.go @@ -55,7 +55,17 @@ func (v Version) ShouldHashValue(value []byte) bool { var ErrParseVersion = errors.New("parsing version failed") // ParseVersion parses a state trie version string. -func ParseVersion(s string) (version Version, err error) { +func ParseVersion[T string | uint32](v T) (version Version, err error) { + var s string + switch value := any(v).(type) { + case string: + s = value + case uint32: + s = fmt.Sprintf("V%d", value) + default: + panic(fmt.Sprintf("unsupported type %T", s)) + } + switch { case strings.EqualFold(s, V0.String()): return V0, nil @@ -66,14 +76,3 @@ func ParseVersion(s string) (version Version, err error) { ErrParseVersion, s, V0, V1) } } - -func EnforceValidVersion(version uint8) { - v := Version(version) - - switch v { - case V0, V1: - return - default: - panic("Invalid state version") - } -} diff --git a/lib/trie/version_test.go b/lib/trie/version_test.go index 93fe755b76..ef1af1ca8b 100644 --- a/lib/trie/version_test.go +++ b/lib/trie/version_test.go @@ -4,6 +4,7 @@ package trie import ( + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -49,32 +50,45 @@ func Test_ParseVersion(t *testing.T) { t.Parallel() testCases := map[string]struct { - s string + v any version Version errWrapped error errMessage string }{ "v0": { - s: "v0", + v: "v0", version: V0, }, "V0": { - s: "V0", + v: "V0", + version: V0, + }, + "0": { + v: uint32(0), version: V0, }, "v1": { - s: "v1", + v: "v1", version: V1, }, "V1": { - s: "V1", + v: "V1", + version: V1, + }, + "1": { + v: uint32(1), version: V1, }, "invalid": { - s: "xyz", + v: "xyz", errWrapped: ErrParseVersion, errMessage: "parsing version failed: \"xyz\" must be one of [v0, v1]", }, + "invalid uint32": { + v: uint32(999), + errWrapped: ErrParseVersion, + errMessage: "parsing version failed: \"V999\" must be one of [v0, v1]", + }, } for name, testCase := range testCases { @@ -82,7 +96,17 @@ func Test_ParseVersion(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - version, err := ParseVersion(testCase.s) + var version Version + + var err error + switch typed := testCase.v.(type) { + case string: + version, err = ParseVersion(typed) + case uint32: + version, err = ParseVersion(typed) + default: + panic(fmt.Sprintf("unsupported type %T", testCase.v)) + } assert.Equal(t, testCase.version, version) assert.ErrorIs(t, err, testCase.errWrapped) From 31413a45761841a517d0bdd2e9a6f529d4561b30 Mon Sep 17 00:00:00 2001 From: Diego Date: Fri, 8 Sep 2023 11:39:59 +0200 Subject: [PATCH 064/128] lint --- lib/trie/version_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/trie/version_test.go b/lib/trie/version_test.go index ef1af1ca8b..9e85010f79 100644 --- a/lib/trie/version_test.go +++ b/lib/trie/version_test.go @@ -84,7 +84,7 @@ func Test_ParseVersion(t *testing.T) { errWrapped: ErrParseVersion, errMessage: "parsing version failed: \"xyz\" must be one of [v0, v1]", }, - "invalid uint32": { + "invalid_uint32": { v: uint32(999), errWrapped: ErrParseVersion, errMessage: "parsing version failed: \"V999\" must be one of [v0, v1]", From 215d4b321109bd3226a0cdd550da12f64facc52b Mon Sep 17 00:00:00 2001 From: Diego Date: Fri, 8 Sep 2023 14:27:26 +0200 Subject: [PATCH 065/128] PR comments --- internal/trie/node/copy.go | 2 +- internal/trie/node/copy_test.go | 44 ++++++++++++++--------------- internal/trie/node/decode.go | 4 +-- internal/trie/node/decode_test.go | 14 +++++----- internal/trie/node/encode.go | 2 +- internal/trie/node/encode_test.go | 30 ++++++++++---------- internal/trie/node/header.go | 4 +-- internal/trie/node/header_test.go | 10 +++---- internal/trie/node/node.go | 4 +-- lib/trie/trie.go | 46 +++++++++++++++---------------- lib/trie/trie_test.go | 16 +++++------ lib/trie/version.go | 4 +-- 12 files changed, 89 insertions(+), 91 deletions(-) diff --git a/internal/trie/node/copy.go b/internal/trie/node/copy.go index c8f0d8f001..f622419456 100644 --- a/internal/trie/node/copy.go +++ b/internal/trie/node/copy.go @@ -90,7 +90,7 @@ func (n *Node) Copy(settings CopySettings) *Node { if settings.CopyStorageValue && n.StorageValue != nil { cpy.StorageValue = make([]byte, len(n.StorageValue)) copy(cpy.StorageValue, n.StorageValue) - cpy.HashedValue = n.HashedValue + cpy.IsHashedValue = n.IsHashedValue } if settings.CopyMerkleValue { diff --git a/internal/trie/node/copy_test.go b/internal/trie/node/copy_test.go index c9a2268cf2..cf080b4ca8 100644 --- a/internal/trie/node/copy_test.go +++ b/internal/trie/node/copy_test.go @@ -112,14 +112,14 @@ func Test_Node_Copy(t *testing.T) { }, "deep_copy_branch_with_hashed_values": { node: &Node{ - PartialKey: []byte{1, 2}, - StorageValue: []byte{3, 4}, - HashedValue: true, + PartialKey: []byte{1, 2}, + StorageValue: []byte{3, 4}, + IsHashedValue: true, Children: padRightChildren([]*Node{ nil, nil, { - PartialKey: []byte{9}, - StorageValue: []byte{1}, - HashedValue: true, + PartialKey: []byte{9}, + StorageValue: []byte{1}, + IsHashedValue: true, }, }), Dirty: true, @@ -127,14 +127,14 @@ func Test_Node_Copy(t *testing.T) { }, settings: DeepCopySettings, expectedNode: &Node{ - PartialKey: []byte{1, 2}, - StorageValue: []byte{3, 4}, - HashedValue: true, + PartialKey: []byte{1, 2}, + StorageValue: []byte{3, 4}, + IsHashedValue: true, Children: padRightChildren([]*Node{ nil, nil, { - PartialKey: []byte{9}, - StorageValue: []byte{1}, - HashedValue: true, + PartialKey: []byte{9}, + StorageValue: []byte{1}, + IsHashedValue: true, }, }), Dirty: true, @@ -172,19 +172,19 @@ func Test_Node_Copy(t *testing.T) { }, "deep_copy_leaf_with_hashed_value": { node: &Node{ - PartialKey: []byte{1, 2}, - StorageValue: []byte{3, 4}, - HashedValue: true, - Dirty: true, - MerkleValue: []byte{5}, + PartialKey: []byte{1, 2}, + StorageValue: []byte{3, 4}, + IsHashedValue: true, + Dirty: true, + MerkleValue: []byte{5}, }, settings: DeepCopySettings, expectedNode: &Node{ - PartialKey: []byte{1, 2}, - StorageValue: []byte{3, 4}, - HashedValue: true, - Dirty: true, - MerkleValue: []byte{5}, + PartialKey: []byte{1, 2}, + StorageValue: []byte{3, 4}, + IsHashedValue: true, + Dirty: true, + MerkleValue: []byte{5}, }, }, } diff --git a/internal/trie/node/decode.go b/internal/trie/node/decode.go index 1e2cf5c614..2c7c22f553 100644 --- a/internal/trie/node/decode.go +++ b/internal/trie/node/decode.go @@ -96,7 +96,7 @@ func decodeBranch(reader io.Reader, variant variant, partialKeyLength uint16) ( return nil, err } node.StorageValue = hashedValue - node.HashedValue = true + node.IsHashedValue = true default: // Ignored } @@ -150,7 +150,7 @@ func decodeLeaf(reader io.Reader, variant variant, partialKeyLength uint16) (nod return nil, err } node.StorageValue = hashedValue - node.HashedValue = true + node.IsHashedValue = true return node, nil } diff --git a/internal/trie/node/decode_test.go b/internal/trie/node/decode_test.go index 56983161b7..49066d641e 100644 --- a/internal/trie/node/decode_test.go +++ b/internal/trie/node/decode_test.go @@ -109,9 +109,9 @@ func Test_Decode(t *testing.T) { hashedValue.ToBytes(), })), n: &Node{ - PartialKey: []byte{9}, - StorageValue: hashedValue.ToBytes(), - HashedValue: true, + PartialKey: []byte{9}, + StorageValue: hashedValue.ToBytes(), + IsHashedValue: true, }, }, "leaf_with_hashed_value_fail_too_short": { @@ -131,10 +131,10 @@ func Test_Decode(t *testing.T) { hashedValue.ToBytes(), })), n: &Node{ - PartialKey: []byte{9}, - Children: make([]*Node, ChildrenCapacity), - StorageValue: hashedValue.ToBytes(), - HashedValue: true, + PartialKey: []byte{9}, + Children: make([]*Node, ChildrenCapacity), + StorageValue: hashedValue.ToBytes(), + IsHashedValue: true, }, }, "branch_with_hashed_value_fail_too_short": { diff --git a/internal/trie/node/encode.go b/internal/trie/node/encode.go index 90eb7b7d94..87821dd534 100644 --- a/internal/trie/node/encode.go +++ b/internal/trie/node/encode.go @@ -49,7 +49,7 @@ func (n *Node) Encode(buffer Buffer) (err error) { // even if it is empty. Do not encode if the branch is without value. // Note leaves and branches with value cannot have a `nil` storage value. if n.StorageValue != nil { - if n.HashedValue { + if n.IsHashedValue { if len(n.StorageValue) != common.HashLength { return fmt.Errorf("%w: expected %d, got: %d", ErrEncodeHashedValueTooShort, common.HashLength, len(n.StorageValue)) } diff --git a/internal/trie/node/encode_test.go b/internal/trie/node/encode_test.go index 53707e5a3f..1593950cbd 100644 --- a/internal/trie/node/encode_test.go +++ b/internal/trie/node/encode_test.go @@ -119,9 +119,9 @@ func Test_Node_Encode(t *testing.T) { }, "leaf_with_hashed_value_success": { node: &Node{ - PartialKey: []byte{1, 2, 3}, - StorageValue: hashedValue.ToBytes(), - HashedValue: true, + PartialKey: []byte{1, 2, 3}, + StorageValue: hashedValue.ToBytes(), + IsHashedValue: true, }, writes: []writeCall{ { @@ -133,9 +133,9 @@ func Test_Node_Encode(t *testing.T) { }, "leaf_with_hashed_value_fail": { node: &Node{ - PartialKey: []byte{1, 2, 3}, - StorageValue: hashedValue.ToBytes(), - HashedValue: true, + PartialKey: []byte{1, 2, 3}, + StorageValue: hashedValue.ToBytes(), + IsHashedValue: true, }, writes: []writeCall{ { @@ -154,9 +154,9 @@ func Test_Node_Encode(t *testing.T) { }, "leaf_with_hashed_value_fail_too_short": { node: &Node{ - PartialKey: []byte{1, 2, 3}, - StorageValue: []byte("tooshort"), - HashedValue: true, + PartialKey: []byte{1, 2, 3}, + StorageValue: []byte("tooshort"), + IsHashedValue: true, }, writes: []writeCall{ { @@ -340,9 +340,9 @@ func Test_Node_Encode(t *testing.T) { }, "branch_with_hashed_value_success": { node: &Node{ - PartialKey: []byte{1, 2, 3}, - StorageValue: hashedValue.ToBytes(), - HashedValue: true, + PartialKey: []byte{1, 2, 3}, + StorageValue: hashedValue.ToBytes(), + IsHashedValue: true, Children: []*Node{ nil, nil, nil, {PartialKey: []byte{9}, StorageValue: []byte{1}}, nil, nil, nil, {PartialKey: []byte{11}, StorageValue: []byte{1}}, @@ -371,9 +371,9 @@ func Test_Node_Encode(t *testing.T) { }, "branch_with_hashed_value_fail_too_short": { node: &Node{ - PartialKey: []byte{1, 2, 3}, - StorageValue: []byte("tooshort"), - HashedValue: true, + PartialKey: []byte{1, 2, 3}, + StorageValue: []byte("tooshort"), + IsHashedValue: true, Children: []*Node{ nil, nil, nil, {PartialKey: []byte{9}, StorageValue: []byte{1}}, nil, nil, nil, {PartialKey: []byte{11}, StorageValue: []byte{1}}, diff --git a/internal/trie/node/header.go b/internal/trie/node/header.go index 8863b410a8..806fe5a020 100644 --- a/internal/trie/node/header.go +++ b/internal/trie/node/header.go @@ -24,14 +24,14 @@ func encodeHeader(node *Node, writer io.Writer) (err error) { // Merge variant byte and partial key length together var nodeVariant variant if node.Kind() == Leaf { - if node.HashedValue { + if node.IsHashedValue { nodeVariant = leafWithHashedValueVariant } else { nodeVariant = leafVariant } } else if node.StorageValue == nil { nodeVariant = branchVariant - } else if node.HashedValue { + } else if node.IsHashedValue { nodeVariant = branchWithHashedValueVariant } else { nodeVariant = branchWithValueVariant diff --git a/internal/trie/node/header_test.go b/internal/trie/node/header_test.go index 483ffb2730..6690ba1167 100644 --- a/internal/trie/node/header_test.go +++ b/internal/trie/node/header_test.go @@ -47,9 +47,9 @@ func Test_encodeHeader(t *testing.T) { }, "branch_with_hashed_value": { node: &Node{ - StorageValue: hashedValue.ToBytes(), - HashedValue: true, - Children: make([]*Node, ChildrenCapacity), + StorageValue: hashedValue.ToBytes(), + IsHashedValue: true, + Children: make([]*Node, ChildrenCapacity), }, writes: []writeCall{ {written: []byte{branchWithHashedValueVariant.bits}}, @@ -126,8 +126,8 @@ func Test_encodeHeader(t *testing.T) { }, "leaf_with_hashed_value": { node: &Node{ - StorageValue: hashedValue.ToBytes(), - HashedValue: true, + StorageValue: hashedValue.ToBytes(), + IsHashedValue: true, }, writes: []writeCall{ {written: []byte{leafWithHashedValueVariant.bits}}, diff --git a/internal/trie/node/node.go b/internal/trie/node/node.go index c5a7c83ce0..e57416b0d2 100644 --- a/internal/trie/node/node.go +++ b/internal/trie/node/node.go @@ -16,8 +16,8 @@ type Node struct { // PartialKey is the partial key bytes in nibbles (0 to f in hexadecimal) PartialKey []byte StorageValue []byte - // HashedValue is true when the StorageValue is a blake2b hash - HashedValue bool + // IsHashedValue is true when the StorageValue is a blake2b hash + IsHashedValue bool // Generation is incremented on every trie Snapshot() call. // Each node also contain a certain Generation number, // which is updated to match the trie Generation once they are diff --git a/lib/trie/trie.go b/lib/trie/trie.go index 4c33f0a1f1..355c628521 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -352,8 +352,8 @@ func (t *Trie) insertKeyLE(keyLE, value []byte, value = []byte{} } - isValueHashed := version.ShouldHashValue(value) - if isValueHashed { + shouldHash := version.ShouldHashValue(value) + if shouldHash { hashedValue := common.MustBlake2bHash(value) // Add original value in db using the hashed value as key @@ -381,11 +381,11 @@ func (t *Trie) insert(parent *Node, key, value []byte, mutated = true nodesCreated = 1 return &Node{ - PartialKey: key, - StorageValue: value, - HashedValue: isValueHashed, - Generation: t.generation, - Dirty: true, + PartialKey: key, + StorageValue: value, + IsHashedValue: isValueHashed, + Generation: t.generation, + Dirty: true, }, mutated, nodesCreated, nil } @@ -427,7 +427,7 @@ func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, isValueHashed b } parentLeaf.StorageValue = value - parentLeaf.HashedValue = isValueHashed + parentLeaf.IsHashedValue = isValueHashed mutated = true return parentLeaf, mutated, nodesCreated, nil } @@ -447,7 +447,7 @@ func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, isValueHashed b if len(key) == commonPrefixLength { // key is included in parent leaf key newBranchParent.StorageValue = value - newBranchParent.HashedValue = isValueHashed + newBranchParent.IsHashedValue = isValueHashed if len(key) < len(parentLeafKey) { // Move the current leaf parent as a child to the new branch. @@ -472,7 +472,7 @@ func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, isValueHashed b if len(parentLeaf.PartialKey) == commonPrefixLength { // the key of the parent leaf is at this new branch newBranchParent.StorageValue = parentLeaf.StorageValue - newBranchParent.HashedValue = parentLeaf.HashedValue + newBranchParent.IsHashedValue = parentLeaf.IsHashedValue } else { // make the leaf a child of the new branch copySettings := node.DefaultCopySettings @@ -491,11 +491,11 @@ func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, isValueHashed b } childIndex := key[commonPrefixLength] newBranchParent.Children[childIndex] = &Node{ - PartialKey: key[commonPrefixLength+1:], - StorageValue: value, - HashedValue: isValueHashed, - Generation: t.generation, - Dirty: true, + PartialKey: key[commonPrefixLength+1:], + StorageValue: value, + IsHashedValue: isValueHashed, + Generation: t.generation, + Dirty: true, } newBranchParent.Descendants++ nodesCreated++ @@ -518,7 +518,7 @@ func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, isValueHash return nil, false, 0, fmt.Errorf("preparing branch for mutation: %w", err) } parentBranch.StorageValue = value - parentBranch.HashedValue = isValueHashed + parentBranch.IsHashedValue = isValueHashed mutated = true return parentBranch, mutated, 0, nil } @@ -532,11 +532,11 @@ func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, isValueHash if child == nil { child = &Node{ - PartialKey: remainingKey, - StorageValue: value, - HashedValue: isValueHashed, - Generation: t.generation, - Dirty: true, + PartialKey: remainingKey, + StorageValue: value, + IsHashedValue: isValueHashed, + Generation: t.generation, + Dirty: true, } nodesCreated = 1 parentBranch, err = t.prepForMutation(parentBranch, copySettings, pendingDeltas) @@ -594,7 +594,7 @@ func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, isValueHash if len(key) <= commonPrefixLength { newParentBranch.StorageValue = value - newParentBranch.HashedValue = isValueHashed + newParentBranch.IsHashedValue = isValueHashed } else { childIndex := key[commonPrefixLength] remainingKey := key[commonPrefixLength+1:] @@ -767,7 +767,7 @@ func retrieve(db db.DBGetter, parent *Node, key []byte) (value []byte) { func retrieveFromLeaf(db db.DBGetter, leaf *Node, key []byte) (value []byte) { if bytes.Equal(leaf.PartialKey, key) { - if leaf.HashedValue { + if leaf.IsHashedValue { // We get the node value, err := db.Get(leaf.StorageValue) if err != nil { diff --git a/lib/trie/trie_test.go b/lib/trie/trie_test.go index d62a4138c2..26b3066e57 100644 --- a/lib/trie/trie_test.go +++ b/lib/trie/trie_test.go @@ -1153,11 +1153,11 @@ func Test_Trie_Put(t *testing.T) { Dirty: true, }, { - PartialKey: []byte{6}, - StorageValue: longValueHash, - HashedValue: true, - Generation: 1, - Dirty: true, + PartialKey: []byte{6}, + StorageValue: longValueHash, + IsHashedValue: true, + Generation: 1, + Dirty: true, }, }), }, @@ -2270,9 +2270,9 @@ func Test_retrieve(t *testing.T) { Children: padRightChildren([]*Node{ nil, nil, nil, nil, { // full key 1, 2, 3, 4, 5 - PartialKey: []byte{5}, - StorageValue: hashedValue, - HashedValue: true, + PartialKey: []byte{5}, + StorageValue: hashedValue, + IsHashedValue: true, }, }), }, diff --git a/lib/trie/version.go b/lib/trie/version.go index ffc782b507..7ccd6d1989 100644 --- a/lib/trie/version.go +++ b/lib/trie/version.go @@ -7,11 +7,9 @@ import ( "errors" "fmt" "strings" - - "github.com/ChainSafe/gossamer/lib/common" ) -const V1MaxValueSize = common.HashLength +const V1MaxValueSize = 32 // Version is the state trie version which dictates how a // Merkle root should be constructed. It is defined in From a5a10c4de4d68bca030b5bef9a529d4714452fa2 Mon Sep 17 00:00:00 2001 From: Diego Date: Fri, 8 Sep 2023 14:31:50 +0200 Subject: [PATCH 066/128] Add comment --- lib/runtime/genesis.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/runtime/genesis.go b/lib/runtime/genesis.go index c1c5cb4d94..fbbe289d0b 100644 --- a/lib/runtime/genesis.go +++ b/lib/runtime/genesis.go @@ -26,6 +26,8 @@ func NewTrieFromGenesis(gen genesis.Genesis) (tr trie.Trie, err error) { ErrGenesisTopNotFound, gen.Name) } + // TODO: I'll set it to V0 since our goal is to work on westend first but we have to revisit it in the future + // to get the version from the runtime tr, err = trie.LoadFromMap(keyValues, trie.V0) if err != nil { return tr, fmt.Errorf("loading genesis top key values into trie: %w", err) From bb1a1c4cb852b33d2cb987d0e00ac1c8f2268056 Mon Sep 17 00:00:00 2001 From: Diego Date: Fri, 8 Sep 2023 14:34:01 +0200 Subject: [PATCH 067/128] Fixes --- lib/trie/trie.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/trie/trie.go b/lib/trie/trie.go index 355c628521..19041b4562 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -364,7 +364,7 @@ func (t *Trie) insertKeyLE(keyLE, value []byte, value = hashedValue.ToBytes() } - root, _, _, err := t.insert(t.root, nibblesKey, value, isValueHashed, pendingDeltas) + root, _, _, err := t.insert(t.root, nibblesKey, value, shouldHash, pendingDeltas) if err != nil { return err } From 0043e3182fcc1406e47b72800158d2c6d75e8aef Mon Sep 17 00:00:00 2001 From: Diego Date: Fri, 8 Sep 2023 14:46:39 +0200 Subject: [PATCH 068/128] Rollback runtime constants changes --- lib/runtime/constants.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/runtime/constants.go b/lib/runtime/constants.go index c6c5346ccf..26d0ccd267 100644 --- a/lib/runtime/constants.go +++ b/lib/runtime/constants.go @@ -6,10 +6,9 @@ package runtime const ( // v0.9 test API wasm // This wasm is generated using https://github.com/ChainSafe/polkadot-spec. - HOST_API_TEST_RUNTIME = "hostapi_runtime" - HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm" - // TODO: change this when https://github.com/ChainSafe/polkadot-spec/pull/17 is merged - HOST_API_TEST_RUNTIME_URL = "https://github.com/dimartiro/polkadot-spec/raw/add-triev1-functions/test/" + + HOST_API_TEST_RUNTIME = "hostapi_runtime" + HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm" + HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/raw/master/test/" + "runtimes/hostapi/hostapi_runtime.compact.wasm" // v0.9.29 polkadot From 28ee3c9302e1e13facb5d449b53551fad9a76f01 Mon Sep 17 00:00:00 2001 From: Diego Date: Fri, 8 Sep 2023 18:50:53 +0200 Subject: [PATCH 069/128] Remove unused version parse switch case --- lib/trie/version.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/trie/version.go b/lib/trie/version.go index c4049c749a..f5481b9082 100644 --- a/lib/trie/version.go +++ b/lib/trie/version.go @@ -62,8 +62,6 @@ func ParseVersion[T string | uint32](v T) (version Version, err error) { s = value case uint32: s = fmt.Sprintf("V%d", value) - default: - panic(fmt.Sprintf("unsupported type %T", s)) } switch { From a8f1d237bd7aadc183c817e570f99c7b965c71b9 Mon Sep 17 00:00:00 2001 From: Diego Date: Fri, 8 Sep 2023 18:53:23 +0200 Subject: [PATCH 070/128] Change panic to test fail --- lib/trie/version_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/trie/version_test.go b/lib/trie/version_test.go index 9e85010f79..19892bdb1d 100644 --- a/lib/trie/version_test.go +++ b/lib/trie/version_test.go @@ -4,7 +4,6 @@ package trie import ( - "fmt" "testing" "github.com/stretchr/testify/assert" @@ -105,7 +104,7 @@ func Test_ParseVersion(t *testing.T) { case uint32: version, err = ParseVersion(typed) default: - panic(fmt.Sprintf("unsupported type %T", testCase.v)) + t.Fail() } assert.Equal(t, testCase.version, version) From 39c39239b44e2db26d0b963d636f0d0d84b9a340 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 12 Sep 2023 21:33:29 +0200 Subject: [PATCH 071/128] Use sharing db in trie --- dot/state/storage.go | 4 +++- dot/state/storage_test.go | 3 ++- dot/state/tries.go | 7 +++++++ lib/trie/db/db.go | 6 ------ lib/trie/trie.go | 31 +++++++++++++++++++++---------- 5 files changed, 33 insertions(+), 18 deletions(-) diff --git a/dot/state/storage.go b/dot/state/storage.go index 8b068ee877..05e19eb328 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -48,6 +48,8 @@ func NewStorageState(db database.Database, blockState *BlockState, tries *Tries) (*StorageState, error) { storageTable := database.NewTable(db, storagePrefix) + tries.SetDB(storageTable) + return &StorageState{ blockState: blockState, tries: tries, @@ -119,7 +121,7 @@ func (s *StorageState) TrieState(root *common.Hash) (*rtstorage.TrieState, error // LoadFromDB loads an encoded trie from the DB where the key is `root` func (s *StorageState) LoadFromDB(root common.Hash) (*trie.Trie, error) { - t := trie.NewEmptyTrie() + t := trie.NewTrie(nil, s.db) err := t.Load(s.db, root) if err != nil { return nil, err diff --git a/dot/state/storage_test.go b/dot/state/storage_test.go index 97471147b4..6452de3c82 100644 --- a/dot/state/storage_test.go +++ b/dot/state/storage_test.go @@ -46,7 +46,8 @@ func TestStorage_StoreAndLoadTrie(t *testing.T) { require.NoError(t, err) ts2 := runtime.NewTrieState(trie) newSnapshot := ts2.Snapshot() - require.Equal(t, ts.Trie(), newSnapshot) + + require.True(t, ts.Trie().Equal(newSnapshot)) } func TestStorage_GetStorageByBlockHash(t *testing.T) { diff --git a/dot/state/tries.go b/dot/state/tries.go index 34a8484a7d..967ce5b350 100644 --- a/dot/state/tries.go +++ b/dot/state/tries.go @@ -8,6 +8,7 @@ import ( "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/lib/trie/db" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) @@ -51,6 +52,12 @@ func NewTries() (tries *Tries) { } } +func (t *Tries) SetDB(db db.Database) { + for _, trie := range t.rootToTrie { + trie.SetDB(db) + } +} + // SetEmptyTrie sets the empty trie in the tries. // Note the empty trie is the same for the v0 and the v1 // state trie versions. diff --git a/lib/trie/db/db.go b/lib/trie/db/db.go index 779380cca0..6ecc5d9273 100644 --- a/lib/trie/db/db.go +++ b/lib/trie/db/db.go @@ -12,7 +12,6 @@ import ( type Database interface { DBGetter DBPutter - Copier } // DBGetter gets a value corresponding to the given key. @@ -25,11 +24,6 @@ type DBPutter interface { Put(key []byte, value []byte) error } -// DBCopy creates a deep copy from a source DB -type Copier interface { - Copy() Database -} - type MemoryDB struct { data map[common.Hash][]byte l sync.RWMutex diff --git a/lib/trie/trie.go b/lib/trie/trie.go index 19041b4562..98fa815de6 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -6,6 +6,7 @@ package trie import ( "bytes" "fmt" + "reflect" "github.com/ChainSafe/gossamer/internal/trie/codec" "github.com/ChainSafe/gossamer/internal/trie/node" @@ -33,6 +34,7 @@ type Trie struct { // NewEmptyTrie creates a trie with a nil root func NewEmptyTrie() *Trie { + //db, _ := database.LoadDatabase("", true) return NewTrie(nil, db.NewEmptyMemoryDB()) } @@ -47,6 +49,23 @@ func NewTrie(root *Node, db db.Database) *Trie { } } +func (t *Trie) Equal(other *Trie) bool { + if t == nil && other == nil { + return true + } + + if t == nil || other == nil { + return false + } + + return t.generation == other.generation && reflect.DeepEqual(t.root, other.root) && + reflect.DeepEqual(t.childTries, other.childTries) && reflect.DeepEqual(t.deltas, other.deltas) +} + +func (t *Trie) SetDB(db db.Database) { + t.db = db +} + // Snapshot creates a copy of the trie. // Note it does not deep copy the trie, but will // copy on write as modifications are done on this new trie. @@ -64,15 +83,10 @@ func (t *Trie) Snapshot() (newTrie *Trie) { } } - var dbCopy db.Database - if t.db != nil { - dbCopy = t.db.Copy() - } - return &Trie{ generation: t.generation + 1, root: t.root, - db: dbCopy, + db: t.db, childTries: childTries, deltas: tracking.New(), } @@ -146,6 +160,7 @@ func (t *Trie) DeepCopy() (trieCopy *Trie) { trieCopy = &Trie{ generation: t.generation, + db: t.db, } if t.deltas != nil { @@ -166,10 +181,6 @@ func (t *Trie) DeepCopy() (trieCopy *Trie) { trieCopy.root = t.root.Copy(copySettings) } - if t.db != nil { - trieCopy.db = t.db.Copy() - } - return trieCopy } From c481b6a89bf45ac9217074988951cac34ee51a6d Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 12 Sep 2023 22:01:05 +0200 Subject: [PATCH 072/128] Fix mocks --- .../{db_mocks_test.go => mocks_database_test.go} | 15 --------------- dot/state/mocks_generate_test.go | 2 +- 2 files changed, 1 insertion(+), 16 deletions(-) rename dot/state/{db_mocks_test.go => mocks_database_test.go} (79%) diff --git a/dot/state/db_mocks_test.go b/dot/state/mocks_database_test.go similarity index 79% rename from dot/state/db_mocks_test.go rename to dot/state/mocks_database_test.go index 0c7fdd4069..079cb5578f 100644 --- a/dot/state/db_mocks_test.go +++ b/dot/state/mocks_database_test.go @@ -7,7 +7,6 @@ package state import ( reflect "reflect" - db "github.com/ChainSafe/gossamer/lib/trie/db" gomock "github.com/golang/mock/gomock" ) @@ -34,20 +33,6 @@ func (m *MockDatabase) EXPECT() *MockDatabaseMockRecorder { return m.recorder } -// Copy mocks base method. -func (m *MockDatabase) Copy() db.Database { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Copy") - ret0, _ := ret[0].(db.Database) - return ret0 -} - -// Copy indicates an expected call of Copy. -func (mr *MockDatabaseMockRecorder) Copy() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Copy", reflect.TypeOf((*MockDatabase)(nil).Copy)) -} - // Get mocks base method. func (m *MockDatabase) Get(arg0 []byte) ([]byte, error) { m.ctrl.T.Helper() diff --git a/dot/state/mocks_generate_test.go b/dot/state/mocks_generate_test.go index c4de67b207..10d9c46059 100644 --- a/dot/state/mocks_generate_test.go +++ b/dot/state/mocks_generate_test.go @@ -7,4 +7,4 @@ package state //go:generate mockgen -destination=mocks_runtime_test.go -package $GOPACKAGE github.com/ChainSafe/gossamer/lib/runtime Instance //go:generate mockgen -destination=mock_gauge_test.go -package $GOPACKAGE github.com/prometheus/client_golang/prometheus Gauge //go:generate mockgen -destination=mock_counter_test.go -package $GOPACKAGE github.com/prometheus/client_golang/prometheus Counter -//go:generate mockgen -destination=db_mocks_test.go -package=$GOPACKAGE github.com/ChainSafe/gossamer/lib/trie/db Database +//go:generate mockgen -destination=mocks_database_test.go -package=$GOPACKAGE github.com/ChainSafe/gossamer/lib/trie/db Database From 8ac581c2fb2d4ecb9021f430206d65b0d0fbc708 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 12 Sep 2023 22:02:53 +0200 Subject: [PATCH 073/128] Remove old in memory db implementation and replace it to pebble --- dot/state/tries.go | 14 +++++++ dot/state/tries_test.go | 8 +++- lib/trie/database_test.go | 2 +- lib/trie/db/db.go | 71 ++++------------------------------- lib/trie/proof/proof_test.go | 4 +- lib/trie/proof/verify.go | 2 +- lib/trie/proof/verify_test.go | 8 ++-- lib/trie/trie.go | 2 +- lib/trie/trie_test.go | 31 +++++++-------- 9 files changed, 53 insertions(+), 89 deletions(-) diff --git a/dot/state/tries.go b/dot/state/tries.go index 967ce5b350..ac0335db83 100644 --- a/dot/state/tries.go +++ b/dot/state/tries.go @@ -58,6 +58,20 @@ func (t *Tries) SetDB(db db.Database) { } } +func (t *Tries) Equal(other *Tries) bool { + if len(t.rootToTrie) != len(other.rootToTrie) { + return false + } + + for hash, trie := range t.rootToTrie { + if !trie.Equal(other.rootToTrie[hash]) { + return false + } + } + + return true +} + // SetEmptyTrie sets the empty trie in the tries. // Note the empty trie is the same for the v0 and the v1 // state trie versions. diff --git a/dot/state/tries_test.go b/dot/state/tries_test.go index 4bdc84e1d2..d04b2de866 100644 --- a/dot/state/tries_test.go +++ b/dot/state/tries_test.go @@ -43,7 +43,7 @@ func Test_Tries_SetEmptyTrie(t *testing.T) { deleteCounter: deleteCounter, } - assert.Equal(t, expectedTries, tries) + assert.True(t, tries.Equal(expectedTries)) } func Test_Tries_SetTrie(t *testing.T) { @@ -124,7 +124,11 @@ func Test_Tries_softSet(t *testing.T) { tries.softSet(testCase.root, testCase.trie) - assert.Equal(t, testCase.expectedRootToTrie, tries.rootToTrie) + assert.Equal(t, len(testCase.expectedRootToTrie), len(tries.rootToTrie)) + + for root, trie := range tries.rootToTrie { + assert.True(t, trie.Equal(testCase.expectedRootToTrie[root])) + } }) } } diff --git a/lib/trie/database_test.go b/lib/trie/database_test.go index b05310729e..8f9d9ddb56 100644 --- a/lib/trie/database_test.go +++ b/lib/trie/database_test.go @@ -309,7 +309,7 @@ func Test_Trie_PutChild_Store_Load(t *testing.T) { err = trieFromDB.Load(db, trie.MustHash()) require.NoError(t, err) - assert.Equal(t, trie.childTries, trieFromDB.childTries) + assert.Equal(t, len(trie.childTries), len(trieFromDB.childTries)) assert.Equal(t, trie.String(), trieFromDB.String()) } } diff --git a/lib/trie/db/db.go b/lib/trie/db/db.go index 6ecc5d9273..ab99478c2c 100644 --- a/lib/trie/db/db.go +++ b/lib/trie/db/db.go @@ -3,9 +3,7 @@ package db import ( - "fmt" - "sync" - + "github.com/ChainSafe/gossamer/internal/database" "github.com/ChainSafe/gossamer/lib/common" ) @@ -24,75 +22,22 @@ type DBPutter interface { Put(key []byte, value []byte) error } -type MemoryDB struct { - data map[common.Hash][]byte - l sync.RWMutex -} - -func NewEmptyMemoryDB() *MemoryDB { - return &MemoryDB{ - data: make(map[common.Hash][]byte), - } +func NewEmptyInMemoryDB() Database { + db, _ := database.LoadDatabase("", true) + return db } -func NewMemoryDBFromProof(encodedNodes [][]byte) (*MemoryDB, error) { - data := make(map[common.Hash][]byte, len(encodedNodes)) - +func NewInMemoryDBFromProof(encodedNodes [][]byte) (Database, error) { + db := NewEmptyInMemoryDB() for _, encodedProofNode := range encodedNodes { nodeHash, err := common.Blake2bHash(encodedProofNode) if err != nil { return nil, err } - data[nodeHash] = encodedProofNode - } - - return &MemoryDB{ - data: data, - }, nil - -} - -func (mdb *MemoryDB) Copy() Database { - newDB := NewEmptyMemoryDB() - copyData := make(map[common.Hash][]byte, len(mdb.data)) - - for k, v := range mdb.data { - copyData[k] = v + db.Put(nodeHash.ToBytes(), encodedProofNode) } - newDB.data = copyData - return newDB -} - -func (mdb *MemoryDB) Get(key []byte) ([]byte, error) { - if len(key) != common.HashLength { - return nil, fmt.Errorf("expected %d bytes length key, given %d (%x)", common.HashLength, len(key), key) - } - var hash common.Hash - copy(hash[:], key) - - mdb.l.RLock() - defer mdb.l.RUnlock() - - if value, found := mdb.data[hash]; found { - return value, nil - } - - return nil, nil -} - -func (mdb *MemoryDB) Put(key []byte, value []byte) error { - if len(key) != common.HashLength { - return fmt.Errorf("expected %d bytes length key, given %d (%x)", common.HashLength, len(key), key) - } - - var hash common.Hash - copy(hash[:], key) - - mdb.l.Lock() - defer mdb.l.Unlock() + return db, nil - mdb.data[hash] = value - return nil } diff --git a/lib/trie/proof/proof_test.go b/lib/trie/proof/proof_test.go index 2e0cf2731a..79267e0cf1 100644 --- a/lib/trie/proof/proof_test.go +++ b/lib/trie/proof/proof_test.go @@ -84,7 +84,7 @@ func TestParachainHeaderStateProof(t *testing.T) { expectedValue := proof7 - proofDB, err := db.NewMemoryDBFromProof(proof) + proofDB, err := db.NewInMemoryDBFromProof(proof) require.NoError(t, err) trie, err := buildTrie(proof, stateRoot, proofDB) @@ -121,7 +121,7 @@ func TestTrieProof(t *testing.T) { } proof := [][]byte{proof1, proof2, proof3} - proofDB, err := db.NewMemoryDBFromProof(proof) + proofDB, err := db.NewInMemoryDBFromProof(proof) require.NoError(t, err) diff --git a/lib/trie/proof/verify.go b/lib/trie/proof/verify.go index 97e439fdec..6fedaf7d87 100644 --- a/lib/trie/proof/verify.go +++ b/lib/trie/proof/verify.go @@ -30,7 +30,7 @@ var logger = log.NewFromGlobal(log.AddContext("pkg", "proof")) // Note this is exported because it is imported and used by: // https://github.com/ComposableFi/ibc-go/blob/6d62edaa1a3cb0768c430dab81bb195e0b0c72db/modules/light-clients/11-beefy/types/client_state.go#L78 func Verify(encodedProofNodes [][]byte, rootHash, key, value []byte) (err error) { - proofDB, err := db.NewMemoryDBFromProof(encodedProofNodes) + proofDB, err := db.NewInMemoryDBFromProof(encodedProofNodes) if err != nil { return err diff --git a/lib/trie/proof/verify_test.go b/lib/trie/proof/verify_test.go index 78e0e5dd50..5910237d9b 100644 --- a/lib/trie/proof/verify_test.go +++ b/lib/trie/proof/verify_test.go @@ -167,7 +167,7 @@ func Test_buildTrie(t *testing.T) { encodeNode(t, leafAShort), } - proofDB, err := db.NewMemoryDBFromProof(encodedProofNodes) + proofDB, err := db.NewInMemoryDBFromProof(encodedProofNodes) assert.NoError(t, err) return testCase{ @@ -186,7 +186,7 @@ func Test_buildTrie(t *testing.T) { encodeNode(t, leafBLarge), } - proofDB, err := db.NewMemoryDBFromProof(encodedProofNodes) + proofDB, err := db.NewInMemoryDBFromProof(encodedProofNodes) assert.NoError(t, err) return testCase{ @@ -206,7 +206,7 @@ func Test_buildTrie(t *testing.T) { encodeNode(t, leafBLarge), } - proofDB, err := db.NewMemoryDBFromProof(encodedProofNodes) + proofDB, err := db.NewInMemoryDBFromProof(encodedProofNodes) assert.NoError(t, err) return testCase{ @@ -235,7 +235,7 @@ func Test_buildTrie(t *testing.T) { encodeNode(t, leafCLarge), // children 2 } - proofDB, err := db.NewMemoryDBFromProof(encodedProofNodes) + proofDB, err := db.NewInMemoryDBFromProof(encodedProofNodes) assert.NoError(t, err) return testCase{ diff --git a/lib/trie/trie.go b/lib/trie/trie.go index 98fa815de6..0130d1a9d8 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -35,7 +35,7 @@ type Trie struct { // NewEmptyTrie creates a trie with a nil root func NewEmptyTrie() *Trie { //db, _ := database.LoadDatabase("", true) - return NewTrie(nil, db.NewEmptyMemoryDB()) + return NewTrie(nil, db.NewEmptyInMemoryDB()) } // NewTrie creates a trie with an existing root node diff --git a/lib/trie/trie_test.go b/lib/trie/trie_test.go index 26b3066e57..1844a88fcb 100644 --- a/lib/trie/trie_test.go +++ b/lib/trie/trie_test.go @@ -34,10 +34,11 @@ func Test_NewEmptyTrie(t *testing.T) { expectedTrie := &Trie{ childTries: make(map[common.Hash]*Trie), deltas: tracking.New(), - db: db.NewEmptyMemoryDB(), + db: db.NewEmptyInMemoryDB(), } trie := NewEmptyTrie() - assert.Equal(t, expectedTrie, trie) + + assert.True(t, trie.Equal(expectedTrie)) } func Test_NewTrie(t *testing.T) { @@ -698,7 +699,7 @@ func Test_Trie_Entries(t *testing.T) { trie := Trie{ root: nil, childTries: make(map[common.Hash]*Trie), - db: db.NewEmptyMemoryDB(), + db: db.NewEmptyInMemoryDB(), } kv := map[string][]byte{ @@ -723,7 +724,7 @@ func Test_Trie_Entries(t *testing.T) { trie := Trie{ root: nil, childTries: make(map[common.Hash]*Trie), - db: db.NewEmptyMemoryDB(), + db: db.NewEmptyInMemoryDB(), } kv := map[string][]byte{ @@ -1086,7 +1087,7 @@ func Test_Trie_Put(t *testing.T) { stateVersion Version key []byte value []byte - expectedTrie Trie + expectedTrie *Trie }{ "trie_v0_with_key_and_value": { trie: Trie{ @@ -1099,7 +1100,7 @@ func Test_Trie_Put(t *testing.T) { }, key: []byte{0x12, 0x16}, value: []byte{2}, - expectedTrie: Trie{ + expectedTrie: &Trie{ generation: 1, deltas: newDeltas("0xa195089c3e8f8b5b36978700ad954aed99e08413cfc1e2b4c00a5d064abe66a9"), root: &Node{ @@ -1132,12 +1133,12 @@ func Test_Trie_Put(t *testing.T) { PartialKey: []byte{1, 2, 0, 5}, StorageValue: []byte{1}, }, - db: db.NewEmptyMemoryDB(), + db: db.NewEmptyInMemoryDB(), }, stateVersion: V1, key: []byte{0x12, 0x16}, value: longValue, - expectedTrie: Trie{ + expectedTrie: &Trie{ generation: 1, deltas: newDeltas("0xa195089c3e8f8b5b36978700ad954aed99e08413cfc1e2b4c00a5d064abe66a9"), root: &Node{ @@ -1162,7 +1163,7 @@ func Test_Trie_Put(t *testing.T) { }), }, db: func() db.Database { - db := db.NewEmptyMemoryDB() + db := db.NewEmptyInMemoryDB() db.Put(longValueHash, longValue) return db }(), @@ -1178,7 +1179,7 @@ func Test_Trie_Put(t *testing.T) { trie := testCase.trie trie.Put(testCase.key, testCase.value, testCase.stateVersion) - assert.Equal(t, testCase.expectedTrie, trie) + assert.True(t, trie.Equal(testCase.expectedTrie)) }) } } @@ -1734,7 +1735,7 @@ func Test_LoadFromMap(t *testing.T) { expectedTrie: Trie{ childTries: map[common.Hash]*Trie{}, deltas: newDeltas(), - db: db.NewEmptyMemoryDB(), + db: db.NewEmptyInMemoryDB(), }, }, "empty_data": { @@ -1742,7 +1743,7 @@ func Test_LoadFromMap(t *testing.T) { expectedTrie: Trie{ childTries: map[common.Hash]*Trie{}, deltas: newDeltas(), - db: db.NewEmptyMemoryDB(), + db: db.NewEmptyInMemoryDB(), }, }, "bad_key": { @@ -1776,7 +1777,7 @@ func Test_LoadFromMap(t *testing.T) { }, childTries: map[common.Hash]*Trie{}, deltas: newDeltas(), - db: db.NewEmptyMemoryDB(), + db: db.NewEmptyInMemoryDB(), }, }, "load_key_values": { @@ -1807,7 +1808,7 @@ func Test_LoadFromMap(t *testing.T) { }, childTries: map[common.Hash]*Trie{}, deltas: newDeltas(), - db: db.NewEmptyMemoryDB(), + db: db.NewEmptyInMemoryDB(), }, }, } @@ -1824,7 +1825,7 @@ func Test_LoadFromMap(t *testing.T) { assert.EqualError(t, err, testCase.errMessage) } - assert.Equal(t, testCase.expectedTrie, trie) + assert.True(t, trie.Equal(&testCase.expectedTrie)) }) } } From 50f52de55e46a5b51460dd888149d21bb993b718 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 12 Sep 2023 22:09:26 +0200 Subject: [PATCH 074/128] Fix lint --- lib/trie/db/db.go | 7 ++++++- lib/trie/trie.go | 1 - 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/trie/db/db.go b/lib/trie/db/db.go index ab99478c2c..542f854c2e 100644 --- a/lib/trie/db/db.go +++ b/lib/trie/db/db.go @@ -3,6 +3,8 @@ package db import ( + "fmt" + "github.com/ChainSafe/gossamer/internal/database" "github.com/ChainSafe/gossamer/lib/common" ) @@ -35,7 +37,10 @@ func NewInMemoryDBFromProof(encodedNodes [][]byte) (Database, error) { return nil, err } - db.Put(nodeHash.ToBytes(), encodedProofNode) + err = db.Put(nodeHash.ToBytes(), encodedProofNode) + if err != nil { + return nil, fmt.Errorf("adding value for hash 0x%x in db: %w", nodeHash, err) + } } return db, nil diff --git a/lib/trie/trie.go b/lib/trie/trie.go index 0130d1a9d8..10e23dc41e 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -34,7 +34,6 @@ type Trie struct { // NewEmptyTrie creates a trie with a nil root func NewEmptyTrie() *Trie { - //db, _ := database.LoadDatabase("", true) return NewTrie(nil, db.NewEmptyInMemoryDB()) } From 191ff3e32d56c32b8a85684ce93a84eae91d938a Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 12 Sep 2023 22:16:44 +0200 Subject: [PATCH 075/128] Fix fuzz test --- lib/trie/trie_endtoend_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/trie/trie_endtoend_test.go b/lib/trie/trie_endtoend_test.go index f994e4c4ab..4a4707134f 100644 --- a/lib/trie/trie_endtoend_test.go +++ b/lib/trie/trie_endtoend_test.go @@ -103,7 +103,7 @@ func TestPutAndGetOddKeyLengths(t *testing.T) { func Fuzz_Trie_PutAndGet_Single(f *testing.F) { f.Fuzz(func(t *testing.T, key, value []byte) { - trie := NewEmptyTrie() + trie := NewTrie(nil, nil) trie.Put(key, value, V0) retrievedValue := trie.Get(key) assert.Equal(t, value, retrievedValue) From 22b42167a4ae110f85a25868f3984cec291de8a3 Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 13 Sep 2023 14:50:59 +0200 Subject: [PATCH 076/128] Remove value nodes from db when hashed node is deleted --- lib/trie/trie.go | 4 ++++ lib/trie/trie_test.go | 31 +++++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/lib/trie/trie.go b/lib/trie/trie.go index 10e23dc41e..5b4d40b7cf 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -142,6 +142,10 @@ func (t *Trie) registerDeletedNodeHash(node *Node, // since the last trie snapshot. nodeHash := common.NewHash(node.MerkleValue) pendingDeltas.RecordDeleted(nodeHash) + + if node.IsHashedValue { + pendingDeltas.RecordDeleted(common.NewHash(node.StorageValue)) + } } return nil diff --git a/lib/trie/trie_test.go b/lib/trie/trie_test.go index 1844a88fcb..de1ed7a71e 100644 --- a/lib/trie/trie_test.go +++ b/lib/trie/trie_test.go @@ -277,11 +277,17 @@ func Test_Trie_registerDeletedNodeHash(t *testing.T) { StorageValue: []byte{2}, } + someSmallNodeWithHashedValue := &Node{ + PartialKey: []byte{1}, + StorageValue: common.MustBlake2bHash([]byte("hash")).ToBytes(), + IsHashedValue: true, + } + testCases := map[string]struct { trie Trie node *Node - pendingDeltas DeltaRecorder - expectedPendingDeltas DeltaRecorder + pendingDeltas *tracking.Deltas + expectedPendingDeltas *tracking.Deltas expectedTrie Trie }{ "dirty_node_not_registered": { @@ -322,6 +328,27 @@ func Test_Trie_registerDeletedNodeHash(t *testing.T) { pendingDeltas: newDeltas(), expectedPendingDeltas: newDeltas("0x98fcd66ba312c29ef193052fd0c14c6e38b158bd5c0235064594cacc1ab5965d"), }, + "clean_v1_node_with_hashed_subvalue": { + node: someSmallNodeWithHashedValue, + trie: Trie{root: someSmallNodeWithHashedValue}, + pendingDeltas: newDeltas(), + expectedPendingDeltas: newDeltas( + "0x4269e2a9cdf14dbb1f94ea10e5e65be796e940f7043bcb71276682712e6730d5", + "0x97edaa69596438136dcd128553e904bc03f526426f727d270b69841fb6cf50d3", + ), + expectedTrie: Trie{ + root: &Node{ + PartialKey: []byte{1}, + StorageValue: common.MustBlake2bHash([]byte("hash")).ToBytes(), + IsHashedValue: true, + MerkleValue: []byte{ + 0x42, 0x69, 0xe2, 0xa9, 0xcd, 0xf1, 0x4d, 0xbb, + 0x1f, 0x94, 0xea, 0x10, 0xe5, 0xe6, 0x5b, 0xe7, + 0x96, 0xe9, 0x40, 0xf7, 0x04, 0x3b, 0xcb, 0x71, + 0x27, 0x66, 0x82, 0x71, 0x2e, 0x67, 0x30, 0xd5}, + }, + }, + }, } for name, testCase := range testCases { From 8d7c6b6e5456ad817c6b74493dba90caaff134bc Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 13 Sep 2023 15:57:09 +0200 Subject: [PATCH 077/128] Add comments --- dot/state/tries.go | 2 ++ lib/trie/trie.go | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/dot/state/tries.go b/dot/state/tries.go index ac0335db83..745fec3abc 100644 --- a/dot/state/tries.go +++ b/dot/state/tries.go @@ -52,12 +52,14 @@ func NewTries() (tries *Tries) { } } +// SetDB is to set the same db for every trie in the collection func (t *Tries) SetDB(db db.Database) { for _, trie := range t.rootToTrie { trie.SetDB(db) } } +// Equal is to compare this Tries with other using the internal Equal for every trie it contains func (t *Tries) Equal(other *Tries) bool { if len(t.rootToTrie) != len(other.rootToTrie) { return false diff --git a/lib/trie/trie.go b/lib/trie/trie.go index 5b4d40b7cf..bdc8bdabdf 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -48,6 +48,7 @@ func NewTrie(root *Node, db db.Database) *Trie { } } +// Equal is to compare one trie with other, this method will ignore the shared db instance func (t *Trie) Equal(other *Trie) bool { if t == nil && other == nil { return true @@ -61,6 +62,7 @@ func (t *Trie) Equal(other *Trie) bool { reflect.DeepEqual(t.childTries, other.childTries) && reflect.DeepEqual(t.deltas, other.deltas) } +// SetDB is to set the db that this trie will use primary to store v1 value nodes func (t *Trie) SetDB(db db.Database) { t.db = db } @@ -143,6 +145,7 @@ func (t *Trie) registerDeletedNodeHash(node *Node, nodeHash := common.NewHash(node.MerkleValue) pendingDeltas.RecordDeleted(nodeHash) + // If this node contains a hashed value we have to remove the value node from the db too if node.IsHashedValue { pendingDeltas.RecordDeleted(common.NewHash(node.StorageValue)) } @@ -370,7 +373,7 @@ func (t *Trie) insertKeyLE(keyLE, value []byte, if shouldHash { hashedValue := common.MustBlake2bHash(value) - // Add original value in db using the hashed value as key + // Add the original value as value node in db using the hashed value as key err = t.db.Put(hashedValue.ToBytes(), value) if err != nil { return err From f32c93906bee75742c3ddb1035f8e7c4b9cf9675 Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 13 Sep 2023 16:15:49 +0200 Subject: [PATCH 078/128] Return empty array instead 0 --- lib/runtime/wazero/imports.go | 11 ++++++----- lib/runtime/wazero/imports_test.go | 3 --- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/lib/runtime/wazero/imports.go b/lib/runtime/wazero/imports.go index 2848a28cb0..9651217570 100644 --- a/lib/runtime/wazero/imports.go +++ b/lib/runtime/wazero/imports.go @@ -32,7 +32,8 @@ var ( log.AddContext("module", "wazero"), ) - noneEncoded []byte = []byte{0x00} + noneEncoded []byte = []byte{0x00} + emptyByteVectorEncoded []byte = scale.MustMarshal([]byte{}) ) const ( @@ -995,7 +996,7 @@ func ext_trie_blake2_256_verify_proof_version_1( err := scale.Unmarshal(toDecProofs, &encodedProofNodes) if err != nil { logger.Errorf("failed scale decoding proof data: %s", err) - return uint32(0) + return 0 } key := read(m, keySpan) @@ -1371,13 +1372,13 @@ func ext_default_child_storage_root_version_2(ctx context.Context, m api.Module, child, err := storage.GetChild(read(m, childStorageKey)) if err != nil { logger.Errorf("failed to retrieve child: %s", err) - return 0 + return mustWrite(m, rtCtx.Allocator, emptyByteVectorEncoded) } childRoot, err := child.Hash() if err != nil { logger.Errorf("failed to encode child root: %s", err) - return 0 + return mustWrite(m, rtCtx.Allocator, emptyByteVectorEncoded) } childRootSlice := childRoot[:] @@ -2367,7 +2368,7 @@ func ext_storage_root_version_2(ctx context.Context, m api.Module, version uint3 _, err := trie.ParseVersion(binary.LittleEndian.Uint32(stateVersionBytes)) if err != nil { logger.Errorf("failed parsing state version: %s", err) - return 0 + return mustWrite(m, rtCtx.Allocator, emptyByteVectorEncoded) } root, err := storage.Root() diff --git a/lib/runtime/wazero/imports_test.go b/lib/runtime/wazero/imports_test.go index a835fa0549..e86ae08994 100644 --- a/lib/runtime/wazero/imports_test.go +++ b/lib/runtime/wazero/imports_test.go @@ -1323,9 +1323,6 @@ func Test_ext_default_child_storage_root_version_2(t *testing.T) { require.NoError(t, err) stateVersion := uint32(trie.V1) - //stateVersionBytes := make([]byte, 4) - //binary.LittleEndian.PutUint32(stateVersionBytes, stateVersion) - encVersion, err := scale.Marshal(&stateVersion) require.NoError(t, err) From 8e076bb2b6aeae6020978f4cf91a02ae2149eade Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 13 Sep 2023 16:51:22 +0200 Subject: [PATCH 079/128] Fix return zero value --- lib/runtime/wazero/imports.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/runtime/wazero/imports.go b/lib/runtime/wazero/imports.go index 9651217570..000d9b9985 100644 --- a/lib/runtime/wazero/imports.go +++ b/lib/runtime/wazero/imports.go @@ -1035,7 +1035,7 @@ func ext_trie_blake2_256_verify_proof_version_2( err = scale.Unmarshal(toDecProofs, &encodedProofNodes) if err != nil { logger.Errorf("failed scale decoding proof data: %s", err) - return uint32(0) + return 0 } key := read(m, keySpan) From c1879439f30e763f1a9a2ee27427669e6c61da69 Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 13 Sep 2023 18:15:33 +0200 Subject: [PATCH 080/128] Revert using pebble as in memory db for tries --- dot/state/tries.go | 15 -------- dot/state/tries_test.go | 8 +--- lib/trie/database_test.go | 2 +- lib/trie/db/db.go | 72 +++++++++++++++++++++++++++++------ lib/trie/proof/proof_test.go | 4 +- lib/trie/proof/verify.go | 2 +- lib/trie/proof/verify_test.go | 8 ++-- lib/trie/trie.go | 2 +- lib/trie/trie_test.go | 31 ++++++++------- 9 files changed, 87 insertions(+), 57 deletions(-) diff --git a/dot/state/tries.go b/dot/state/tries.go index 745fec3abc..042dd6282e 100644 --- a/dot/state/tries.go +++ b/dot/state/tries.go @@ -59,21 +59,6 @@ func (t *Tries) SetDB(db db.Database) { } } -// Equal is to compare this Tries with other using the internal Equal for every trie it contains -func (t *Tries) Equal(other *Tries) bool { - if len(t.rootToTrie) != len(other.rootToTrie) { - return false - } - - for hash, trie := range t.rootToTrie { - if !trie.Equal(other.rootToTrie[hash]) { - return false - } - } - - return true -} - // SetEmptyTrie sets the empty trie in the tries. // Note the empty trie is the same for the v0 and the v1 // state trie versions. diff --git a/dot/state/tries_test.go b/dot/state/tries_test.go index d04b2de866..4bdc84e1d2 100644 --- a/dot/state/tries_test.go +++ b/dot/state/tries_test.go @@ -43,7 +43,7 @@ func Test_Tries_SetEmptyTrie(t *testing.T) { deleteCounter: deleteCounter, } - assert.True(t, tries.Equal(expectedTries)) + assert.Equal(t, expectedTries, tries) } func Test_Tries_SetTrie(t *testing.T) { @@ -124,11 +124,7 @@ func Test_Tries_softSet(t *testing.T) { tries.softSet(testCase.root, testCase.trie) - assert.Equal(t, len(testCase.expectedRootToTrie), len(tries.rootToTrie)) - - for root, trie := range tries.rootToTrie { - assert.True(t, trie.Equal(testCase.expectedRootToTrie[root])) - } + assert.Equal(t, testCase.expectedRootToTrie, tries.rootToTrie) }) } } diff --git a/lib/trie/database_test.go b/lib/trie/database_test.go index 8f9d9ddb56..b05310729e 100644 --- a/lib/trie/database_test.go +++ b/lib/trie/database_test.go @@ -309,7 +309,7 @@ func Test_Trie_PutChild_Store_Load(t *testing.T) { err = trieFromDB.Load(db, trie.MustHash()) require.NoError(t, err) - assert.Equal(t, len(trie.childTries), len(trieFromDB.childTries)) + assert.Equal(t, trie.childTries, trieFromDB.childTries) assert.Equal(t, trie.String(), trieFromDB.String()) } } diff --git a/lib/trie/db/db.go b/lib/trie/db/db.go index 542f854c2e..6ecc5d9273 100644 --- a/lib/trie/db/db.go +++ b/lib/trie/db/db.go @@ -4,8 +4,8 @@ package db import ( "fmt" + "sync" - "github.com/ChainSafe/gossamer/internal/database" "github.com/ChainSafe/gossamer/lib/common" ) @@ -24,25 +24,75 @@ type DBPutter interface { Put(key []byte, value []byte) error } -func NewEmptyInMemoryDB() Database { - db, _ := database.LoadDatabase("", true) - return db +type MemoryDB struct { + data map[common.Hash][]byte + l sync.RWMutex } -func NewInMemoryDBFromProof(encodedNodes [][]byte) (Database, error) { - db := NewEmptyInMemoryDB() +func NewEmptyMemoryDB() *MemoryDB { + return &MemoryDB{ + data: make(map[common.Hash][]byte), + } +} + +func NewMemoryDBFromProof(encodedNodes [][]byte) (*MemoryDB, error) { + data := make(map[common.Hash][]byte, len(encodedNodes)) + for _, encodedProofNode := range encodedNodes { nodeHash, err := common.Blake2bHash(encodedProofNode) if err != nil { return nil, err } - err = db.Put(nodeHash.ToBytes(), encodedProofNode) - if err != nil { - return nil, fmt.Errorf("adding value for hash 0x%x in db: %w", nodeHash, err) - } + data[nodeHash] = encodedProofNode + } + + return &MemoryDB{ + data: data, + }, nil + +} + +func (mdb *MemoryDB) Copy() Database { + newDB := NewEmptyMemoryDB() + copyData := make(map[common.Hash][]byte, len(mdb.data)) + + for k, v := range mdb.data { + copyData[k] = v } - return db, nil + newDB.data = copyData + return newDB +} + +func (mdb *MemoryDB) Get(key []byte) ([]byte, error) { + if len(key) != common.HashLength { + return nil, fmt.Errorf("expected %d bytes length key, given %d (%x)", common.HashLength, len(key), key) + } + var hash common.Hash + copy(hash[:], key) + + mdb.l.RLock() + defer mdb.l.RUnlock() + + if value, found := mdb.data[hash]; found { + return value, nil + } + + return nil, nil +} + +func (mdb *MemoryDB) Put(key []byte, value []byte) error { + if len(key) != common.HashLength { + return fmt.Errorf("expected %d bytes length key, given %d (%x)", common.HashLength, len(key), key) + } + + var hash common.Hash + copy(hash[:], key) + + mdb.l.Lock() + defer mdb.l.Unlock() + mdb.data[hash] = value + return nil } diff --git a/lib/trie/proof/proof_test.go b/lib/trie/proof/proof_test.go index 79267e0cf1..2e0cf2731a 100644 --- a/lib/trie/proof/proof_test.go +++ b/lib/trie/proof/proof_test.go @@ -84,7 +84,7 @@ func TestParachainHeaderStateProof(t *testing.T) { expectedValue := proof7 - proofDB, err := db.NewInMemoryDBFromProof(proof) + proofDB, err := db.NewMemoryDBFromProof(proof) require.NoError(t, err) trie, err := buildTrie(proof, stateRoot, proofDB) @@ -121,7 +121,7 @@ func TestTrieProof(t *testing.T) { } proof := [][]byte{proof1, proof2, proof3} - proofDB, err := db.NewInMemoryDBFromProof(proof) + proofDB, err := db.NewMemoryDBFromProof(proof) require.NoError(t, err) diff --git a/lib/trie/proof/verify.go b/lib/trie/proof/verify.go index 6fedaf7d87..97e439fdec 100644 --- a/lib/trie/proof/verify.go +++ b/lib/trie/proof/verify.go @@ -30,7 +30,7 @@ var logger = log.NewFromGlobal(log.AddContext("pkg", "proof")) // Note this is exported because it is imported and used by: // https://github.com/ComposableFi/ibc-go/blob/6d62edaa1a3cb0768c430dab81bb195e0b0c72db/modules/light-clients/11-beefy/types/client_state.go#L78 func Verify(encodedProofNodes [][]byte, rootHash, key, value []byte) (err error) { - proofDB, err := db.NewInMemoryDBFromProof(encodedProofNodes) + proofDB, err := db.NewMemoryDBFromProof(encodedProofNodes) if err != nil { return err diff --git a/lib/trie/proof/verify_test.go b/lib/trie/proof/verify_test.go index 5910237d9b..78e0e5dd50 100644 --- a/lib/trie/proof/verify_test.go +++ b/lib/trie/proof/verify_test.go @@ -167,7 +167,7 @@ func Test_buildTrie(t *testing.T) { encodeNode(t, leafAShort), } - proofDB, err := db.NewInMemoryDBFromProof(encodedProofNodes) + proofDB, err := db.NewMemoryDBFromProof(encodedProofNodes) assert.NoError(t, err) return testCase{ @@ -186,7 +186,7 @@ func Test_buildTrie(t *testing.T) { encodeNode(t, leafBLarge), } - proofDB, err := db.NewInMemoryDBFromProof(encodedProofNodes) + proofDB, err := db.NewMemoryDBFromProof(encodedProofNodes) assert.NoError(t, err) return testCase{ @@ -206,7 +206,7 @@ func Test_buildTrie(t *testing.T) { encodeNode(t, leafBLarge), } - proofDB, err := db.NewInMemoryDBFromProof(encodedProofNodes) + proofDB, err := db.NewMemoryDBFromProof(encodedProofNodes) assert.NoError(t, err) return testCase{ @@ -235,7 +235,7 @@ func Test_buildTrie(t *testing.T) { encodeNode(t, leafCLarge), // children 2 } - proofDB, err := db.NewInMemoryDBFromProof(encodedProofNodes) + proofDB, err := db.NewMemoryDBFromProof(encodedProofNodes) assert.NoError(t, err) return testCase{ diff --git a/lib/trie/trie.go b/lib/trie/trie.go index bdc8bdabdf..ce22a1e42a 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -34,7 +34,7 @@ type Trie struct { // NewEmptyTrie creates a trie with a nil root func NewEmptyTrie() *Trie { - return NewTrie(nil, db.NewEmptyInMemoryDB()) + return NewTrie(nil, db.NewEmptyMemoryDB()) } // NewTrie creates a trie with an existing root node diff --git a/lib/trie/trie_test.go b/lib/trie/trie_test.go index de1ed7a71e..af8efa0cc4 100644 --- a/lib/trie/trie_test.go +++ b/lib/trie/trie_test.go @@ -34,11 +34,10 @@ func Test_NewEmptyTrie(t *testing.T) { expectedTrie := &Trie{ childTries: make(map[common.Hash]*Trie), deltas: tracking.New(), - db: db.NewEmptyInMemoryDB(), + db: db.NewEmptyMemoryDB(), } trie := NewEmptyTrie() - - assert.True(t, trie.Equal(expectedTrie)) + assert.Equal(t, expectedTrie, trie) } func Test_NewTrie(t *testing.T) { @@ -726,7 +725,7 @@ func Test_Trie_Entries(t *testing.T) { trie := Trie{ root: nil, childTries: make(map[common.Hash]*Trie), - db: db.NewEmptyInMemoryDB(), + db: db.NewEmptyMemoryDB(), } kv := map[string][]byte{ @@ -751,7 +750,7 @@ func Test_Trie_Entries(t *testing.T) { trie := Trie{ root: nil, childTries: make(map[common.Hash]*Trie), - db: db.NewEmptyInMemoryDB(), + db: db.NewEmptyMemoryDB(), } kv := map[string][]byte{ @@ -1114,7 +1113,7 @@ func Test_Trie_Put(t *testing.T) { stateVersion Version key []byte value []byte - expectedTrie *Trie + expectedTrie Trie }{ "trie_v0_with_key_and_value": { trie: Trie{ @@ -1127,7 +1126,7 @@ func Test_Trie_Put(t *testing.T) { }, key: []byte{0x12, 0x16}, value: []byte{2}, - expectedTrie: &Trie{ + expectedTrie: Trie{ generation: 1, deltas: newDeltas("0xa195089c3e8f8b5b36978700ad954aed99e08413cfc1e2b4c00a5d064abe66a9"), root: &Node{ @@ -1160,12 +1159,12 @@ func Test_Trie_Put(t *testing.T) { PartialKey: []byte{1, 2, 0, 5}, StorageValue: []byte{1}, }, - db: db.NewEmptyInMemoryDB(), + db: db.NewEmptyMemoryDB(), }, stateVersion: V1, key: []byte{0x12, 0x16}, value: longValue, - expectedTrie: &Trie{ + expectedTrie: Trie{ generation: 1, deltas: newDeltas("0xa195089c3e8f8b5b36978700ad954aed99e08413cfc1e2b4c00a5d064abe66a9"), root: &Node{ @@ -1190,7 +1189,7 @@ func Test_Trie_Put(t *testing.T) { }), }, db: func() db.Database { - db := db.NewEmptyInMemoryDB() + db := db.NewEmptyMemoryDB() db.Put(longValueHash, longValue) return db }(), @@ -1206,7 +1205,7 @@ func Test_Trie_Put(t *testing.T) { trie := testCase.trie trie.Put(testCase.key, testCase.value, testCase.stateVersion) - assert.True(t, trie.Equal(testCase.expectedTrie)) + assert.Equal(t, testCase.expectedTrie, trie) }) } } @@ -1762,7 +1761,7 @@ func Test_LoadFromMap(t *testing.T) { expectedTrie: Trie{ childTries: map[common.Hash]*Trie{}, deltas: newDeltas(), - db: db.NewEmptyInMemoryDB(), + db: db.NewEmptyMemoryDB(), }, }, "empty_data": { @@ -1770,7 +1769,7 @@ func Test_LoadFromMap(t *testing.T) { expectedTrie: Trie{ childTries: map[common.Hash]*Trie{}, deltas: newDeltas(), - db: db.NewEmptyInMemoryDB(), + db: db.NewEmptyMemoryDB(), }, }, "bad_key": { @@ -1804,7 +1803,7 @@ func Test_LoadFromMap(t *testing.T) { }, childTries: map[common.Hash]*Trie{}, deltas: newDeltas(), - db: db.NewEmptyInMemoryDB(), + db: db.NewEmptyMemoryDB(), }, }, "load_key_values": { @@ -1835,7 +1834,7 @@ func Test_LoadFromMap(t *testing.T) { }, childTries: map[common.Hash]*Trie{}, deltas: newDeltas(), - db: db.NewEmptyInMemoryDB(), + db: db.NewEmptyMemoryDB(), }, }, } @@ -1852,7 +1851,7 @@ func Test_LoadFromMap(t *testing.T) { assert.EqualError(t, err, testCase.errMessage) } - assert.True(t, trie.Equal(&testCase.expectedTrie)) + assert.Equal(t, testCase.expectedTrie, trie) }) } } From 97ce2724ce792d6e27a10d48637b38f78bcf1b78 Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 14 Sep 2023 15:30:17 +0200 Subject: [PATCH 081/128] Add more test cases --- lib/trie/database_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/trie/database_test.go b/lib/trie/database_test.go index b05310729e..d512eb75ab 100644 --- a/lib/trie/database_test.go +++ b/lib/trie/database_test.go @@ -36,6 +36,15 @@ func Test_Trie_Store_Load(t *testing.T) { assert.Equal(t, trie.String(), trieFromDB.String()) } +func Test_Trie_Load_EmptyHash(t *testing.T) { + t.Parallel() + + db := newTestDB(t) + trieFromDB := NewEmptyTrie() + err := trieFromDB.Load(db, EmptyHash) + require.NoError(t, err) +} + func Test_Trie_WriteDirty_Put(t *testing.T) { t.Parallel() @@ -278,6 +287,17 @@ func Test_GetFromDB(t *testing.T) { } } +func Test_GetFromDB_EmptyHash(t *testing.T) { + t.Parallel() + + db := newTestDB(t) + + value, err := GetFromDB(db, EmptyHash, []byte("test")) + assert.NoError(t, err) + assert.Nil(t, value) + +} + func Test_Trie_PutChild_Store_Load(t *testing.T) { t.Parallel() From 26f3eb5ac7a3212dc29c6d705965410d1a72580c Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 14 Sep 2023 17:48:39 +0200 Subject: [PATCH 082/128] Fix documentations --- dot/state/interfaces.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dot/state/interfaces.go b/dot/state/interfaces.go index 8e53101645..55b3f426fe 100644 --- a/dot/state/interfaces.go +++ b/dot/state/interfaces.go @@ -29,7 +29,7 @@ type GetPutter interface { Putter } -// Batcher has methods to get values and create a +// GetterPutterNewBatcher has methods to get values and create a // new batch. type GetterPutterNewBatcher interface { Getter From dfefbbfbe6eee63577bd5f61892a037481676f62 Mon Sep 17 00:00:00 2001 From: Diego Date: Fri, 29 Sep 2023 12:16:18 +0200 Subject: [PATCH 083/128] Fix ClearChildStorage with versioning --- lib/runtime/interfaces.go | 2 +- lib/runtime/storage/trie.go | 4 ++-- lib/runtime/storage/trie_test.go | 2 +- lib/runtime/wazero/imports.go | 2 +- lib/trie/child_storage.go | 4 ++-- lib/trie/child_storage_test.go | 6 +++--- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/runtime/interfaces.go b/lib/runtime/interfaces.go index 6693e52c06..7f5439dae6 100644 --- a/lib/runtime/interfaces.go +++ b/lib/runtime/interfaces.go @@ -21,7 +21,7 @@ type Storage interface { DeleteChild(keyToChild []byte) (err error) DeleteChildLimit(keyToChild []byte, limit *[]byte) ( deleted uint32, allDeleted bool, err error) - ClearChildStorage(keyToChild, key []byte) error + ClearChildStorage(keyToChild, key []byte, version trie.Version) error NextKey([]byte) []byte ClearPrefixInChild(keyToChild, prefix []byte) error ClearPrefixInChildWithLimit(keyToChild, prefix []byte, limit uint32) (uint32, bool, error) diff --git a/lib/runtime/storage/trie.go b/lib/runtime/storage/trie.go index 35690efc2c..dcbf600309 100644 --- a/lib/runtime/storage/trie.go +++ b/lib/runtime/storage/trie.go @@ -232,10 +232,10 @@ func (s *TrieState) DeleteChildLimit(key []byte, limit *[]byte) ( } // ClearChildStorage removes the child storage entry from the trie -func (s *TrieState) ClearChildStorage(keyToChild, key []byte) error { +func (s *TrieState) ClearChildStorage(keyToChild, key []byte, version trie.Version) error { s.lock.Lock() defer s.lock.Unlock() - return s.t.ClearFromChild(keyToChild, key) + return s.t.ClearFromChild(keyToChild, key, version) } // ClearPrefixInChild clears all the keys from the child trie that have the given prefix diff --git a/lib/runtime/storage/trie_test.go b/lib/runtime/storage/trie_test.go index aafd0099a2..6728df653a 100644 --- a/lib/runtime/storage/trie_test.go +++ b/lib/runtime/storage/trie_test.go @@ -74,7 +74,7 @@ func TestTrieState_SetAndClearFromChild(t *testing.T) { } for _, tc := range testCases { - err := ts.ClearChildStorage([]byte(tc), []byte(tc)) + err := ts.ClearChildStorage([]byte(tc), []byte(tc), trie.V0) require.NoError(t, err) res, err := ts.GetChildStorage([]byte(tc), []byte(tc)) diff --git a/lib/runtime/wazero/imports.go b/lib/runtime/wazero/imports.go index 000d9b9985..d38040868f 100644 --- a/lib/runtime/wazero/imports.go +++ b/lib/runtime/wazero/imports.go @@ -1179,7 +1179,7 @@ func ext_default_child_storage_clear_version_1(ctx context.Context, m api.Module keyToChild := read(m, childStorageKey) key := read(m, keySpan) - err := storage.ClearChildStorage(keyToChild, key) + err := storage.ClearChildStorage(keyToChild, key, trie.V0) if err != nil { logger.Errorf("failed to clear child storage: %s", err) } diff --git a/lib/trie/child_storage.go b/lib/trie/child_storage.go index d998a09779..4ca063ed7f 100644 --- a/lib/trie/child_storage.go +++ b/lib/trie/child_storage.go @@ -106,7 +106,7 @@ func (t *Trie) DeleteChild(keyToChild []byte) (err error) { } // ClearFromChild removes the child storage entry -func (t *Trie) ClearFromChild(keyToChild, key []byte) error { +func (t *Trie) ClearFromChild(keyToChild, key []byte, version Version) error { child, err := t.GetChild(keyToChild) if err != nil { return err @@ -131,5 +131,5 @@ func (t *Trie) ClearFromChild(keyToChild, key []byte) error { return t.DeleteChild(keyToChild) } - return t.SetChild(keyToChild, child) + return t.SetChild(keyToChild, child, version) } diff --git a/lib/trie/child_storage_test.go b/lib/trie/child_storage_test.go index ec0ad2fd01..3c84279921 100644 --- a/lib/trie/child_storage_test.go +++ b/lib/trie/child_storage_test.go @@ -49,7 +49,7 @@ func TestPutAndClearFromChild(t *testing.T) { err := parentTrie.SetChild(childKey, childTrie, V0) assert.NoError(t, err) - err = parentTrie.ClearFromChild(childKey, keyInChild) + err = parentTrie.ClearFromChild(childKey, keyInChild, V0) assert.NoError(t, err) childTrie, err = parentTrie.GetChild(childKey) @@ -98,7 +98,7 @@ func TestChildTrieHashAfterClear(t *testing.T) { contributedWith := make([]byte, 8) binary.BigEndian.PutUint64(contributedWith, contributed) - err := trieThatHoldsAChildTrie.PutIntoChild(keyToChild, keyInChild, contributedWith) + err := trieThatHoldsAChildTrie.PutIntoChild(keyToChild, keyInChild, contributedWith, V0) require.NoError(t, err) // the parent trie hash SHOULT NOT BE EQUAL to the original @@ -111,7 +111,7 @@ func TestChildTrieHashAfterClear(t *testing.T) { require.Equal(t, contributed, binary.BigEndian.Uint64(valueStored)) // clear child trie key value - err = trieThatHoldsAChildTrie.ClearFromChild(keyToChild, keyInChild) + err = trieThatHoldsAChildTrie.ClearFromChild(keyToChild, keyInChild, V0) require.NoError(t, err) // the parent trie hash SHOULD BE EQUAL to the original From 1dbf91bce6650ce425656d5715966e9b88c73012 Mon Sep 17 00:00:00 2001 From: Diego Date: Fri, 29 Sep 2023 12:23:04 +0200 Subject: [PATCH 084/128] Fix tests --- lib/runtime/storage/trie_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/runtime/storage/trie_test.go b/lib/runtime/storage/trie_test.go index 6728df653a..13e62fa533 100644 --- a/lib/runtime/storage/trie_test.go +++ b/lib/runtime/storage/trie_test.go @@ -77,9 +77,8 @@ func TestTrieState_SetAndClearFromChild(t *testing.T) { err := ts.ClearChildStorage([]byte(tc), []byte(tc), trie.V0) require.NoError(t, err) - res, err := ts.GetChildStorage([]byte(tc), []byte(tc)) - require.NoError(t, err) - require.Equal(t, []uint8(nil), res) + _, err = ts.GetChildStorage([]byte(tc), []byte(tc)) + require.ErrorContains(t, err, "child trie does not exist at key") } } From 436100c16bda73e8f4a1b56164a49c2fa9ea045b Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 2 Oct 2023 12:14:23 +0200 Subject: [PATCH 085/128] Add todo reminder --- lib/trie/trie.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/trie/trie.go b/lib/trie/trie.go index ce22a1e42a..b89ce2eefe 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -147,6 +147,8 @@ func (t *Trie) registerDeletedNodeHash(node *Node, // If this node contains a hashed value we have to remove the value node from the db too if node.IsHashedValue { + // TODO: fix this! we could have an issue since we are using the same shared value node for N hashed nodes + // One way to fix it is having a reference counter for each value node pendingDeltas.RecordDeleted(common.NewHash(node.StorageValue)) } } From c2bdf2f0d941c76318ea4427a9f4538b59199612 Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 2 Oct 2023 14:20:17 +0200 Subject: [PATCH 086/128] Fix merge trie nodes preserve isHashed property --- internal/trie/node/node.go | 2 ++ internal/trie/node/node_test.go | 12 ++++++++++++ lib/trie/print_test.go | 4 ++++ lib/trie/trie.go | 9 +++++---- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/internal/trie/node/node.go b/internal/trie/node/node.go index e57416b0d2..4510fabd89 100644 --- a/internal/trie/node/node.go +++ b/internal/trie/node/node.go @@ -7,6 +7,7 @@ package node import ( "fmt" + "strconv" "github.com/qdm12/gotree" ) @@ -57,6 +58,7 @@ func (n *Node) StringNode() (stringNode *gotree.Node) { stringNode.Appendf("Dirty: %t", n.Dirty) stringNode.Appendf("Key: " + bytesToString(n.PartialKey)) stringNode.Appendf("Storage value: " + bytesToString(n.StorageValue)) + stringNode.Appendf("IsHashed: " + strconv.FormatBool(n.IsHashedValue)) if n.Descendants > 0 { // must be a branch stringNode.Appendf("Descendants: %d", n.Descendants) } diff --git a/internal/trie/node/node_test.go b/internal/trie/node/node_test.go index af4f2269b8..d24aec011e 100644 --- a/internal/trie/node/node_test.go +++ b/internal/trie/node/node_test.go @@ -27,6 +27,7 @@ func Test_Node_String(t *testing.T) { ├── Dirty: true ├── Key: 0x0102 ├── Storage value: 0x0304 +├── IsHashed: false └── Merkle value: nil`, }, "leaf_with_storage_value_higher_than_1024": { @@ -40,6 +41,7 @@ func Test_Node_String(t *testing.T) { ├── Dirty: true ├── Key: 0x0102 ├── Storage value: 0x0000000000000000...0000000000000000 +├── IsHashed: false └── Merkle value: nil`, }, "branch_with_storage_value_smaller_than_1024": { @@ -66,6 +68,7 @@ func Test_Node_String(t *testing.T) { ├── Dirty: true ├── Key: 0x0102 ├── Storage value: 0x0304 +├── IsHashed: false ├── Descendants: 3 ├── Merkle value: nil ├── Child 3 @@ -74,6 +77,7 @@ func Test_Node_String(t *testing.T) { | ├── Dirty: false | ├── Key: nil | ├── Storage value: nil +| ├── IsHashed: false | └── Merkle value: nil ├── Child 7 | └── Branch @@ -81,6 +85,7 @@ func Test_Node_String(t *testing.T) { | ├── Dirty: false | ├── Key: nil | ├── Storage value: nil +| ├── IsHashed: false | ├── Descendants: 1 | ├── Merkle value: nil | └── Child 0 @@ -89,6 +94,7 @@ func Test_Node_String(t *testing.T) { | ├── Dirty: false | ├── Key: nil | ├── Storage value: nil +| ├── IsHashed: false | └── Merkle value: nil └── Child 11 └── Leaf @@ -96,6 +102,7 @@ func Test_Node_String(t *testing.T) { ├── Dirty: false ├── Key: nil ├── Storage value: nil + ├── IsHashed: false └── Merkle value: nil`, }, "branch_with_storage_value_higher_than_1024": { @@ -122,6 +129,7 @@ func Test_Node_String(t *testing.T) { ├── Dirty: true ├── Key: 0x0102 ├── Storage value: 0x0000000000000000...0000000000000000 +├── IsHashed: false ├── Descendants: 3 ├── Merkle value: nil ├── Child 3 @@ -130,6 +138,7 @@ func Test_Node_String(t *testing.T) { | ├── Dirty: false | ├── Key: nil | ├── Storage value: nil +| ├── IsHashed: false | └── Merkle value: nil ├── Child 7 | └── Branch @@ -137,6 +146,7 @@ func Test_Node_String(t *testing.T) { | ├── Dirty: false | ├── Key: nil | ├── Storage value: nil +| ├── IsHashed: false | ├── Descendants: 1 | ├── Merkle value: nil | └── Child 0 @@ -145,6 +155,7 @@ func Test_Node_String(t *testing.T) { | ├── Dirty: false | ├── Key: nil | ├── Storage value: nil +| ├── IsHashed: false | └── Merkle value: nil └── Child 11 └── Leaf @@ -152,6 +163,7 @@ func Test_Node_String(t *testing.T) { ├── Dirty: false ├── Key: nil ├── Storage value: nil + ├── IsHashed: false └── Merkle value: nil`, }, } diff --git a/lib/trie/print_test.go b/lib/trie/print_test.go index ff71d5123f..7610b34697 100644 --- a/lib/trie/print_test.go +++ b/lib/trie/print_test.go @@ -32,6 +32,7 @@ func Test_Trie_String(t *testing.T) { ├── Dirty: false ├── Key: 0x010203 ├── Storage value: 0x030405 +├── IsHashed: false └── Merkle value: nil`, }, "branch_root": { @@ -60,6 +61,7 @@ func Test_Trie_String(t *testing.T) { ├── Dirty: false ├── Key: nil ├── Storage value: 0x0102 +├── IsHashed: false ├── Descendants: 2 ├── Merkle value: nil ├── Child 0 @@ -68,6 +70,7 @@ func Test_Trie_String(t *testing.T) { | ├── Dirty: false | ├── Key: 0x010203 | ├── Storage value: 0x030405 +| ├── IsHashed: false | └── Merkle value: nil └── Child 3 └── Leaf @@ -75,6 +78,7 @@ func Test_Trie_String(t *testing.T) { ├── Dirty: false ├── Key: 0x010203 ├── Storage value: 0x030405 + ├── IsHashed: false └── Merkle value: nil`, }, } diff --git a/lib/trie/trie.go b/lib/trie/trie.go index b89ce2eefe..6229cd6679 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -1395,10 +1395,11 @@ func (t *Trie) handleDeletion(branch *Node, key []byte, if child.Kind() == node.Leaf { newLeafKey := concatenateSlices(branch.PartialKey, intToByteSlice(childIndex), child.PartialKey) return &Node{ - PartialKey: newLeafKey, - StorageValue: child.StorageValue, - Dirty: true, - Generation: branch.Generation, + PartialKey: newLeafKey, + StorageValue: child.StorageValue, + IsHashedValue: child.IsHashedValue, + Dirty: true, + Generation: branch.Generation, }, branchChildMerged, nil } From 6e1e6122dcf2bc8c19455bc89a2be6ca4d18baf7 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 3 Oct 2023 13:38:16 +0200 Subject: [PATCH 087/128] Simplify runtime functions reusing v2 in v1 --- lib/common/hash.go | 8 ++- lib/runtime/wazero/imports.go | 123 +++------------------------------- lib/trie/version.go | 32 +++++++-- 3 files changed, 41 insertions(+), 122 deletions(-) diff --git a/lib/common/hash.go b/lib/common/hash.go index 3f2ebaf99d..28798afb46 100644 --- a/lib/common/hash.go +++ b/lib/common/hash.go @@ -18,6 +18,8 @@ const ( HashLength = 32 ) +var EmptyHash = Hash{} + // Hash used to store a blake2b hash type Hash [32]byte @@ -40,7 +42,7 @@ func HashValidator(field reflect.Value) interface{} { // Try to convert to hash type. if valuer, ok := field.Interface().(Hash); ok { // Check if the hash is empty. - if valuer == (Hash{}) { + if valuer == (EmptyHash) { return "" } return valuer.ToBytes() @@ -50,7 +52,7 @@ func HashValidator(field reflect.Value) interface{} { // IsEmpty returns true if the hash is empty, false otherwise. func (h Hash) IsEmpty() bool { //skipcq: GO-W1029 - return h == Hash{} + return h == EmptyHash } // String returns the hex string for the hash @@ -79,7 +81,7 @@ func ReadHash(r io.Reader) (Hash, error) { buf := make([]byte, 32) _, err := r.Read(buf) if err != nil { - return Hash{}, err + return EmptyHash, err } h := [32]byte{} copy(h[:], buf) diff --git a/lib/runtime/wazero/imports.go b/lib/runtime/wazero/imports.go index 0659de5259..9c2dddb4f9 100644 --- a/lib/runtime/wazero/imports.go +++ b/lib/runtime/wazero/imports.go @@ -780,51 +780,7 @@ func ext_crypto_finish_batch_verify_version_1(ctx context.Context, m api.Module) } func ext_trie_blake2_256_root_version_1(ctx context.Context, m api.Module, dataSpan uint64) uint32 { - rtCtx := ctx.Value(runtimeContextKey).(*runtime.Context) - if rtCtx == nil { - panic("nil runtime context") - } - - data := read(m, dataSpan) - - t := trie.NewEmptyTrie() - - type kv struct { - Key, Value []byte - } - - // this function is expecting an array of (key, value) tuples - var kvs []kv - if err := scale.Unmarshal(data, &kvs); err != nil { - logger.Errorf("failed scale decoding data: %s", err) - return 0 - } - - for _, kv := range kvs { - err := t.Put(kv.Key, kv.Value, trie.V0) - if err != nil { - logger.Errorf("failed putting key 0x%x and value 0x%x into trie: %s", - kv.Key, kv.Value, err) - return 0 - } - } - - // allocate memory for value and copy value to memory - ptr, err := rtCtx.Allocator.Allocate(32) - if err != nil { - logger.Errorf("failed allocating: %s", err) - return 0 - } - - hash, err := t.Hash() - if err != nil { - logger.Errorf("failed computing trie Merkle root hash: %s", err) - return 0 - } - - logger.Debugf("root hash is %s", hash) - m.Memory().Write(ptr, hash[:]) - return ptr + return ext_trie_blake2_256_root_version_2(ctx, m, dataSpan, 0) } func ext_trie_blake2_256_root_version_2(ctx context.Context, m api.Module, dataSpan uint64, version uint32) uint32 { @@ -842,80 +798,19 @@ func ext_trie_blake2_256_root_version_2(ctx context.Context, m api.Module, dataS data := read(m, dataSpan) - t := trie.NewEmptyTrie() - - type kv struct { - Key, Value []byte - } - // this function is expecting an array of (key, value) tuples - var kvs []kv - if err := scale.Unmarshal(data, &kvs); err != nil { + var entries trie.Entries + if err := scale.Unmarshal(data, &entries); err != nil { logger.Errorf("failed scale decoding data: %s", err) return 0 } - for _, kv := range kvs { - err := t.Put(kv.Key, kv.Value, stateVersion) - if err != nil { - logger.Errorf("failed putting key 0x%x and value 0x%x into trie: %s", - kv.Key, kv.Value, err) - return 0 - } - } - - // allocate memory for value and copy value to memory - ptr, err := rtCtx.Allocator.Allocate(32) - if err != nil { - logger.Errorf("failed allocating: %s", err) - return 0 - } - - hash, err := t.Hash() + hash, err := stateVersion.Root(entries) if err != nil { logger.Errorf("failed computing trie Merkle root hash: %s", err) return 0 } - logger.Debugf("root hash is %s", hash) - m.Memory().Write(ptr, hash[:]) - return ptr -} - -func ext_trie_blake2_256_ordered_root_version_1(ctx context.Context, m api.Module, dataSpan uint64) uint32 { - rtCtx := ctx.Value(runtimeContextKey).(*runtime.Context) - if rtCtx == nil { - panic("nil runtime context") - } - - data := read(m, dataSpan) - - t := trie.NewEmptyTrie() - var values [][]byte - err := scale.Unmarshal(data, &values) - if err != nil { - logger.Errorf("failed scale decoding data: %s", err) - return 0 - } - - for i, value := range values { - key, err := scale.Marshal(big.NewInt(int64(i))) - if err != nil { - logger.Errorf("failed scale encoding value index %d: %s", i, err) - return 0 - } - logger.Tracef( - "put key=0x%x and value=0x%x", - key, value) - - err = t.Put(key, value, trie.V0) - if err != nil { - logger.Errorf("failed putting key 0x%x and value 0x%x into trie: %s", - key, value, err) - return 0 - } - } - // allocate memory for value and copy value to memory ptr, err := rtCtx.Allocator.Allocate(32) if err != nil { @@ -923,17 +818,15 @@ func ext_trie_blake2_256_ordered_root_version_1(ctx context.Context, m api.Modul return 0 } - hash, err := t.Hash() - if err != nil { - logger.Errorf("failed computing trie Merkle root hash: %s", err) - return 0 - } - logger.Debugf("root hash is %s", hash) m.Memory().Write(ptr, hash[:]) return ptr } +func ext_trie_blake2_256_ordered_root_version_1(ctx context.Context, m api.Module, dataSpan uint64) uint32 { + return ext_trie_blake2_256_ordered_root_version_2(ctx, m, dataSpan, 0) +} + func ext_trie_blake2_256_ordered_root_version_2( ctx context.Context, m api.Module, dataSpan uint64, version uint32) uint32 { rtCtx := ctx.Value(runtimeContextKey).(*runtime.Context) diff --git a/lib/trie/version.go b/lib/trie/version.go index 38366fbb09..fbc60609fa 100644 --- a/lib/trie/version.go +++ b/lib/trie/version.go @@ -7,9 +7,14 @@ import ( "errors" "fmt" "strings" + + "github.com/ChainSafe/gossamer/lib/common" ) -const V1MaxValueSize = 32 +const ( + NoMaxValueSize = -1 + V1MaxValueSize = 32 +) // Version is the state trie version which dictates how a // Merkle root should be constructed. It is defined in @@ -28,6 +33,8 @@ const ( // See https://github.com/paritytech/substrate/blob/5e76587825b9a9d52d8cb02ba38828adf606157b/primitives/storage/src/lib.rs#L435-L439 const DefaultStateVersion = V1 +type Entries []struct{ Key, Value []byte } + func (v Version) String() string { switch v { case V0: @@ -39,17 +46,34 @@ func (v Version) String() string { } } -func (v Version) ShouldHashValue(value []byte) bool { +func (v Version) MaxInlineValueSize() int { switch v { case V0: - return false + return NoMaxValueSize case V1: - return len(value) >= V1MaxValueSize + return V1MaxValueSize default: panic(fmt.Sprintf("unknown version %d", v)) } } +func (v Version) ShouldHashValue(value []byte) bool { + return v.MaxInlineValueSize() != NoMaxValueSize && len(value) > v.MaxInlineValueSize() +} + +func (v Version) Root(entries Entries) (common.Hash, error) { + t := NewEmptyTrie() + + for _, kv := range entries { + err := t.Put(kv.Key, kv.Value, v) + if err != nil { + return common.EmptyHash, err + } + } + + return t.Hash() +} + var ErrParseVersion = errors.New("parsing version failed") // ParseVersion parses a state trie version string. From 1096a93c737551002a951c0b4c7db925b8675b51 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 3 Oct 2023 14:01:18 +0200 Subject: [PATCH 088/128] Reuse new root from entries function in ext_ordered_root --- lib/runtime/wazero/imports.go | 15 ++++----------- lib/trie/version.go | 3 ++- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/lib/runtime/wazero/imports.go b/lib/runtime/wazero/imports.go index 9c2dddb4f9..ec9af1229d 100644 --- a/lib/runtime/wazero/imports.go +++ b/lib/runtime/wazero/imports.go @@ -843,7 +843,6 @@ func ext_trie_blake2_256_ordered_root_version_2( return 0 } - t := trie.NewEmptyTrie() var values [][]byte err = scale.Unmarshal(data, &values) if err != nil { @@ -851,22 +850,16 @@ func ext_trie_blake2_256_ordered_root_version_2( return 0 } + var entries trie.Entries + for i, value := range values { key, err := scale.Marshal(big.NewInt(int64(i))) if err != nil { logger.Errorf("failed scale encoding value index %d: %s", i, err) return 0 } - logger.Tracef( - "put key=0x%x and value=0x%x", - key, value) - err = t.Put(key, value, stateVersion) - if err != nil { - logger.Errorf("failed putting key 0x%x and value 0x%x into trie: %s", - key, value, err) - return 0 - } + entries = append(entries, trie.Entry{Key: key, Value: value}) } // allocate memory for value and copy value to memory @@ -876,7 +869,7 @@ func ext_trie_blake2_256_ordered_root_version_2( return 0 } - hash, err := t.Hash() + hash, err := stateVersion.Root(entries) if err != nil { logger.Errorf("failed computing trie Merkle root hash: %s", err) return 0 diff --git a/lib/trie/version.go b/lib/trie/version.go index fbc60609fa..962066234a 100644 --- a/lib/trie/version.go +++ b/lib/trie/version.go @@ -33,7 +33,8 @@ const ( // See https://github.com/paritytech/substrate/blob/5e76587825b9a9d52d8cb02ba38828adf606157b/primitives/storage/src/lib.rs#L435-L439 const DefaultStateVersion = V1 -type Entries []struct{ Key, Value []byte } +type Entry struct{ Key, Value []byte } +type Entries []Entry func (v Version) String() string { switch v { From 4d6434a50c491288eeeb0d2d973d93ee09dd7ac6 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 3 Oct 2023 14:16:53 +0200 Subject: [PATCH 089/128] Add tests for new root function --- lib/trie/version_test.go | 98 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/lib/trie/version_test.go b/lib/trie/version_test.go index 19892bdb1d..801da07366 100644 --- a/lib/trie/version_test.go +++ b/lib/trie/version_test.go @@ -6,6 +6,7 @@ package trie import ( "testing" + "github.com/ChainSafe/gossamer/lib/common" "github.com/stretchr/testify/assert" ) @@ -168,3 +169,100 @@ func Test_ShouldHashValue(t *testing.T) { }) } } + +func Test_Version_MaxInlineValueSize(t *testing.T) { + t.Parallel() + + testCases := map[string]struct { + version Version + maxInline int + panicMessage string + }{ + "v0": { + version: V0, + maxInline: NoMaxValueSize, + }, + "v1": { + version: V1, + maxInline: V1MaxValueSize, + }, + "invalid": { + version: Version(99), + panicMessage: "unknown version 99", + }, + } + + for name, testCase := range testCases { + testCase := testCase + t.Run(name, func(t *testing.T) { + t.Parallel() + + if testCase.panicMessage != "" { + assert.PanicsWithValue(t, testCase.panicMessage, func() { + _ = testCase.version.MaxInlineValueSize() + }) + return + } + + maxInline := testCase.version.MaxInlineValueSize() + assert.Equal(t, testCase.maxInline, maxInline) + }) + } +} + +func Test_Version_Root(t *testing.T) { + t.Parallel() + + testCases := map[string]struct { + version Version + input Entries + expected common.Hash + err error + panicMessage string + }{ + "v0": { + version: V0, + input: Entries{ + Entry{Key: []byte("key1"), Value: []byte("value1")}, + Entry{Key: []byte("key2"), Value: []byte("value2")}, + Entry{Key: []byte("key3"), Value: []byte("verylargevaluewithmorethan32byteslength")}, + }, + expected: common.Hash{0x71, 0x5, 0x2d, 0x48, 0x70, 0x46, 0x58, 0xa8, 0x43, 0x5f, 0xb9, 0xcb, 0xc7, 0xef, 0x69, 0xc7, 0x5d, 0xad, 0x2f, 0x64, 0x0, 0x1c, 0xb3, 0xb, 0xfa, 0x1, 0xf, 0x7d, 0x60, 0x9e, 0x26, 0x57}, + }, + "v1": { + version: V1, + input: Entries{ + Entry{Key: []byte("key1"), Value: []byte("value1")}, + Entry{Key: []byte("key2"), Value: []byte("value2")}, + Entry{Key: []byte("key3"), Value: []byte("verylargevaluewithmorethan32byteslength")}, + }, + expected: common.Hash{0x6a, 0x4a, 0x73, 0x27, 0x57, 0x26, 0x3b, 0xf2, 0xbc, 0x4e, 0x3, 0xa3, 0x41, 0xe3, 0xf8, 0xea, 0x63, 0x5f, 0x78, 0x99, 0x6e, 0xc0, 0x6a, 0x6a, 0x96, 0x5d, 0x50, 0x97, 0xa2, 0x91, 0x1c, 0x29}, + }, + "invalid": { + version: Version(99), + panicMessage: "unknown version 99", + }, + } + + for name, testCase := range testCases { + testCase := testCase + t.Run(name, func(t *testing.T) { + t.Parallel() + + if testCase.panicMessage != "" { + assert.PanicsWithValue(t, testCase.panicMessage, func() { + _ = testCase.version.MaxInlineValueSize() + }) + return + } + + maxInline, err := testCase.version.Root(testCase.input) + if testCase.err != nil { + assert.ErrorIs(t, err, testCase.err) + } else { + assert.NoError(t, err) + assert.Equal(t, testCase.expected, maxInline) + } + }) + } +} From d8df35e4a0e36525e4c2d398fda0def793d57a27 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 3 Oct 2023 23:45:09 +0200 Subject: [PATCH 090/128] Fix lint lll --- lib/trie/version_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/trie/version_test.go b/lib/trie/version_test.go index 801da07366..0c3f80f4df 100644 --- a/lib/trie/version_test.go +++ b/lib/trie/version_test.go @@ -227,7 +227,10 @@ func Test_Version_Root(t *testing.T) { Entry{Key: []byte("key2"), Value: []byte("value2")}, Entry{Key: []byte("key3"), Value: []byte("verylargevaluewithmorethan32byteslength")}, }, - expected: common.Hash{0x71, 0x5, 0x2d, 0x48, 0x70, 0x46, 0x58, 0xa8, 0x43, 0x5f, 0xb9, 0xcb, 0xc7, 0xef, 0x69, 0xc7, 0x5d, 0xad, 0x2f, 0x64, 0x0, 0x1c, 0xb3, 0xb, 0xfa, 0x1, 0xf, 0x7d, 0x60, 0x9e, 0x26, 0x57}, + expected: common.Hash{ + 0x71, 0x5, 0x2d, 0x48, 0x70, 0x46, 0x58, 0xa8, 0x43, 0x5f, 0xb9, 0xcb, 0xc7, 0xef, 0x69, 0xc7, 0x5d, + 0xad, 0x2f, 0x64, 0x0, 0x1c, 0xb3, 0xb, 0xfa, 0x1, 0xf, 0x7d, 0x60, 0x9e, 0x26, 0x57, + }, }, "v1": { version: V1, @@ -236,7 +239,10 @@ func Test_Version_Root(t *testing.T) { Entry{Key: []byte("key2"), Value: []byte("value2")}, Entry{Key: []byte("key3"), Value: []byte("verylargevaluewithmorethan32byteslength")}, }, - expected: common.Hash{0x6a, 0x4a, 0x73, 0x27, 0x57, 0x26, 0x3b, 0xf2, 0xbc, 0x4e, 0x3, 0xa3, 0x41, 0xe3, 0xf8, 0xea, 0x63, 0x5f, 0x78, 0x99, 0x6e, 0xc0, 0x6a, 0x6a, 0x96, 0x5d, 0x50, 0x97, 0xa2, 0x91, 0x1c, 0x29}, + expected: common.Hash{ + 0x6a, 0x4a, 0x73, 0x27, 0x57, 0x26, 0x3b, 0xf2, 0xbc, 0x4e, 0x3, 0xa3, 0x41, 0xe3, 0xf8, 0xea, 0x63, + 0x5f, 0x78, 0x99, 0x6e, 0xc0, 0x6a, 0x6a, 0x96, 0x5d, 0x50, 0x97, 0xa2, 0x91, 0x1c, 0x29, + }, }, "invalid": { version: Version(99), From 8c92fbb9d8ca965452146bae91b49461a36e3b19 Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 4 Oct 2023 00:12:13 +0200 Subject: [PATCH 091/128] Improve trie versions functions --- lib/trie/version.go | 13 +------- lib/trie/version_test.go | 67 +++------------------------------------- 2 files changed, 6 insertions(+), 74 deletions(-) diff --git a/lib/trie/version.go b/lib/trie/version.go index 962066234a..2ffd6e88dd 100644 --- a/lib/trie/version.go +++ b/lib/trie/version.go @@ -47,19 +47,8 @@ func (v Version) String() string { } } -func (v Version) MaxInlineValueSize() int { - switch v { - case V0: - return NoMaxValueSize - case V1: - return V1MaxValueSize - default: - panic(fmt.Sprintf("unknown version %d", v)) - } -} - func (v Version) ShouldHashValue(value []byte) bool { - return v.MaxInlineValueSize() != NoMaxValueSize && len(value) > v.MaxInlineValueSize() + return v == V1 && len(value) > V1MaxValueSize } func (v Version) Root(entries Entries) (common.Hash, error) { diff --git a/lib/trie/version_test.go b/lib/trie/version_test.go index 0c3f80f4df..970dc5b628 100644 --- a/lib/trie/version_test.go +++ b/lib/trie/version_test.go @@ -170,55 +170,13 @@ func Test_ShouldHashValue(t *testing.T) { } } -func Test_Version_MaxInlineValueSize(t *testing.T) { - t.Parallel() - - testCases := map[string]struct { - version Version - maxInline int - panicMessage string - }{ - "v0": { - version: V0, - maxInline: NoMaxValueSize, - }, - "v1": { - version: V1, - maxInline: V1MaxValueSize, - }, - "invalid": { - version: Version(99), - panicMessage: "unknown version 99", - }, - } - - for name, testCase := range testCases { - testCase := testCase - t.Run(name, func(t *testing.T) { - t.Parallel() - - if testCase.panicMessage != "" { - assert.PanicsWithValue(t, testCase.panicMessage, func() { - _ = testCase.version.MaxInlineValueSize() - }) - return - } - - maxInline := testCase.version.MaxInlineValueSize() - assert.Equal(t, testCase.maxInline, maxInline) - }) - } -} - func Test_Version_Root(t *testing.T) { t.Parallel() testCases := map[string]struct { - version Version - input Entries - expected common.Hash - err error - panicMessage string + version Version + input Entries + expected common.Hash }{ "v0": { version: V0, @@ -244,10 +202,6 @@ func Test_Version_Root(t *testing.T) { 0x5f, 0x78, 0x99, 0x6e, 0xc0, 0x6a, 0x6a, 0x96, 0x5d, 0x50, 0x97, 0xa2, 0x91, 0x1c, 0x29, }, }, - "invalid": { - version: Version(99), - panicMessage: "unknown version 99", - }, } for name, testCase := range testCases { @@ -255,20 +209,9 @@ func Test_Version_Root(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - if testCase.panicMessage != "" { - assert.PanicsWithValue(t, testCase.panicMessage, func() { - _ = testCase.version.MaxInlineValueSize() - }) - return - } - maxInline, err := testCase.version.Root(testCase.input) - if testCase.err != nil { - assert.ErrorIs(t, err, testCase.err) - } else { - assert.NoError(t, err) - assert.Equal(t, testCase.expected, maxInline) - } + assert.NoError(t, err) + assert.Equal(t, testCase.expected, maxInline) }) } } From 784f96e1bf3e9fdf172aba6303b0c86825dfbda4 Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 4 Oct 2023 11:35:03 +0200 Subject: [PATCH 092/128] Fix test --- lib/trie/version_test.go | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/lib/trie/version_test.go b/lib/trie/version_test.go index 970dc5b628..dfb7a5c85c 100644 --- a/lib/trie/version_test.go +++ b/lib/trie/version_test.go @@ -121,10 +121,9 @@ func Test_ShouldHashValue(t *testing.T) { t.Parallel() testCases := map[string]struct { - version Version - value []byte - shouldHash bool - panicMessage string + version Version + value []byte + shouldHash bool }{ "v0_small_value": { version: V0, @@ -146,10 +145,6 @@ func Test_ShouldHashValue(t *testing.T) { value: []byte("newvaluewithmorethan32byteslength"), shouldHash: true, }, - "invalid": { - version: Version(99), - panicMessage: "unknown version 99", - }, } for name, testCase := range testCases { @@ -157,13 +152,6 @@ func Test_ShouldHashValue(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - if testCase.panicMessage != "" { - assert.PanicsWithValue(t, testCase.panicMessage, func() { - _ = testCase.version.ShouldHashValue(testCase.value) - }) - return - } - shouldHash := testCase.version.ShouldHashValue(testCase.value) assert.Equal(t, testCase.shouldHash, shouldHash) }) From a276576559aa2d37a7ae26a577ac5ab960891118 Mon Sep 17 00:00:00 2001 From: Diego Date: Sun, 8 Oct 2023 11:32:47 +0100 Subject: [PATCH 093/128] PR comments --- lib/trie/trie.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/trie/trie.go b/lib/trie/trie.go index 6229cd6679..fc5b28fa89 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -374,13 +374,14 @@ func (t *Trie) insertKeyLE(keyLE, value []byte, shouldHash := version.ShouldHashValue(value) if shouldHash { hashedValue := common.MustBlake2bHash(value) - + valueKey := hashedValue.ToBytes() // Add the original value as value node in db using the hashed value as key - err = t.db.Put(hashedValue.ToBytes(), value) + err = t.db.Put(valueKey, value) if err != nil { return err } - value = hashedValue.ToBytes() + // Put the value key as this trie node value + value = valueKey } root, _, _, err := t.insert(t.root, nibblesKey, value, shouldHash, pendingDeltas) From 8cf3b4a575dabf3086beb747993e2147182ff7dd Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 6 Nov 2023 12:30:40 -0300 Subject: [PATCH 094/128] PR review comments --- lib/runtime/storage/trie_test.go | 27 ++++++++++++--------------- lib/trie/database_test.go | 1 - lib/trie/db/db.go | 12 ++++++------ 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/lib/runtime/storage/trie_test.go b/lib/runtime/storage/trie_test.go index 13e62fa533..b70710e2be 100644 --- a/lib/runtime/storage/trie_test.go +++ b/lib/runtime/storage/trie_test.go @@ -41,25 +41,22 @@ func TestTrieState_SetGet(t *testing.T) { } func TestTrieState_SetGetChildStorage(t *testing.T) { - testFunc := func(ts *TrieState) { - for _, tc := range testCases { - childTrie := trie.NewEmptyTrie() - err := ts.SetChild([]byte(tc), childTrie, trie.V0) - require.NoError(t, err) + ts := &TrieState{t: trie.NewEmptyTrie()} - err = ts.SetChildStorage([]byte(tc), []byte(tc), []byte(tc), trie.V0) - require.NoError(t, err) - } + for _, tc := range testCases { + childTrie := trie.NewEmptyTrie() + err := ts.SetChild([]byte(tc), childTrie, trie.V0) + require.NoError(t, err) - for _, tc := range testCases { - res, err := ts.GetChildStorage([]byte(tc), []byte(tc)) - require.NoError(t, err) - require.Equal(t, []byte(tc), res) - } + err = ts.SetChildStorage([]byte(tc), []byte(tc), []byte(tc), trie.V0) + require.NoError(t, err) } - ts := &TrieState{t: trie.NewEmptyTrie()} - testFunc(ts) + for _, tc := range testCases { + res, err := ts.GetChildStorage([]byte(tc), []byte(tc)) + require.NoError(t, err) + require.Equal(t, []byte(tc), res) + } } func TestTrieState_SetAndClearFromChild(t *testing.T) { diff --git a/lib/trie/database_test.go b/lib/trie/database_test.go index d512eb75ab..cb458826e1 100644 --- a/lib/trie/database_test.go +++ b/lib/trie/database_test.go @@ -295,7 +295,6 @@ func Test_GetFromDB_EmptyHash(t *testing.T) { value, err := GetFromDB(db, EmptyHash, []byte("test")) assert.NoError(t, err) assert.Nil(t, value) - } func Test_Trie_PutChild_Store_Load(t *testing.T) { diff --git a/lib/trie/db/db.go b/lib/trie/db/db.go index 6ecc5d9273..02579d4906 100644 --- a/lib/trie/db/db.go +++ b/lib/trie/db/db.go @@ -25,8 +25,8 @@ type DBPutter interface { } type MemoryDB struct { - data map[common.Hash][]byte - l sync.RWMutex + data map[common.Hash][]byte + mutex sync.RWMutex } func NewEmptyMemoryDB() *MemoryDB { @@ -72,8 +72,8 @@ func (mdb *MemoryDB) Get(key []byte) ([]byte, error) { var hash common.Hash copy(hash[:], key) - mdb.l.RLock() - defer mdb.l.RUnlock() + mdb.mutex.RLock() + defer mdb.mutex.RUnlock() if value, found := mdb.data[hash]; found { return value, nil @@ -90,8 +90,8 @@ func (mdb *MemoryDB) Put(key []byte, value []byte) error { var hash common.Hash copy(hash[:], key) - mdb.l.Lock() - defer mdb.l.Unlock() + mdb.mutex.Lock() + defer mdb.mutex.Unlock() mdb.data[hash] = value return nil From a9a25e9613e7b1a6e8e9190e1ecccda2eacadffd Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 6 Nov 2023 12:34:44 -0300 Subject: [PATCH 095/128] Add godoc for exported method and consts --- lib/trie/version.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/trie/version.go b/lib/trie/version.go index 2ffd6e88dd..ab3f2e729f 100644 --- a/lib/trie/version.go +++ b/lib/trie/version.go @@ -12,7 +12,9 @@ import ( ) const ( + // NoMaxValueSize is the numeric representation used to indicate that there is no max value size. NoMaxValueSize = -1 + // V1MaxValueSize is the maximum size of a value to be hashed in state trie version 1. V1MaxValueSize = 32 ) @@ -29,13 +31,20 @@ const ( V1 ) +// ErrParseVersion is returned when parsing a state trie version fails. +var ErrParseVersion = errors.New("parsing version failed") + // DefaultStateVersion sets the state version we should use as default // See https://github.com/paritytech/substrate/blob/5e76587825b9a9d52d8cb02ba38828adf606157b/primitives/storage/src/lib.rs#L435-L439 const DefaultStateVersion = V1 +// Entry is a key-value pair used to build a trie type Entry struct{ Key, Value []byte } + +// Entries is a list of entry used to build a trie type Entries []Entry +// String returns a string representation of trie version func (v Version) String() string { switch v { case V0: @@ -47,10 +56,12 @@ func (v Version) String() string { } } +// ShouldHashValue returns true if the value should be hashed based on trie version func (v Version) ShouldHashValue(value []byte) bool { return v == V1 && len(value) > V1MaxValueSize } +// Root returns the root hash of the trie built using the given entries func (v Version) Root(entries Entries) (common.Hash, error) { t := NewEmptyTrie() @@ -64,8 +75,6 @@ func (v Version) Root(entries Entries) (common.Hash, error) { return t.Hash() } -var ErrParseVersion = errors.New("parsing version failed") - // ParseVersion parses a state trie version string. func ParseVersion[T string | uint32](v T) (version Version, err error) { var s string From 487e1ea0aa536cfc2353b1f3914f9f4fee92a898 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 7 Nov 2023 13:08:53 -0300 Subject: [PATCH 096/128] Do not store hashed values --- cmd/gossamer/commands/import_state.go | 11 +- dot/core/helpers_test.go | 4 +- dot/core/service_integration_test.go | 4 +- dot/import.go | 8 +- dot/import_integration_test.go | 6 +- dot/import_test.go | 5 +- dot/node_integration_test.go | 2 +- dot/rpc/modules/author_integration_test.go | 2 +- .../modules/childstate_integration_test.go | 12 +- dot/rpc/modules/childstate_test.go | 12 +- dot/rpc/modules/dev_integration_test.go | 3 +- dot/rpc/modules/state_integration_test.go | 7 +- dot/rpc/modules/system_integration_test.go | 2 +- dot/state/base_test.go | 2 +- dot/state/service_integration_test.go | 4 +- dot/state/storage_notify_test.go | 7 +- dot/state/storage_test.go | 61 +------- dot/utils_integration_test.go | 4 +- lib/runtime/genesis.go | 2 +- lib/runtime/interfaces.go | 8 +- lib/runtime/storage/trie.go | 16 +-- lib/runtime/storage/trie_test.go | 36 ++--- lib/runtime/wazero/imports.go | 12 +- lib/runtime/wazero/imports_test.go | 136 +++++++++--------- lib/runtime/wazero/instance_test.go | 18 +-- lib/trie/child_storage.go | 14 +- lib/trie/child_storage_test.go | 18 +-- lib/trie/database_test.go | 6 +- lib/trie/helpers_test.go | 2 +- lib/trie/mem_test.go | 4 +- lib/trie/proof/generate_test.go | 3 +- lib/trie/proof/proof_test.go | 3 +- lib/trie/trie.go | 71 ++++----- lib/trie/trie_endtoend_test.go | 44 +++--- lib/trie/trie_test.go | 73 ++-------- lib/trie/version.go | 23 +-- lib/trie/version_test.go | 41 ------ 37 files changed, 257 insertions(+), 429 deletions(-) diff --git a/cmd/gossamer/commands/import_state.go b/cmd/gossamer/commands/import_state.go index 6ee5ac4671..850101f81d 100644 --- a/cmd/gossamer/commands/import_state.go +++ b/cmd/gossamer/commands/import_state.go @@ -63,16 +63,7 @@ func execImportState(cmd *cobra.Command) error { return fmt.Errorf("header-file must be specified") } - stateVersionFlag, err := cmd.Flags().GetString("state-version") - if err != nil { - return fmt.Errorf("failed to get state-version: %s", err) - } - stateVersion, err := trie.ParseVersion(stateVersionFlag) - if err != nil { - return fmt.Errorf("failed to parse state-version: %s", err) - } - basePath = utils.ExpandDir(basePath) - return dot.ImportState(basePath, stateFile, headerFile, firstSlot, stateVersion) + return dot.ImportState(basePath, stateFile, headerFile, firstSlot) } diff --git a/dot/core/helpers_test.go b/dot/core/helpers_test.go index 9483881d39..53a2af4d78 100644 --- a/dot/core/helpers_test.go +++ b/dot/core/helpers_test.go @@ -109,9 +109,9 @@ func createTestService(t *testing.T, genesisFilePath string, cfgRuntime, err := wazero_runtime.NewRuntimeFromGenesis(rtCfg) require.NoError(t, err) - cfgRuntime.Context.Storage.Put(aliceBalanceKey, encodedAccountInfo, trie.V0) + cfgRuntime.Context.Storage.Put(aliceBalanceKey, encodedAccountInfo) // this key is System.UpgradedToDualRefCount -> set to true since all accounts have been upgraded to v0.9 format - cfgRuntime.Context.Storage.Put(common.UpgradedToDualRefKey, []byte{1}, trie.V0) + cfgRuntime.Context.Storage.Put(common.UpgradedToDualRefKey, []byte{1}) cfgBlockState.StoreRuntime(cfgBlockState.BestBlockHash(), cfgRuntime) diff --git a/dot/core/service_integration_test.go b/dot/core/service_integration_test.go index 38c49fd986..dcb8c2f346 100644 --- a/dot/core/service_integration_test.go +++ b/dot/core/service_integration_test.go @@ -590,7 +590,7 @@ func createBlockUsingNewRuntime(t *testing.T, bestBlockHash common.Hash, newRunt testRuntime, err := os.ReadFile(newRuntimePath) require.NoError(t, err) - trieState.Put(common.CodeKey, testRuntime, trie.V0) + trieState.Put(common.CodeKey, testRuntime) primaryDigestData := types.NewBabePrimaryPreDigest(0, uint64(1), [32]byte{}, [64]byte{}) digest := types.NewDigest() @@ -679,7 +679,7 @@ func TestService_HandleRuntimeChangesAfterCodeSubstitutes(t *testing.T) { ts, err = s.storageState.TrieState(nil) require.NoError(t, err) - ts.Put(common.CodeKey, testRuntime, trie.V0) + ts.Put(common.CodeKey, testRuntime) rtUpdateBhash := newBlock.Header.Hash() // update runtime for new block diff --git a/dot/import.go b/dot/import.go index 177ba3edda..c5d9c85f4e 100644 --- a/dot/import.go +++ b/dot/import.go @@ -20,8 +20,8 @@ import ( ) // ImportState imports the state in the given files to the database with the given path. -func ImportState(basepath, stateFP, headerFP string, firstSlot uint64, stateVersion trie.Version) error { - tr, err := newTrieFromPairs(stateFP, stateVersion) +func ImportState(basepath, stateFP, headerFP string, firstSlot uint64) error { + tr, err := newTrieFromPairs(stateFP) if err != nil { return err } @@ -41,7 +41,7 @@ func ImportState(basepath, stateFP, headerFP string, firstSlot uint64, stateVers return srv.Import(header, tr, firstSlot) } -func newTrieFromPairs(filename string, stateVersion trie.Version) (*trie.Trie, error) { +func newTrieFromPairs(filename string) (*trie.Trie, error) { data, err := os.ReadFile(filepath.Clean(filename)) if err != nil { return nil, err @@ -62,7 +62,7 @@ func newTrieFromPairs(filename string, stateVersion trie.Version) (*trie.Trie, e entries[pairArr[0].(string)] = pairArr[1].(string) } - tr, err := trie.LoadFromMap(entries, stateVersion) + tr, err := trie.LoadFromMap(entries) if err != nil { return nil, err } diff --git a/dot/import_integration_test.go b/dot/import_integration_test.go index e0e9714244..ca235afef6 100644 --- a/dot/import_integration_test.go +++ b/dot/import_integration_test.go @@ -13,7 +13,6 @@ import ( "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/internal/log" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/pkg/scale" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -54,7 +53,7 @@ func TestImportState_Integration(t *testing.T) { headerFP := setupHeaderFile(t) const firstSlot = uint64(262493679) - err = ImportState(config.BasePath, stateFP, headerFP, firstSlot, trie.V0) + err = ImportState(config.BasePath, stateFP, headerFP, firstSlot) require.NoError(t, err) // confirm data is imported into db stateConfig := state.Config{ @@ -87,7 +86,6 @@ func TestImportState(t *testing.T) { type args struct { basepath string stateFP string - version trie.Version headerFP string firstSlot uint64 } @@ -115,7 +113,7 @@ func TestImportState(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - err := ImportState(tt.args.basepath, tt.args.stateFP, tt.args.headerFP, tt.args.firstSlot, tt.args.version) + err := ImportState(tt.args.basepath, tt.args.stateFP, tt.args.headerFP, tt.args.firstSlot) if tt.err != nil { assert.EqualError(t, err, tt.err.Error()) } else { diff --git a/dot/import_test.go b/dot/import_test.go index 6229c97f9e..fa53b453ce 100644 --- a/dot/import_test.go +++ b/dot/import_test.go @@ -12,7 +12,6 @@ import ( "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -147,7 +146,6 @@ func Test_newTrieFromPairs(t *testing.T) { tests := []struct { name string filename string - version trie.Version want common.Hash err error }{ @@ -159,7 +157,6 @@ func Test_newTrieFromPairs(t *testing.T) { { name: "working example", filename: setupStateFile(t), - version: trie.V0, want: common.MustHexToHash("0x09f9ca28df0560c2291aa16b56e15e07d1e1927088f51356d522722aa90ca7cb"), }, } @@ -168,7 +165,7 @@ func Test_newTrieFromPairs(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - got, err := newTrieFromPairs(tt.filename, tt.version) + got, err := newTrieFromPairs(tt.filename) if tt.err != nil { assert.EqualError(t, err, tt.err.Error()) } else { diff --git a/dot/node_integration_test.go b/dot/node_integration_test.go index f62e1fdfe6..19e048b92e 100644 --- a/dot/node_integration_test.go +++ b/dot/node_integration_test.go @@ -385,7 +385,7 @@ func TestInitNode_LoadStorageRoot(t *testing.T) { node, err := NewNode(config, ks) require.NoError(t, err) - expected, err := trie.LoadFromMap(gen.GenesisFields().Raw["top"], trie.V0) + expected, err := trie.LoadFromMap(gen.GenesisFields().Raw["top"]) require.NoError(t, err) expectedRoot, err := expected.Hash() diff --git a/dot/rpc/modules/author_integration_test.go b/dot/rpc/modules/author_integration_test.go index f1da5fe3fe..9501e4288d 100644 --- a/dot/rpc/modules/author_integration_test.go +++ b/dot/rpc/modules/author_integration_test.go @@ -69,7 +69,7 @@ func useInstanceFromRuntimeV0929(t *testing.T, rtStorage *storage.TrieState) (in bytes, err := os.ReadFile(testRuntimeFilePath) require.NoError(t, err) - rtStorage.Put(common.CodeKey, bytes, trie.V0) + rtStorage.Put(common.CodeKey, bytes) cfg := wazero_runtime.Config{ Role: 0, diff --git a/dot/rpc/modules/childstate_integration_test.go b/dot/rpc/modules/childstate_integration_test.go index c7bc986adf..c9e7481051 100644 --- a/dot/rpc/modules/childstate_integration_test.go +++ b/dot/rpc/modules/childstate_integration_test.go @@ -244,15 +244,15 @@ func setupChildStateStorage(t *testing.T) (*ChildStateModule, common.Hash) { tr, err := st.Storage.TrieState(nil) require.NoError(t, err) - tr.Put([]byte(":first_key"), []byte(":value1"), trie.V0) - tr.Put([]byte(":second_key"), []byte(":second_value"), trie.V0) + tr.Put([]byte(":first_key"), []byte(":value1")) + tr.Put([]byte(":second_key"), []byte(":second_value")) childTr := trie.NewEmptyTrie() - childTr.Put([]byte(":child_first"), []byte(":child_first_value"), trie.V0) - childTr.Put([]byte(":child_second"), []byte(":child_second_value"), trie.V0) - childTr.Put([]byte(":another_child"), []byte("value"), trie.V0) + childTr.Put([]byte(":child_first"), []byte(":child_first_value")) + childTr.Put([]byte(":child_second"), []byte(":child_second_value")) + childTr.Put([]byte(":another_child"), []byte("value")) - err = tr.SetChild([]byte(":child_storage_key"), childTr, trie.V0) + err = tr.SetChild([]byte(":child_storage_key"), childTr) require.NoError(t, err) stateRoot, err := tr.Root() diff --git a/dot/rpc/modules/childstate_test.go b/dot/rpc/modules/childstate_test.go index 8cbae5cf8b..c84df630ce 100644 --- a/dot/rpc/modules/childstate_test.go +++ b/dot/rpc/modules/childstate_test.go @@ -24,15 +24,15 @@ func createTestTrieState(t *testing.T) (*trie.Trie, common.Hash) { _, genesisTrie, _ := newWestendLocalGenesisWithTrieAndHeader(t) tr := rtstorage.NewTrieState(&genesisTrie) - tr.Put([]byte(":first_key"), []byte(":value1"), trie.V0) - tr.Put([]byte(":second_key"), []byte(":second_value"), trie.V0) + tr.Put([]byte(":first_key"), []byte(":value1")) + tr.Put([]byte(":second_key"), []byte(":second_value")) childTr := trie.NewEmptyTrie() - childTr.Put([]byte(":child_first"), []byte(":child_first_value"), trie.V0) - childTr.Put([]byte(":child_second"), []byte(":child_second_value"), trie.V0) - childTr.Put([]byte(":another_child"), []byte("value"), trie.V0) + childTr.Put([]byte(":child_first"), []byte(":child_first_value")) + childTr.Put([]byte(":child_second"), []byte(":child_second_value")) + childTr.Put([]byte(":another_child"), []byte("value")) - err := tr.SetChild([]byte(":child_storage_key"), childTr, trie.V0) + err := tr.SetChild([]byte(":child_storage_key"), childTr) require.NoError(t, err) stateRoot, err := tr.Root() diff --git a/dot/rpc/modules/dev_integration_test.go b/dot/rpc/modules/dev_integration_test.go index ebda0e5773..c779dbca5b 100644 --- a/dot/rpc/modules/dev_integration_test.go +++ b/dot/rpc/modules/dev_integration_test.go @@ -68,8 +68,7 @@ func newBABEService(t *testing.T) *babe.Service { "0d4a9e054df4e01000000000000001cbd2d43530a44705ad088af313e18f80b53ef16b36177cd4b77b846f2a5f07c010000"+ "00000000004603307f855321776922daeea21ee31720388d097cdaac66f05a6f8462b317570100000000000000be1d9d59d"+ "e1283380100550a7b024501cb62d6cc40e3db35fcc5cf341814986e01000000000000001206960f920a23f7f4c43cc9081"+ - "ec2ed0721f31a9bef2c10fd7602e16e08a32c0100000000000000"), - trie.V0) + "ec2ed0721f31a9bef2c10fd7602e16e08a32c0100000000000000")) cfg := &babe.ServiceConfig{ BlockState: bs, diff --git a/dot/rpc/modules/state_integration_test.go b/dot/rpc/modules/state_integration_test.go index bbd0a5eef6..04cd189554 100644 --- a/dot/rpc/modules/state_integration_test.go +++ b/dot/rpc/modules/state_integration_test.go @@ -17,7 +17,6 @@ import ( "github.com/ChainSafe/gossamer/dot/rpc/modules/mocks" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -566,13 +565,13 @@ func setupStateModule(t *testing.T) (*StateModule, *common.Hash, *common.Hash) { ts, err := chain.Storage.TrieState(nil) require.NoError(t, err) - err = ts.Put([]byte(`:key2`), []byte(`value2`), trie.V0) + err = ts.Put([]byte(`:key2`), []byte(`value2`)) require.NoError(t, err) - err = ts.Put([]byte(`:key1`), []byte(`value1`), trie.V0) + err = ts.Put([]byte(`:key1`), []byte(`value1`)) require.NoError(t, err) - err = ts.SetChildStorage([]byte(`:child1`), []byte(`:key1`), []byte(`:childValue1`), trie.V0) + err = ts.SetChildStorage([]byte(`:child1`), []byte(`:key1`), []byte(`:childValue1`)) require.NoError(t, err) sr1, err := ts.Root() diff --git a/dot/rpc/modules/system_integration_test.go b/dot/rpc/modules/system_integration_test.go index b2a7dc745d..453ac8bb72 100644 --- a/dot/rpc/modules/system_integration_test.go +++ b/dot/rpc/modules/system_integration_test.go @@ -315,7 +315,7 @@ func setupSystemModule(t *testing.T) *SystemModule { aliceAcctEncoded, err := scale.Marshal(aliceAcctInfo) require.NoError(t, err) - ts.Put(aliceAcctStoKey, aliceAcctEncoded, trie.V0) + ts.Put(aliceAcctStoKey, aliceAcctEncoded) err = chain.Storage.StoreTrie(ts, nil) require.NoError(t, err) diff --git a/dot/state/base_test.go b/dot/state/base_test.go index 1286373e33..a6bedcde74 100644 --- a/dot/state/base_test.go +++ b/dot/state/base_test.go @@ -23,7 +23,7 @@ func TestTrie_StoreAndLoadFromDB(t *testing.T) { for keyString, value := range kv { key := []byte(keyString) - tt.Put(key, value, trie.V0) + tt.Put(key, value) } err := tt.WriteDirty(db) diff --git a/dot/state/service_integration_test.go b/dot/state/service_integration_test.go index 3d3317ed3c..d5b51186e7 100644 --- a/dot/state/service_integration_test.go +++ b/dot/state/service_integration_test.go @@ -390,7 +390,7 @@ func TestService_Import(t *testing.T) { "bnm", } for _, tc := range testCases { - tr.Put([]byte(tc), []byte(tc), trie.V0) + tr.Put([]byte(tc), []byte(tc)) } digest := types.NewDigest() @@ -437,7 +437,7 @@ func generateBlockWithRandomTrie(t *testing.T, serv *Service, rand := time.Now().UnixNano() key := []byte("testKey" + fmt.Sprint(rand)) value := []byte("testValue" + fmt.Sprint(rand)) - err = trieState.Put(key, value, trie.V0) + err = trieState.Put(key, value) require.NoError(t, err) trieStateRoot, err := trieState.Root() diff --git a/dot/state/storage_notify_test.go b/dot/state/storage_notify_test.go index 45965916ae..564458dc7c 100644 --- a/dot/state/storage_notify_test.go +++ b/dot/state/storage_notify_test.go @@ -9,7 +9,6 @@ import ( "time" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" ) @@ -45,7 +44,7 @@ func TestStorageState_RegisterStorageObserver(t *testing.T) { ss.RegisterStorageObserver(mockobs) defer ss.UnregisterStorageObserver(mockobs) - ts.Put([]byte("mackcom"), []byte("wuz here"), trie.V0) + ts.Put([]byte("mackcom"), []byte("wuz here")) err = ss.StoreTrie(ts, nil) require.NoError(t, err) @@ -82,7 +81,7 @@ func TestStorageState_RegisterStorageObserver_Multi(t *testing.T) { key1 := []byte("key1") value1 := []byte("value1") - ts.Put(key1, value1, trie.V0) + ts.Put(key1, value1) err = ss.StoreTrie(ts, nil) require.NoError(t, err) @@ -121,7 +120,7 @@ func TestStorageState_RegisterStorageObserver_Multi_Filter(t *testing.T) { ss.RegisterStorageObserver(mockobs) } - ts.Put(key1, value1, trie.V0) + ts.Put(key1, value1) err = ss.StoreTrie(ts, nil) require.NoError(t, err) diff --git a/dot/state/storage_test.go b/dot/state/storage_test.go index 6452de3c82..9f745a8891 100644 --- a/dot/state/storage_test.go +++ b/dot/state/storage_test.go @@ -57,7 +57,7 @@ func TestStorage_GetStorageByBlockHash(t *testing.T) { key := []byte("testkey") value := []byte("testvalue") - ts.Put(key, value, trie.V0) + ts.Put(key, value) root, err := ts.Root() require.NoError(t, err) @@ -89,7 +89,7 @@ func TestStorage_TrieState(t *testing.T) { storage := newTestStorageState(t) ts, err := storage.TrieState(&trie.EmptyHash) require.NoError(t, err) - ts.Put([]byte("noot"), []byte("washere"), trie.V0) + ts.Put([]byte("noot"), []byte("washere")) root, err := ts.Root() require.NoError(t, err) @@ -105,7 +105,7 @@ func TestStorage_TrieState(t *testing.T) { require.Equal(t, ts.Trie().MustHash(), ts3.Trie().MustHash()) } -func TestStorage_LoadFromDB_v0(t *testing.T) { +func TestStorage_LoadFromDB(t *testing.T) { storage := newTestStorageState(t) ts, err := storage.TrieState(&trie.EmptyHash) require.NoError(t, err) @@ -122,54 +122,7 @@ func TestStorage_LoadFromDB_v0(t *testing.T) { } for _, kv := range trieKV { - ts.Put(kv.key, kv.value, trie.V0) - } - - root, err := ts.Root() - require.NoError(t, err) - - // Write trie to disk. - err = storage.StoreTrie(ts, nil) - require.NoError(t, err) - - // Clear trie from cache and fetch data from disk. - storage.blockState.tries.delete(root) - - data, err := storage.GetStorage(&root, trieKV[0].key) - require.NoError(t, err) - require.Equal(t, trieKV[0].value, data) - - storage.blockState.tries.delete(root) - - prefixKeys, err := storage.GetKeysWithPrefix(&root, []byte("ke")) - require.NoError(t, err) - require.Equal(t, 2, len(prefixKeys)) - - storage.blockState.tries.delete(root) - - entries, err := storage.Entries(&root) - require.NoError(t, err) - require.Equal(t, 5, len(entries)) -} - -func TestStorage_LoadFromDB_v1(t *testing.T) { - storage := newTestStorageState(t) - ts, err := storage.TrieState(&trie.EmptyHash) - require.NoError(t, err) - - trieKV := []struct { - key []byte - value []byte - }{ - {value: []byte{}}, - {[]byte("key1"), []byte("value1")}, - {[]byte("key2"), []byte("value2")}, - {[]byte("xyzKey1"), []byte("xyzValue1")}, - {[]byte("long"), []byte("newvaluewithmorethan32byteslength")}, - } - - for _, kv := range trieKV { - ts.Put(kv.key, kv.value, trie.V1) + ts.Put(kv.key, kv.value) } root, err := ts.Root() @@ -206,7 +159,7 @@ func TestStorage_StoreTrie_NotSyncing(t *testing.T) { key := []byte("testkey") value := []byte("testvalue") - ts.Put(key, value, trie.V0) + ts.Put(key, value) err = storage.StoreTrie(ts, nil) require.NoError(t, err) @@ -237,9 +190,9 @@ func TestGetStorageChildAndGetStorageFromChild(t *testing.T) { } testChildTrie := trie.NewTrie(trieRoot, trieDB) - testChildTrie.Put([]byte("keyInsidechild"), []byte("voila"), trie.V0) + testChildTrie.Put([]byte("keyInsidechild"), []byte("voila")) - err = genTrie.SetChild([]byte("keyToChild"), testChildTrie, trie.V0) + err = genTrie.SetChild([]byte("keyToChild"), testChildTrie) require.NoError(t, err) tries := newTriesEmpty() diff --git a/dot/utils_integration_test.go b/dot/utils_integration_test.go index 2b08c3dc03..7f4e0a67a3 100644 --- a/dot/utils_integration_test.go +++ b/dot/utils_integration_test.go @@ -27,7 +27,7 @@ func TestTrieSnapshot(t *testing.T) { for k, v := range genRaw.Genesis.Raw["top"] { val := []byte(v) - tri.Put([]byte(k), val, trie.V0) + tri.Put([]byte(k), val) } deepCopyTrie := tri.DeepCopy() @@ -51,7 +51,7 @@ func TestTrieSnapshot(t *testing.T) { // Modify the current trie. value[0] = 'w' - newTrie.Put(key, value, trie.V0) + newTrie.Put(key, value) // Get the updated root hash of all tries. tHash, err = tri.Hash() diff --git a/lib/runtime/genesis.go b/lib/runtime/genesis.go index fbbe289d0b..54fc27f824 100644 --- a/lib/runtime/genesis.go +++ b/lib/runtime/genesis.go @@ -28,7 +28,7 @@ func NewTrieFromGenesis(gen genesis.Genesis) (tr trie.Trie, err error) { // TODO: I'll set it to V0 since our goal is to work on westend first but we have to revisit it in the future // to get the version from the runtime - tr, err = trie.LoadFromMap(keyValues, trie.V0) + tr, err = trie.LoadFromMap(keyValues) if err != nil { return tr, fmt.Errorf("loading genesis top key values into trie: %w", err) } diff --git a/lib/runtime/interfaces.go b/lib/runtime/interfaces.go index 7f5439dae6..2f813a21f6 100644 --- a/lib/runtime/interfaces.go +++ b/lib/runtime/interfaces.go @@ -11,17 +11,17 @@ import ( // Storage runtime interface. type Storage interface { - Put(key []byte, value []byte, version trie.Version) (err error) + Put(key []byte, value []byte) (err error) Get(key []byte) []byte Root() (common.Hash, error) - SetChild(keyToChild []byte, child *trie.Trie, version trie.Version) error - SetChildStorage(keyToChild, key, value []byte, version trie.Version) error + SetChild(keyToChild []byte, child *trie.Trie) error + SetChildStorage(keyToChild, key, value []byte) error GetChildStorage(keyToChild, key []byte) ([]byte, error) Delete(key []byte) (err error) DeleteChild(keyToChild []byte) (err error) DeleteChildLimit(keyToChild []byte, limit *[]byte) ( deleted uint32, allDeleted bool, err error) - ClearChildStorage(keyToChild, key []byte, version trie.Version) error + ClearChildStorage(keyToChild, key []byte) error NextKey([]byte) []byte ClearPrefixInChild(keyToChild, prefix []byte) error ClearPrefixInChildWithLimit(keyToChild, prefix []byte, limit uint32) (uint32, bool, error) diff --git a/lib/runtime/storage/trie.go b/lib/runtime/storage/trie.go index dcbf600309..b88e8fb56a 100644 --- a/lib/runtime/storage/trie.go +++ b/lib/runtime/storage/trie.go @@ -70,10 +70,10 @@ func (s *TrieState) RollbackStorageTransaction() { } // Put puts a key-value pair in the trie -func (s *TrieState) Put(key, value []byte, version trie.Version) (err error) { +func (s *TrieState) Put(key, value []byte) (err error) { s.lock.Lock() defer s.lock.Unlock() - return s.t.Put(key, value, version) + return s.t.Put(key, value) } // Get gets a value from the trie @@ -146,17 +146,17 @@ func (s *TrieState) TrieEntries() map[string][]byte { } // SetChild sets the child trie at the given key -func (s *TrieState) SetChild(keyToChild []byte, child *trie.Trie, version trie.Version) error { +func (s *TrieState) SetChild(keyToChild []byte, child *trie.Trie) error { s.lock.Lock() defer s.lock.Unlock() - return s.t.SetChild(keyToChild, child, version) + return s.t.SetChild(keyToChild, child) } // SetChildStorage sets a key-value pair in a child trie -func (s *TrieState) SetChildStorage(keyToChild, key, value []byte, version trie.Version) error { +func (s *TrieState) SetChildStorage(keyToChild, key, value []byte) error { s.lock.Lock() defer s.lock.Unlock() - return s.t.PutIntoChild(keyToChild, key, value, version) + return s.t.PutIntoChild(keyToChild, key, value) } // GetChild returns the child trie at the given key @@ -232,10 +232,10 @@ func (s *TrieState) DeleteChildLimit(key []byte, limit *[]byte) ( } // ClearChildStorage removes the child storage entry from the trie -func (s *TrieState) ClearChildStorage(keyToChild, key []byte, version trie.Version) error { +func (s *TrieState) ClearChildStorage(keyToChild, key []byte) error { s.lock.Lock() defer s.lock.Unlock() - return s.t.ClearFromChild(keyToChild, key, version) + return s.t.ClearFromChild(keyToChild, key) } // ClearPrefixInChild clears all the keys from the child trie that have the given prefix diff --git a/lib/runtime/storage/trie_test.go b/lib/runtime/storage/trie_test.go index b70710e2be..1e1075f550 100644 --- a/lib/runtime/storage/trie_test.go +++ b/lib/runtime/storage/trie_test.go @@ -27,7 +27,7 @@ var testCases = []string{ func TestTrieState_SetGet(t *testing.T) { testFunc := func(ts *TrieState) { for _, tc := range testCases { - ts.Put([]byte(tc), []byte(tc), trie.V0) + ts.Put([]byte(tc), []byte(tc)) } for _, tc := range testCases { @@ -45,10 +45,10 @@ func TestTrieState_SetGetChildStorage(t *testing.T) { for _, tc := range testCases { childTrie := trie.NewEmptyTrie() - err := ts.SetChild([]byte(tc), childTrie, trie.V0) + err := ts.SetChild([]byte(tc), childTrie) require.NoError(t, err) - err = ts.SetChildStorage([]byte(tc), []byte(tc), []byte(tc), trie.V0) + err = ts.SetChildStorage([]byte(tc), []byte(tc), []byte(tc)) require.NoError(t, err) } @@ -63,15 +63,15 @@ func TestTrieState_SetAndClearFromChild(t *testing.T) { testFunc := func(ts *TrieState) { for _, tc := range testCases { childTrie := trie.NewEmptyTrie() - err := ts.SetChild([]byte(tc), childTrie, trie.V0) + err := ts.SetChild([]byte(tc), childTrie) require.NoError(t, err) - err = ts.SetChildStorage([]byte(tc), []byte(tc), []byte(tc), trie.V0) + err = ts.SetChildStorage([]byte(tc), []byte(tc), []byte(tc)) require.NoError(t, err) } for _, tc := range testCases { - err := ts.ClearChildStorage([]byte(tc), []byte(tc), trie.V0) + err := ts.ClearChildStorage([]byte(tc), []byte(tc)) require.NoError(t, err) _, err = ts.GetChildStorage([]byte(tc), []byte(tc)) @@ -86,7 +86,7 @@ func TestTrieState_SetAndClearFromChild(t *testing.T) { func TestTrieState_Delete(t *testing.T) { testFunc := func(ts *TrieState) { for _, tc := range testCases { - ts.Put([]byte(tc), []byte(tc), trie.V0) + ts.Put([]byte(tc), []byte(tc)) } ts.Delete([]byte(testCases[0])) @@ -101,7 +101,7 @@ func TestTrieState_Delete(t *testing.T) { func TestTrieState_Root(t *testing.T) { testFunc := func(ts *TrieState) { for _, tc := range testCases { - ts.Put([]byte(tc), []byte(tc), trie.V0) + ts.Put([]byte(tc), []byte(tc)) } expected := ts.MustRoot() @@ -122,7 +122,7 @@ func TestTrieState_ClearPrefix(t *testing.T) { } for i, key := range keys { - ts.Put([]byte(key), []byte{byte(i)}, trie.V0) + ts.Put([]byte(key), []byte{byte(i)}) } ts.ClearPrefix([]byte("noo")) @@ -148,12 +148,12 @@ func TestTrieState_ClearPrefixInChild(t *testing.T) { } for i, key := range keys { - child.Put([]byte(key), []byte{byte(i)}, trie.V0) + child.Put([]byte(key), []byte{byte(i)}) } keyToChild := []byte("keytochild") - err := ts.SetChild(keyToChild, child, trie.V0) + err := ts.SetChild(keyToChild, child) require.NoError(t, err) err = ts.ClearPrefixInChild(keyToChild, []byte("noo")) @@ -174,7 +174,7 @@ func TestTrieState_NextKey(t *testing.T) { ts := &TrieState{t: trie.NewEmptyTrie()} for _, tc := range testCases { - ts.Put([]byte(tc), []byte(tc), trie.V0) + ts.Put([]byte(tc), []byte(tc)) } sort.Slice(testCases, func(i, j int) bool { @@ -195,12 +195,12 @@ func TestTrieState_CommitStorageTransaction(t *testing.T) { ts := &TrieState{t: trie.NewEmptyTrie()} for _, tc := range testCases { - ts.Put([]byte(tc), []byte(tc), trie.V0) + ts.Put([]byte(tc), []byte(tc)) } ts.BeginStorageTransaction() testValue := []byte("noot") - ts.Put([]byte(testCases[0]), testValue, trie.V0) + ts.Put([]byte(testCases[0]), testValue) ts.CommitStorageTransaction() val := ts.Get([]byte(testCases[0])) @@ -211,12 +211,12 @@ func TestTrieState_RollbackStorageTransaction(t *testing.T) { ts := &TrieState{t: trie.NewEmptyTrie()} for _, tc := range testCases { - ts.Put([]byte(tc), []byte(tc), trie.V0) + ts.Put([]byte(tc), []byte(tc)) } ts.BeginStorageTransaction() testValue := []byte("noot") - ts.Put([]byte(testCases[0]), testValue, trie.V0) + ts.Put([]byte(testCases[0]), testValue) ts.RollbackStorageTransaction() val := ts.Get([]byte(testCases[0])) @@ -234,12 +234,12 @@ func TestTrieState_DeleteChildLimit(t *testing.T) { } for i, key := range keys { - child.Put([]byte(key), []byte{byte(i)}, trie.V0) + child.Put([]byte(key), []byte{byte(i)}) } keyToChild := []byte("keytochild") - err := ts.SetChild(keyToChild, child, trie.V0) + err := ts.SetChild(keyToChild, child) require.NoError(t, err) testLimitBytes := make([]byte, 4) diff --git a/lib/runtime/wazero/imports.go b/lib/runtime/wazero/imports.go index ec9af1229d..5ee7400dd1 100644 --- a/lib/runtime/wazero/imports.go +++ b/lib/runtime/wazero/imports.go @@ -1058,7 +1058,7 @@ func ext_default_child_storage_set_version_1( cp := make([]byte, len(value)) copy(cp, value) - err := storage.SetChildStorage(childStorageKey, key, cp, trie.V0) + err := storage.SetChildStorage(childStorageKey, key, cp) if err != nil { logger.Errorf("failed to set value in child storage: %s", err) panic(err) @@ -1075,7 +1075,7 @@ func ext_default_child_storage_clear_version_1(ctx context.Context, m api.Module keyToChild := read(m, childStorageKey) key := read(m, keySpan) - err := storage.ClearChildStorage(keyToChild, key, trie.V0) + err := storage.ClearChildStorage(keyToChild, key) if err != nil { logger.Errorf("failed to clear child storage: %s", err) } @@ -1925,7 +1925,7 @@ func ext_offchain_http_request_add_header_version_1( return ptr } -func storageAppend(storage runtime.Storage, key, valueToAppend []byte, version trie.Version) (err error) { +func storageAppend(storage runtime.Storage, key, valueToAppend []byte) (err error) { // this function assumes the item in storage is a SCALE encoded array of items // the valueToAppend is a new item, so it appends the item and increases the length prefix by 1 currentValue := storage.Get(key) @@ -1976,7 +1976,7 @@ func storageAppend(storage runtime.Storage, key, valueToAppend []byte, version t } } - err = storage.Put(key, value, version) + err = storage.Put(key, value) if err != nil { return fmt.Errorf("putting key and value in storage: %w", err) } @@ -2000,7 +2000,7 @@ func ext_storage_append_version_1(ctx context.Context, m api.Module, keySpan, va cp := make([]byte, len(valueAppend)) copy(cp, valueAppend) - err := storageAppend(storage, key, cp, trie.V0) + err := storageAppend(storage, key, cp) if err != nil { logger.Errorf("failed appending to storage: %s", err) } @@ -2299,7 +2299,7 @@ func ext_storage_set_version_1(ctx context.Context, m api.Module, keySpan, value logger.Debugf( "key 0x%x has value 0x%x", key, value) - err := storage.Put(key, cp, trie.V0) + err := storage.Put(key, cp) if err != nil { panic(err) } diff --git a/lib/runtime/wazero/imports_test.go b/lib/runtime/wazero/imports_test.go index a370f0f1c1..703546cd70 100644 --- a/lib/runtime/wazero/imports_test.go +++ b/lib/runtime/wazero/imports_test.go @@ -564,8 +564,8 @@ func Test_ext_trie_blake2_256_root_version_1(t *testing.T) { require.NoError(t, err) tt := trie.NewEmptyTrie() - tt.Put([]byte("noot"), []byte("was"), trie.V0) - tt.Put([]byte("here"), []byte("??"), trie.V0) + tt.Put([]byte("noot"), []byte("was")) + tt.Put([]byte("here"), []byte("??")) expected := tt.MustHash() require.Equal(t, expected[:], hash) @@ -593,8 +593,8 @@ func Test_ext_trie_blake2_256_root_version_2(t *testing.T) { require.NoError(t, err) tt := trie.NewEmptyTrie() - tt.Put([]byte("dimartiro"), []byte("was"), trie.V1) - tt.Put([]byte("here"), []byte("??"), trie.V1) + tt.Put([]byte("dimartiro"), []byte("was")) + tt.Put([]byte("here"), []byte("??")) expected := tt.MustHash() require.Equal(t, expected[:], hash) @@ -648,17 +648,17 @@ func Test_ext_trie_blake2_256_verify_proof_version_1(t *testing.T) { require.NoError(t, err) otherTrie := trie.NewEmptyTrie() - otherTrie.Put([]byte("simple"), []byte("cat"), trie.V0) + otherTrie.Put([]byte("simple"), []byte("cat")) otherHash, err := otherTrie.Hash() require.NoError(t, err) tr := trie.NewEmptyTrie() - tr.Put([]byte("do"), []byte("verb"), trie.V0) - tr.Put([]byte("domain"), []byte("website"), trie.V0) - tr.Put([]byte("other"), []byte("random"), trie.V0) - tr.Put([]byte("otherwise"), []byte("randomstuff"), trie.V0) - tr.Put([]byte("cat"), []byte("another animal"), trie.V0) + tr.Put([]byte("do"), []byte("verb")) + tr.Put([]byte("domain"), []byte("website")) + tr.Put([]byte("other"), []byte("random")) + tr.Put([]byte("otherwise"), []byte("randomstuff")) + tr.Put([]byte("cat"), []byte("another animal")) err = tr.WriteDirty(memdb) require.NoError(t, err) @@ -740,17 +740,17 @@ func Test_ext_trie_blake2_256_verify_proof_version_2(t *testing.T) { require.NoError(t, err) otherTrie := trie.NewEmptyTrie() - otherTrie.Put([]byte("simple"), []byte("cat"), trie.V1) + otherTrie.Put([]byte("simple"), []byte("cat")) otherHash, err := otherTrie.Hash() require.NoError(t, err) tr := trie.NewEmptyTrie() - tr.Put([]byte("do"), []byte("verb"), trie.V1) - tr.Put([]byte("domain"), []byte("website"), trie.V1) - tr.Put([]byte("other"), []byte("random"), trie.V1) - tr.Put([]byte("otherwise"), []byte("randomstuff"), trie.V1) - tr.Put([]byte("cat"), []byte("another animal"), trie.V1) + tr.Put([]byte("do"), []byte("verb")) + tr.Put([]byte("domain"), []byte("website")) + tr.Put([]byte("other"), []byte("random")) + tr.Put([]byte("otherwise"), []byte("randomstuff")) + tr.Put([]byte("cat"), []byte("another animal")) err = tr.WriteDirty(memdb) require.NoError(t, err) @@ -913,10 +913,10 @@ func Test_ext_default_child_storage_read_version_1(t *testing.T) { setupInstance: func(t *testing.T) *Instance { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie(), trie.V0) + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) require.NoError(t, err) - err = inst.Context.Storage.SetChildStorage(testChildKey, testKey, testValue, trie.V0) + err = inst.Context.Storage.SetChildStorage(testChildKey, testKey, testValue) require.NoError(t, err) return inst }, @@ -974,7 +974,7 @@ func Test_ext_default_child_storage_set_version_1(t *testing.T) { setupInstance: func(t *testing.T) *Instance { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie(), trie.V0) + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) require.NoError(t, err) return inst @@ -1053,10 +1053,10 @@ func Test_ext_default_child_storage_set_version_1(t *testing.T) { func Test_ext_default_child_storage_clear_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie(), trie.V0) + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) require.NoError(t, err) - err = inst.Context.Storage.SetChildStorage(testChildKey, testKey, testValue, trie.V0) + err = inst.Context.Storage.SetChildStorage(testChildKey, testKey, testValue) require.NoError(t, err) // Confirm if value is set @@ -1093,11 +1093,11 @@ func Test_ext_default_child_storage_clear_prefix_version_1(t *testing.T) { {[]byte("keyThree"), []byte("value3")}, } - err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie(), trie.V0) + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) require.NoError(t, err) for _, kv := range testKeyValuePair { - err = inst.Context.Storage.SetChildStorage(testChildKey, kv.key, kv.value, trie.V0) + err = inst.Context.Storage.SetChildStorage(testChildKey, kv.key, kv.value) require.NoError(t, err) } @@ -1123,10 +1123,10 @@ func Test_ext_default_child_storage_clear_prefix_version_1(t *testing.T) { func Test_ext_default_child_storage_exists_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie(), trie.V0) + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) require.NoError(t, err) - err = inst.Context.Storage.SetChildStorage(testChildKey, testKey, testValue, trie.V0) + err = inst.Context.Storage.SetChildStorage(testChildKey, testKey, testValue) require.NoError(t, err) encChildKey, err := scale.Marshal(testChildKey) @@ -1152,10 +1152,10 @@ func Test_ext_default_child_storage_get_version_1(t *testing.T) { "value_exists_expected_value": { setupInstance: func(t *testing.T) *Instance { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie(), trie.V0) + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) require.NoError(t, err) - err = inst.Context.Storage.SetChildStorage(testChildKey, testKey, testValue, trie.V0) + err = inst.Context.Storage.SetChildStorage(testChildKey, testKey, testValue) require.NoError(t, err) return inst }, @@ -1210,11 +1210,11 @@ func Test_ext_default_child_storage_next_key_version_1(t *testing.T) { setupInstance: func(t *testing.T) *Instance { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie(), trie.V0) + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) require.NoError(t, err) for _, kv := range testKeyValuePair { - err = inst.Context.Storage.SetChildStorage(testChildKey, kv.key, kv.value, trie.V0) + err = inst.Context.Storage.SetChildStorage(testChildKey, kv.key, kv.value) require.NoError(t, err) } @@ -1232,11 +1232,11 @@ func Test_ext_default_child_storage_next_key_version_1(t *testing.T) { setupInstance: func(t *testing.T) *Instance { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie(), trie.V0) + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) require.NoError(t, err) kv := testKeyValuePair[0] - err = inst.Context.Storage.SetChildStorage(testChildKey, kv.key, kv.value, trie.V0) + err = inst.Context.Storage.SetChildStorage(testChildKey, kv.key, kv.value) require.NoError(t, err) return inst @@ -1274,10 +1274,10 @@ func Test_ext_default_child_storage_next_key_version_1(t *testing.T) { func Test_ext_default_child_storage_root_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie(), trie.V0) + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) require.NoError(t, err) - err = inst.Context.Storage.SetChildStorage(testChildKey, testKey, testValue, trie.V0) + err = inst.Context.Storage.SetChildStorage(testChildKey, testKey, testValue) require.NoError(t, err) child, err := inst.Context.Storage.GetChild(testChildKey) @@ -1306,10 +1306,10 @@ func Test_ext_default_child_storage_root_version_1(t *testing.T) { func Test_ext_default_child_storage_root_version_2(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie(), trie.V1) + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) require.NoError(t, err) - err = inst.Context.Storage.SetChildStorage(testChildKey, testKey, testValue, trie.V1) + err = inst.Context.Storage.SetChildStorage(testChildKey, testKey, testValue) require.NoError(t, err) child, err := inst.Context.Storage.GetChild(testChildKey) @@ -1345,7 +1345,7 @@ func Test_ext_default_child_storage_root_version_2(t *testing.T) { func Test_ext_default_child_storage_storage_kill_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) - err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie(), trie.V0) + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) require.NoError(t, err) // Confirm if value is set @@ -1367,9 +1367,9 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_all(t *testing. inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) - tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) - err := inst.Context.Storage.SetChild(testChildKey, tr, trie.V0) + tr.Put([]byte(`key2`), []byte(`value2`)) + tr.Put([]byte(`key1`), []byte(`value1`)) + err := inst.Context.Storage.SetChild(testChildKey, tr) require.NoError(t, err) // Confirm if value is set @@ -1400,9 +1400,9 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_1(t *testing.T) inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) - tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) - err := inst.Context.Storage.SetChild(testChildKey, tr, trie.V0) + tr.Put([]byte(`key2`), []byte(`value2`)) + tr.Put([]byte(`key1`), []byte(`value1`)) + err := inst.Context.Storage.SetChild(testChildKey, tr) require.NoError(t, err) // Confirm if value is set @@ -1433,9 +1433,9 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_none(t *testing inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) - tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) - err := inst.Context.Storage.SetChild(testChildKey, tr, trie.V0) + tr.Put([]byte(`key2`), []byte(`value2`)) + tr.Put([]byte(`key1`), []byte(`value1`)) + err := inst.Context.Storage.SetChild(testChildKey, tr) require.NoError(t, err) // Confirm if value is set @@ -1463,10 +1463,10 @@ func Test_ext_default_child_storage_storage_kill_version_3(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() - tr.Put([]byte(`key2`), []byte(`value2`), trie.V0) - tr.Put([]byte(`key1`), []byte(`value1`), trie.V0) - tr.Put([]byte(`key3`), []byte(`value3`), trie.V0) - err := inst.Context.Storage.SetChild(testChildKey, tr, trie.V0) + tr.Put([]byte(`key2`), []byte(`value2`)) + tr.Put([]byte(`key1`), []byte(`value1`)) + tr.Put([]byte(`key3`), []byte(`value3`)) + err := inst.Context.Storage.SetChild(testChildKey, tr) require.NoError(t, err) testLimitBytes := make([]byte, 4) @@ -1664,11 +1664,11 @@ func Test_ext_default_child_storage_clear_prefix_version_2(t *testing.T) { {[]byte("keyThree"), []byte("value3")}, } - err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie(), trie.V0) + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) require.NoError(t, err) for _, kv := range testKeyValuePair { - err = inst.Context.Storage.SetChildStorage(testChildKey, kv.key, kv.value, trie.V0) + err = inst.Context.Storage.SetChildStorage(testChildKey, kv.key, kv.value) require.NoError(t, err) } @@ -1868,7 +1868,7 @@ func Test_ext_storage_clear_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") - inst.Context.Storage.Put(testkey, []byte{1}, trie.V0) + inst.Context.Storage.Put(testkey, []byte{1}) enc, err := scale.Marshal(testkey) require.NoError(t, err) @@ -1884,10 +1884,10 @@ func Test_ext_storage_clear_prefix_version_1_hostAPI(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("static") - inst.Context.Storage.Put(testkey, []byte("Inverse"), trie.V0) + inst.Context.Storage.Put(testkey, []byte("Inverse")) testkey2 := []byte("even-keeled") - inst.Context.Storage.Put(testkey2, []byte("Future-proofed"), trie.V0) + inst.Context.Storage.Put(testkey2, []byte("Future-proofed")) enc, err := scale.Marshal(testkey[:3]) require.NoError(t, err) @@ -1906,10 +1906,10 @@ func Test_ext_storage_clear_prefix_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") - inst.Context.Storage.Put(testkey, []byte{1}, trie.V0) + inst.Context.Storage.Put(testkey, []byte{1}) testkey2 := []byte("spaghet") - inst.Context.Storage.Put(testkey2, []byte{2}, trie.V0) + inst.Context.Storage.Put(testkey2, []byte{2}) enc, err := scale.Marshal(testkey[:3]) require.NoError(t, err) @@ -1928,20 +1928,20 @@ func Test_ext_storage_clear_prefix_version_2(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") - inst.Context.Storage.Put(testkey, []byte{1}, trie.V0) + inst.Context.Storage.Put(testkey, []byte{1}) testkey2 := []byte("noot1") - inst.Context.Storage.Put(testkey2, []byte{1}, trie.V0) + inst.Context.Storage.Put(testkey2, []byte{1}) testkey3 := []byte("noot2") - inst.Context.Storage.Put(testkey3, []byte{1}, trie.V0) + inst.Context.Storage.Put(testkey3, []byte{1}) testkey4 := []byte("noot3") - inst.Context.Storage.Put(testkey4, []byte{1}, trie.V0) + inst.Context.Storage.Put(testkey4, []byte{1}) testkey5 := []byte("spaghet") testValue5 := []byte{2} - inst.Context.Storage.Put(testkey5, testValue5, trie.V0) + inst.Context.Storage.Put(testkey5, testValue5) enc, err := scale.Marshal(testkey[:3]) require.NoError(t, err) @@ -2001,7 +2001,7 @@ func Test_ext_storage_get_version_1(t *testing.T) { testkey := []byte("noot") testvalue := []byte{1, 2} - inst.Context.Storage.Put(testkey, testvalue, trie.V0) + inst.Context.Storage.Put(testkey, testvalue) enc, err := scale.Marshal(testkey) require.NoError(t, err) @@ -2044,7 +2044,7 @@ func Test_ext_storage_exists_version_1(t *testing.T) { instance := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) if testCase.value != nil { - instance.Context.Storage.Put(testCase.key, testCase.value, trie.V0) + instance.Context.Storage.Put(testCase.key, testCase.value) } encodedKey, err := scale.Marshal(testCase.key) @@ -2066,10 +2066,10 @@ func Test_ext_storage_next_key_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") - inst.Context.Storage.Put(testkey, []byte{1}, trie.V0) + inst.Context.Storage.Put(testkey, []byte{1}) nextkey := []byte("oot") - inst.Context.Storage.Put(nextkey, []byte{1}, trie.V0) + inst.Context.Storage.Put(nextkey, []byte{1}) enc, err := scale.Marshal(testkey) require.NoError(t, err) @@ -2089,7 +2089,7 @@ func Test_ext_storage_read_version_1(t *testing.T) { testkey := []byte("noot") testvalue := []byte("washere") - inst.Context.Storage.Put(testkey, testvalue, trie.V0) + inst.Context.Storage.Put(testkey, testvalue) testoffset := uint32(2) testBufferSize := uint32(100) @@ -2117,7 +2117,7 @@ func Test_ext_storage_read_version_1_again(t *testing.T) { testkey := []byte("noot") testvalue := []byte("_was_here_") - inst.Context.Storage.Put(testkey, testvalue, trie.V0) + inst.Context.Storage.Put(testkey, testvalue) testoffset := uint32(8) testBufferSize := uint32(5) @@ -2146,7 +2146,7 @@ func Test_ext_storage_read_version_1_OffsetLargerThanValue(t *testing.T) { testkey := []byte("noot") testvalue := []byte("washere") - inst.Context.Storage.Put(testkey, testvalue, trie.V0) + inst.Context.Storage.Put(testkey, testvalue) testoffset := uint32(len(testvalue)) testBufferSize := uint32(8) diff --git a/lib/runtime/wazero/instance_test.go b/lib/runtime/wazero/instance_test.go index 39f3f41d9d..511dfc3ed8 100644 --- a/lib/runtime/wazero/instance_test.go +++ b/lib/runtime/wazero/instance_test.go @@ -224,9 +224,9 @@ func TestWestendRuntime_ValidateTransaction(t *testing.T) { encBal, err := scale.Marshal(accInfo) require.NoError(t, err) - rt.Context.Storage.Put(aliceBalanceKey, encBal, trie.V0) + rt.Context.Storage.Put(aliceBalanceKey, encBal) // this key is System.UpgradedToDualRefCount -> set to true since all accounts have been upgraded to v0.9 format - rt.Context.Storage.Put(common.UpgradedToDualRefKey, []byte{1}, trie.V0) + rt.Context.Storage.Put(common.UpgradedToDualRefKey, []byte{1}) genesisHeader := &types.Header{ Number: 0, @@ -257,7 +257,7 @@ func TestInstance_GrandpaAuthorities_NodeRuntime(t *testing.T) { require.NoError(t, err) key := common.MustHexToBytes(genesis.GrandpaAuthoritiesKeyHex) - tt.Put(key, value, trie.V0) + tt.Put(key, value) rt := NewTestInstanceWithTrie(t, runtime.WESTEND_RUNTIME_v0929, tt) @@ -285,7 +285,7 @@ func TestInstance_GrandpaAuthorities_PolkadotRuntime(t *testing.T) { require.NoError(t, err) key := common.MustHexToBytes(genesis.GrandpaAuthoritiesKeyHex) - tt.Put(key, value, trie.V0) + tt.Put(key, value) rt := NewTestInstanceWithTrie(t, runtime.POLKADOT_RUNTIME_v0929, tt) @@ -328,13 +328,13 @@ func TestInstance_BabeGenerateKeyOwnershipProof(t *testing.T) { randomnessValue, err := common.HexToHash("0x01") require.NoError(t, err) key := common.MustHexToBytes(genesis.BABERandomnessKeyHex) - tt.Put(key, randomnessValue[:], trie.V0) + tt.Put(key, randomnessValue[:]) authorityValue, err := common.HexToBytes("0x08eea1eabcac7d2c8a6459b7322cf997874482bfc3d2ec7a80888a3a7d714103640100000000000000b64994460e59b30364cad3c92e3df6052f9b0ebbb8f88460c194dc5794d6d7170100000000000000") //nolint:lll require.NoError(t, err) key = common.MustHexToBytes(genesis.BABEAuthoritiesKeyHex) - tt.Put(key, authorityValue, trie.V0) + tt.Put(key, authorityValue) rt := NewTestInstanceWithTrie(t, testCase.targetRuntime, tt) @@ -541,13 +541,13 @@ func TestInstance_BabeConfiguration_WestendRuntime_WithAuthorities(t *testing.T) randomnessValue, err := common.HexToHash("0x01") require.NoError(t, err) key := common.MustHexToBytes(genesis.BABERandomnessKeyHex) - tt.Put(key, randomnessValue[:], trie.V0) + tt.Put(key, randomnessValue[:]) authorityValue, err := common.HexToBytes("0x08eea1eabcac7d2c8a6459b7322cf997874482bfc3d2ec7a80888a3a7d714103640100000000000000b64994460e59b30364cad3c92e3df6052f9b0ebbb8f88460c194dc5794d6d7170100000000000000") //nolint:lll require.NoError(t, err) key = common.MustHexToBytes(genesis.BABEAuthoritiesKeyHex) - tt.Put(key, authorityValue, trie.V0) + tt.Put(key, authorityValue) rt := NewTestInstanceWithTrie(t, runtime.WESTEND_RUNTIME_v0929, tt) @@ -1094,7 +1094,7 @@ func newTrieFromPairs(t *testing.T, filename string) *trie.Trie { entries[pairArr[0].(string)] = pairArr[1].(string) } - tr, err := trie.LoadFromMap(entries, trie.V0) + tr, err := trie.LoadFromMap(entries) require.NoError(t, err) return &tr } diff --git a/lib/trie/child_storage.go b/lib/trie/child_storage.go index 4ca063ed7f..00dd28d20b 100644 --- a/lib/trie/child_storage.go +++ b/lib/trie/child_storage.go @@ -18,7 +18,7 @@ var ErrChildTrieDoesNotExist = errors.New("child trie does not exist") // SetChild inserts a child trie into the main trie at key :child_storage:[keyToChild] // A child trie is added as a node (K, V) in the main trie. K is the child storage key // associated to the child trie, and V is the root hash of the child trie. -func (t *Trie) SetChild(keyToChild []byte, child *Trie, version Version) error { +func (t *Trie) SetChild(keyToChild []byte, child *Trie) error { childHash, err := child.Hash() if err != nil { return err @@ -28,7 +28,7 @@ func (t *Trie) SetChild(keyToChild []byte, child *Trie, version Version) error { copy(key, ChildStorageKeyPrefix) copy(key[len(ChildStorageKeyPrefix):], keyToChild) - err = t.Put(key, childHash.ToBytes(), version) + err = t.Put(key, childHash.ToBytes()) if err != nil { return fmt.Errorf("putting child trie root hash %s in trie: %w", childHash, err) } @@ -52,7 +52,7 @@ func (t *Trie) GetChild(keyToChild []byte) (*Trie, error) { } // PutIntoChild puts a key-value pair into the child trie located in the main trie at key :child_storage:[keyToChild] -func (t *Trie) PutIntoChild(keyToChild, key, value []byte, version Version) error { +func (t *Trie) PutIntoChild(keyToChild, key, value []byte) error { child, err := t.GetChild(keyToChild) if err != nil { if errors.Is(err, ErrChildTrieDoesNotExist) { @@ -67,13 +67,13 @@ func (t *Trie) PutIntoChild(keyToChild, key, value []byte, version Version) erro return err } - err = child.Put(key, value, version) + err = child.Put(key, value) if err != nil { return fmt.Errorf("putting into child trie located at key 0x%x: %w", keyToChild, err) } delete(t.childTries, origChildHash) - return t.SetChild(keyToChild, child, version) + return t.SetChild(keyToChild, child) } // GetFromChild retrieves a key-value pair from the child trie located @@ -106,7 +106,7 @@ func (t *Trie) DeleteChild(keyToChild []byte) (err error) { } // ClearFromChild removes the child storage entry -func (t *Trie) ClearFromChild(keyToChild, key []byte, version Version) error { +func (t *Trie) ClearFromChild(keyToChild, key []byte) error { child, err := t.GetChild(keyToChild) if err != nil { return err @@ -131,5 +131,5 @@ func (t *Trie) ClearFromChild(keyToChild, key []byte, version Version) error { return t.DeleteChild(keyToChild) } - return t.SetChild(keyToChild, child, version) + return t.SetChild(keyToChild, child) } diff --git a/lib/trie/child_storage_test.go b/lib/trie/child_storage_test.go index 3c84279921..c77d8f7a58 100644 --- a/lib/trie/child_storage_test.go +++ b/lib/trie/child_storage_test.go @@ -16,7 +16,7 @@ func TestPutAndGetChild(t *testing.T) { childTrie := buildSmallTrie() parentTrie := NewEmptyTrie() - err := parentTrie.SetChild(childKey, childTrie, V0) + err := parentTrie.SetChild(childKey, childTrie) assert.NoError(t, err) childTrieRes, err := parentTrie.GetChild(childKey) @@ -30,7 +30,7 @@ func TestPutAndDeleteChild(t *testing.T) { childTrie := buildSmallTrie() parentTrie := NewEmptyTrie() - err := parentTrie.SetChild(childKey, childTrie, V0) + err := parentTrie.SetChild(childKey, childTrie) assert.NoError(t, err) err = parentTrie.DeleteChild(childKey) @@ -46,10 +46,10 @@ func TestPutAndClearFromChild(t *testing.T) { childTrie := buildSmallTrie() parentTrie := NewEmptyTrie() - err := parentTrie.SetChild(childKey, childTrie, V0) + err := parentTrie.SetChild(childKey, childTrie) assert.NoError(t, err) - err = parentTrie.ClearFromChild(childKey, keyInChild, V0) + err = parentTrie.ClearFromChild(childKey, keyInChild) assert.NoError(t, err) childTrie, err = parentTrie.GetChild(childKey) @@ -64,12 +64,12 @@ func TestPutAndGetFromChild(t *testing.T) { childTrie := buildSmallTrie() parentTrie := NewEmptyTrie() - err := parentTrie.SetChild(childKey, childTrie, V0) + err := parentTrie.SetChild(childKey, childTrie) assert.NoError(t, err) testKey := []byte("child_key") testValue := []byte("child_value") - err = parentTrie.PutIntoChild(childKey, testKey, testValue, V0) + err = parentTrie.PutIntoChild(childKey, testKey, testValue) assert.NoError(t, err) valueRes, err := parentTrie.GetFromChild(childKey, testKey) @@ -79,7 +79,7 @@ func TestPutAndGetFromChild(t *testing.T) { testKey = []byte("child_key_again") testValue = []byte("child_value_again") - err = parentTrie.PutIntoChild(childKey, testKey, testValue, V0) + err = parentTrie.PutIntoChild(childKey, testKey, testValue) assert.NoError(t, err) valueRes, err = parentTrie.GetFromChild(childKey, testKey) @@ -98,7 +98,7 @@ func TestChildTrieHashAfterClear(t *testing.T) { contributedWith := make([]byte, 8) binary.BigEndian.PutUint64(contributedWith, contributed) - err := trieThatHoldsAChildTrie.PutIntoChild(keyToChild, keyInChild, contributedWith, V0) + err := trieThatHoldsAChildTrie.PutIntoChild(keyToChild, keyInChild, contributedWith) require.NoError(t, err) // the parent trie hash SHOULT NOT BE EQUAL to the original @@ -111,7 +111,7 @@ func TestChildTrieHashAfterClear(t *testing.T) { require.Equal(t, contributed, binary.BigEndian.Uint64(valueStored)) // clear child trie key value - err = trieThatHoldsAChildTrie.ClearFromChild(keyToChild, keyInChild, V0) + err = trieThatHoldsAChildTrie.ClearFromChild(keyToChild, keyInChild) require.NoError(t, err) // the parent trie hash SHOULD BE EQUAL to the original diff --git a/lib/trie/database_test.go b/lib/trie/database_test.go index cb458826e1..2a43b5826d 100644 --- a/lib/trie/database_test.go +++ b/lib/trie/database_test.go @@ -60,7 +60,7 @@ func Test_Trie_WriteDirty_Put(t *testing.T) { for keyString, value := range keyValues { key := []byte(keyString) - trie.Put(key, value, V0) + trie.Put(key, value) err := trie.WriteDirty(db) require.NoError(t, err) @@ -81,7 +81,7 @@ func Test_Trie_WriteDirty_Put(t *testing.T) { newValue := make([]byte, len(existingValue)) copy(newValue, existingValue) newValue = append(newValue, 99) - trie.Put(existingKey, newValue, V0) + trie.Put(existingKey, newValue) err = trie.WriteDirty(db) require.NoError(t, err) @@ -318,7 +318,7 @@ func Test_Trie_PutChild_Store_Load(t *testing.T) { } for _, keyToChildTrie := range keysToChildTries { - err := trie.SetChild(keyToChildTrie, childTrie, V0) + err := trie.SetChild(keyToChildTrie, childTrie) require.NoError(t, err) err = trie.WriteDirty(db) diff --git a/lib/trie/helpers_test.go b/lib/trie/helpers_test.go index b36385e098..cbae542394 100644 --- a/lib/trie/helpers_test.go +++ b/lib/trie/helpers_test.go @@ -93,7 +93,7 @@ func makeSeededTrie(t *testing.T, size int) ( for keyString, value := range keyValues { key := []byte(keyString) - trie.Put(key, value, V0) + trie.Put(key, value) } return trie, keyValues diff --git a/lib/trie/mem_test.go b/lib/trie/mem_test.go index 757da3185f..56096571bd 100644 --- a/lib/trie/mem_test.go +++ b/lib/trie/mem_test.go @@ -93,7 +93,7 @@ func populateTrieAtPrefix(trie *Trie, for keyString, value := range kv { key := append(prefix, []byte(keyString)...) //skipcq: CRT-D0001 - trie.Put(key, value, V0) + trie.Put(key, value) } } @@ -113,6 +113,6 @@ func mutateTrieLeavesAtPrefix(trie *Trie, } } - trie.Put(key, newValue, V0) + trie.Put(key, newValue) } } diff --git a/lib/trie/proof/generate_test.go b/lib/trie/proof/generate_test.go index 6517ed8bd0..49f519b3c0 100644 --- a/lib/trie/proof/generate_test.go +++ b/lib/trie/proof/generate_test.go @@ -828,7 +828,6 @@ func Test_lenCommonPrefix(t *testing.T) { // so the code is kept to this inefficient-looking append, // which is in the end quite performant still. func Benchmark_walkRoot(b *testing.B) { - stateVersion := trie.V0 trie := trie.NewEmptyTrie() // Build a deep trie. @@ -839,7 +838,7 @@ func Benchmark_walkRoot(b *testing.B) { const trieValueSize = 10 value := make([]byte, trieValueSize) - trie.Put(key, value, stateVersion) + trie.Put(key, value) } longestKeyLE := make([]byte, trieDepth) diff --git a/lib/trie/proof/proof_test.go b/lib/trie/proof/proof_test.go index 2e0cf2731a..954c48cb7f 100644 --- a/lib/trie/proof/proof_test.go +++ b/lib/trie/proof/proof_test.go @@ -16,7 +16,6 @@ import ( func Test_Generate_Verify(t *testing.T) { t.Parallel() - stateVersion := trie.V0 keys := []string{ "cat", @@ -30,7 +29,7 @@ func Test_Generate_Verify(t *testing.T) { for i, key := range keys { value := fmt.Sprintf("%x-%d", key, i) - trie.Put([]byte(key), []byte(value), stateVersion) + trie.Put([]byte(key), []byte(value)) } rootHash, err := trie.Hash() diff --git a/lib/trie/trie.go b/lib/trie/trie.go index fc5b28fa89..c0d77c7d20 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -353,17 +353,17 @@ func findNextKeyChild(children []*Node, startIndex byte, // Put inserts a value into the trie at the // key specified in little Endian format. -func (t *Trie) Put(keyLE, value []byte, version Version) (err error) { +func (t *Trie) Put(keyLE, value []byte) (err error) { pendingDeltas := tracking.New() defer func() { const success = true t.handleTrackedDeltas(success, pendingDeltas) }() - return t.insertKeyLE(keyLE, value, pendingDeltas, version) + return t.insertKeyLE(keyLE, value, pendingDeltas) } func (t *Trie) insertKeyLE(keyLE, value []byte, - pendingDeltas DeltaRecorder, version Version) (err error) { + pendingDeltas DeltaRecorder) (err error) { nibblesKey := codec.KeyLEToNibbles(keyLE) if value == nil { // Force nil value to be inserted to []byte{} since `nil` means there @@ -371,20 +371,7 @@ func (t *Trie) insertKeyLE(keyLE, value []byte, value = []byte{} } - shouldHash := version.ShouldHashValue(value) - if shouldHash { - hashedValue := common.MustBlake2bHash(value) - valueKey := hashedValue.ToBytes() - // Add the original value as value node in db using the hashed value as key - err = t.db.Put(valueKey, value) - if err != nil { - return err - } - // Put the value key as this trie node value - value = valueKey - } - - root, _, _, err := t.insert(t.root, nibblesKey, value, shouldHash, pendingDeltas) + root, _, _, err := t.insert(t.root, nibblesKey, value, pendingDeltas) if err != nil { return err } @@ -394,24 +381,22 @@ func (t *Trie) insertKeyLE(keyLE, value []byte, // insert inserts a value in the trie at the key specified. // It may create one or more new nodes or update an existing node. -func (t *Trie) insert(parent *Node, key, value []byte, - isValueHashed bool, pendingDeltas DeltaRecorder) (newParent *Node, +func (t *Trie) insert(parent *Node, key, value []byte, pendingDeltas DeltaRecorder) (newParent *Node, mutated bool, nodesCreated uint32, err error) { if parent == nil { mutated = true nodesCreated = 1 return &Node{ - PartialKey: key, - StorageValue: value, - IsHashedValue: isValueHashed, - Generation: t.generation, - Dirty: true, + PartialKey: key, + StorageValue: value, + Generation: t.generation, + Dirty: true, }, mutated, nodesCreated, nil } if parent.Kind() == node.Branch { newParent, mutated, nodesCreated, err = t.insertInBranch( - parent, key, value, isValueHashed, pendingDeltas) + parent, key, value, pendingDeltas) if err != nil { // `insertInBranch` may call `insert` so do not wrap the // error since this may be a deep recursive call. @@ -421,7 +406,7 @@ func (t *Trie) insert(parent *Node, key, value []byte, } newParent, mutated, nodesCreated, err = t.insertInLeaf( - parent, key, value, isValueHashed, pendingDeltas) + parent, key, value, pendingDeltas) if err != nil { return nil, false, 0, fmt.Errorf("inserting in leaf: %w", err) } @@ -429,7 +414,7 @@ func (t *Trie) insert(parent *Node, key, value []byte, return newParent, mutated, nodesCreated, nil } -func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, isValueHashed bool, +func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, pendingDeltas DeltaRecorder) ( newParent *Node, mutated bool, nodesCreated uint32, err error) { if bytes.Equal(parentLeaf.PartialKey, key) { @@ -447,7 +432,6 @@ func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, isValueHashed b } parentLeaf.StorageValue = value - parentLeaf.IsHashedValue = isValueHashed mutated = true return parentLeaf, mutated, nodesCreated, nil } @@ -467,7 +451,6 @@ func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, isValueHashed b if len(key) == commonPrefixLength { // key is included in parent leaf key newBranchParent.StorageValue = value - newBranchParent.IsHashedValue = isValueHashed if len(key) < len(parentLeafKey) { // Move the current leaf parent as a child to the new branch. @@ -511,11 +494,10 @@ func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, isValueHashed b } childIndex := key[commonPrefixLength] newBranchParent.Children[childIndex] = &Node{ - PartialKey: key[commonPrefixLength+1:], - StorageValue: value, - IsHashedValue: isValueHashed, - Generation: t.generation, - Dirty: true, + PartialKey: key[commonPrefixLength+1:], + StorageValue: value, + Generation: t.generation, + Dirty: true, } newBranchParent.Descendants++ nodesCreated++ @@ -523,7 +505,7 @@ func (t *Trie) insertInLeaf(parentLeaf *Node, key, value []byte, isValueHashed b return newBranchParent, mutated, nodesCreated, nil } -func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, isValueHashed bool, +func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, pendingDeltas DeltaRecorder) ( newParent *Node, mutated bool, nodesCreated uint32, err error) { copySettings := node.DefaultCopySettings @@ -538,7 +520,6 @@ func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, isValueHash return nil, false, 0, fmt.Errorf("preparing branch for mutation: %w", err) } parentBranch.StorageValue = value - parentBranch.IsHashedValue = isValueHashed mutated = true return parentBranch, mutated, 0, nil } @@ -552,11 +533,10 @@ func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, isValueHash if child == nil { child = &Node{ - PartialKey: remainingKey, - StorageValue: value, - IsHashedValue: isValueHashed, - Generation: t.generation, - Dirty: true, + PartialKey: remainingKey, + StorageValue: value, + Generation: t.generation, + Dirty: true, } nodesCreated = 1 parentBranch, err = t.prepForMutation(parentBranch, copySettings, pendingDeltas) @@ -569,7 +549,7 @@ func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, isValueHash return parentBranch, mutated, nodesCreated, nil } - child, mutated, nodesCreated, err = t.insert(child, remainingKey, value, isValueHashed, pendingDeltas) + child, mutated, nodesCreated, err = t.insert(child, remainingKey, value, pendingDeltas) if err != nil { // do not wrap error since `insert` may call `insertInBranch` recursively return nil, false, 0, err @@ -614,13 +594,12 @@ func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, isValueHash if len(key) <= commonPrefixLength { newParentBranch.StorageValue = value - newParentBranch.IsHashedValue = isValueHashed } else { childIndex := key[commonPrefixLength] remainingKey := key[commonPrefixLength+1:] var additionalNodesCreated uint32 newParentBranch.Children[childIndex], _, additionalNodesCreated, err = t.insert( - nil, remainingKey, value, isValueHashed, pendingDeltas) + nil, remainingKey, value, pendingDeltas) if err != nil { // do not wrap error since `insert` may call `insertInBranch` recursively return nil, false, 0, err @@ -636,7 +615,7 @@ func (t *Trie) insertInBranch(parentBranch *Node, key, value []byte, isValueHash // LoadFromMap loads the given data mapping of key to value into a new empty trie. // The keys are in hexadecimal little Endian encoding and the values // are hexadecimal encoded. -func LoadFromMap(data map[string]string, version Version) (trie Trie, err error) { +func LoadFromMap(data map[string]string) (trie Trie, err error) { trie = *NewEmptyTrie() pendingDeltas := tracking.New() @@ -655,7 +634,7 @@ func LoadFromMap(data map[string]string, version Version) (trie Trie, err error) return Trie{}, fmt.Errorf("cannot convert value hex to bytes: %w", err) } - err = trie.insertKeyLE(keyLEBytes, valueBytes, pendingDeltas, version) + err = trie.insertKeyLE(keyLEBytes, valueBytes, pendingDeltas) if err != nil { return Trie{}, fmt.Errorf("inserting key value pair in trie: %w", err) } diff --git a/lib/trie/trie_endtoend_test.go b/lib/trie/trie_endtoend_test.go index 4a4707134f..eb73aecaa7 100644 --- a/lib/trie/trie_endtoend_test.go +++ b/lib/trie/trie_endtoend_test.go @@ -40,7 +40,7 @@ func buildSmallTrie() *Trie { } for _, test := range tests { - trie.Put(test.key, test.value, V0) + trie.Put(test.key, test.value) } return trie @@ -50,7 +50,7 @@ func runTests(t *testing.T, trie *Trie, tests []keyValues) { for _, test := range tests { switch test.op { case put: - trie.Put(test.key, test.value, V0) + trie.Put(test.key, test.value) case get: val := trie.Get(test.key) assert.Equal(t, test.value, val) @@ -104,7 +104,7 @@ func TestPutAndGetOddKeyLengths(t *testing.T) { func Fuzz_Trie_PutAndGet_Single(f *testing.F) { f.Fuzz(func(t *testing.T, key, value []byte) { trie := NewTrie(nil, nil) - trie.Put(key, value, V0) + trie.Put(key, value) retrievedValue := trie.Get(key) assert.Equal(t, value, retrievedValue) }) @@ -119,7 +119,7 @@ func Test_Trie_PutAndGet_Multiple(t *testing.T) { keyValues := generateKeyValues(t, generator, numberOfKeyValuePairs) for keyString, value := range keyValues { key := []byte(keyString) - trie.Put(key, value, V0) + trie.Put(key, value) // Check value is inserted correctly. retrievedValue := trie.Get(key) @@ -287,7 +287,7 @@ func TestTrieDiff(t *testing.T) { } for _, test := range tests { - trie.Put(test.key, test.value, V0) + trie.Put(test.key, test.value) } newTrie := trie.Snapshot() @@ -303,7 +303,7 @@ func TestTrieDiff(t *testing.T) { } for _, test := range tests { - newTrie.Put(test.key, test.value, V0) + newTrie.Put(test.key, test.value) } deletedNodeHashes := newTrie.deltas.Deleted() @@ -340,7 +340,7 @@ func TestDelete(t *testing.T) { for keyString, value := range kv { key := []byte(keyString) - trie.Put(key, value, V0) + trie.Put(key, value) } dcTrie := trie.DeepCopy() @@ -423,7 +423,7 @@ func TestClearPrefix(t *testing.T) { trie := NewEmptyTrie() for _, test := range tests { - trie.Put(test.key, test.value, V0) + trie.Put(test.key, test.value) } dcTrie := trie.DeepCopy() @@ -508,7 +508,7 @@ func TestClearPrefix_Small(t *testing.T) { "other", } for _, key := range keys { - ssTrie.Put([]byte(key), []byte(key), V0) + ssTrie.Put([]byte(key), []byte(key)) } ssTrie.ClearPrefix([]byte("noo")) @@ -589,8 +589,8 @@ func TestTrie_ClearPrefixVsDelete(t *testing.T) { trieClearPrefix := NewEmptyTrie() for _, test := range testCase { - trieDelete.Put(test.key, test.value, V0) - trieClearPrefix.Put(test.key, test.value, V0) + trieDelete.Put(test.key, test.value) + trieClearPrefix.Put(test.key, test.value) } prefixedKeys := trieDelete.GetKeysWithPrefix(prefix) @@ -618,7 +618,7 @@ func TestSnapshot(t *testing.T) { expectedTrie := NewEmptyTrie() for _, test := range tests { - expectedTrie.Put(test.key, test.value, V0) + expectedTrie.Put(test.key, test.value) } // put all keys except first @@ -627,11 +627,11 @@ func TestSnapshot(t *testing.T) { if i == 0 { continue } - parentTrie.Put(test.key, test.value, V0) + parentTrie.Put(test.key, test.value) } newTrie := parentTrie.Snapshot() - newTrie.Put(tests[0].key, tests[0].value, V0) + newTrie.Put(tests[0].key, tests[0].value) require.Equal(t, expectedTrie.MustHash(), newTrie.MustHash()) require.NotEqual(t, parentTrie.MustHash(), newTrie.MustHash()) @@ -658,7 +658,7 @@ func Test_Trie_NextKey_Random(t *testing.T) { for _, key := range sortedKeys { value := []byte{1} - trie.Put(key, value, V0) + trie.Put(key, value) } for i, key := range sortedKeys { @@ -682,7 +682,7 @@ func Benchmark_Trie_Hash(b *testing.B) { trie := NewEmptyTrie() for keyString, value := range kv { key := []byte(keyString) - trie.Put(key, value, V0) + trie.Put(key, value) } b.StartTimer() @@ -723,7 +723,7 @@ func TestTrie_ConcurrentSnapshotWrites(t *testing.T) { switch op { case put: - expectedTries[i].Put(k, k, V0) + expectedTries[i].Put(k, k) case del: expectedTries[i].Delete(k) case clearPrefix: @@ -754,7 +754,7 @@ func TestTrie_ConcurrentSnapshotWrites(t *testing.T) { for _, operation := range operations { switch operation.op { case put: - trie.Put(operation.key, operation.key, V0) + trie.Put(operation.key, operation.key) case del: trie.Delete(operation.key) case clearPrefix: @@ -835,7 +835,7 @@ func TestTrie_ClearPrefixLimit(t *testing.T) { trieClearPrefix := NewEmptyTrie() for _, test := range testCase { - trieClearPrefix.Put(test.key, test.value, V0) + trieClearPrefix.Put(test.key, test.value) } num, allDeleted, err := trieClearPrefix.ClearPrefixLimit(prefix, uint32(lim)) @@ -941,7 +941,7 @@ func TestTrie_ClearPrefixLimitSnapshot(t *testing.T) { trieClearPrefix := NewEmptyTrie() for _, test := range testCase { - trieClearPrefix.Put(test.key, test.value, V0) + trieClearPrefix.Put(test.key, test.value) } dcTrie := trieClearPrefix.DeepCopy() @@ -1027,7 +1027,7 @@ func Test_encodeRoot_fuzz(t *testing.T) { kv := generateKeyValues(t, generator, kvSize) for keyString, value := range kv { key := []byte(keyString) - trie.Put(key, value, V0) + trie.Put(key, value) retrievedValue := trie.Get(key) assert.Equal(t, value, retrievedValue) @@ -1082,7 +1082,7 @@ func Test_Trie_Descendants_Fuzz(t *testing.T) { }) for _, key := range keys { - trie.Put(key, kv[string(key)], V0) + trie.Put(key, kv[string(key)]) } testDescendants(t, trie.root) diff --git a/lib/trie/trie_test.go b/lib/trie/trie_test.go index af8efa0cc4..2666583daf 100644 --- a/lib/trie/trie_test.go +++ b/lib/trie/trie_test.go @@ -736,7 +736,7 @@ func Test_Trie_Entries(t *testing.T) { } for k, v := range kv { - trie.Put([]byte(k), v, V0) + trie.Put([]byte(k), v) } entries := trie.Entries() @@ -761,7 +761,7 @@ func Test_Trie_Entries(t *testing.T) { } for k, v := range kv { - trie.Put([]byte(k), v, V1) + trie.Put([]byte(k), v) } entries := trie.Entries() @@ -1105,17 +1105,13 @@ func Test_nextKey(t *testing.T) { func Test_Trie_Put(t *testing.T) { t.Parallel() - longValue := []byte("newvaluewithmorethan32byteslength") - longValueHash := common.MustBlake2bHash(longValue).ToBytes() - testCases := map[string]struct { trie Trie - stateVersion Version key []byte value []byte expectedTrie Trie }{ - "trie_v0_with_key_and_value": { + "trie_with_key_and_value": { trie: Trie{ generation: 1, deltas: newDeltas(), @@ -1151,50 +1147,6 @@ func Test_Trie_Put(t *testing.T) { }, }, }, - "trie_v1_with_key_and_value": { - trie: Trie{ - generation: 1, - deltas: newDeltas(), - root: &Node{ - PartialKey: []byte{1, 2, 0, 5}, - StorageValue: []byte{1}, - }, - db: db.NewEmptyMemoryDB(), - }, - stateVersion: V1, - key: []byte{0x12, 0x16}, - value: longValue, - expectedTrie: Trie{ - generation: 1, - deltas: newDeltas("0xa195089c3e8f8b5b36978700ad954aed99e08413cfc1e2b4c00a5d064abe66a9"), - root: &Node{ - PartialKey: []byte{1, 2}, - Generation: 1, - Dirty: true, - Descendants: 2, - Children: padRightChildren([]*Node{ - { - PartialKey: []byte{5}, - StorageValue: []byte{1}, - Generation: 1, - Dirty: true, - }, - { - PartialKey: []byte{6}, - StorageValue: longValueHash, - IsHashedValue: true, - Generation: 1, - Dirty: true, - }, - }), - }, - db: func() db.Database { - db := db.NewEmptyMemoryDB() - db.Put(longValueHash, longValue) - return db - }(), - }, - }, } for name, testCase := range testCases { @@ -1203,7 +1155,7 @@ func Test_Trie_Put(t *testing.T) { t.Parallel() trie := testCase.trie - trie.Put(testCase.key, testCase.value, testCase.stateVersion) + trie.Put(testCase.key, testCase.value) assert.Equal(t, testCase.expectedTrie, trie) }) @@ -1218,7 +1170,6 @@ func Test_Trie_insert(t *testing.T) { parent *Node key []byte value []byte - isValueHashed bool pendingDeltas DeltaRecorder newNode *Node mutated bool @@ -1431,7 +1382,7 @@ func Test_Trie_insert(t *testing.T) { expectedTrie := *trie.DeepCopy() newNode, mutated, nodesCreated, err := trie.insert( - testCase.parent, testCase.key, testCase.value, testCase.isValueHashed, + testCase.parent, testCase.key, testCase.value, testCase.pendingDeltas) require.NoError(t, err) @@ -1451,7 +1402,6 @@ func Test_Trie_insertInBranch(t *testing.T) { parent *Node key []byte value []byte - isValueHashed bool pendingDeltas DeltaRecorder newNode *Node mutated bool @@ -1731,7 +1681,7 @@ func Test_Trie_insertInBranch(t *testing.T) { trie := new(Trie) newNode, mutated, nodesCreated, err := trie.insertInBranch( - testCase.parent, testCase.key, testCase.value, testCase.isValueHashed, + testCase.parent, testCase.key, testCase.value, testCase.pendingDeltas) assert.ErrorIs(t, err, testCase.errSentinel) @@ -1751,11 +1701,10 @@ func Test_LoadFromMap(t *testing.T) { t.Parallel() testCases := map[string]struct { - data map[string]string - stateTrieVersion Version - expectedTrie Trie - errWrapped error - errMessage string + data map[string]string + expectedTrie Trie + errWrapped error + errMessage string }{ "nil_data": { expectedTrie: Trie{ @@ -1844,7 +1793,7 @@ func Test_LoadFromMap(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - trie, err := LoadFromMap(testCase.data, testCase.stateTrieVersion) + trie, err := LoadFromMap(testCase.data) assert.ErrorIs(t, err, testCase.errWrapped) if testCase.errWrapped != nil { diff --git a/lib/trie/version.go b/lib/trie/version.go index ab3f2e729f..a760fa0d94 100644 --- a/lib/trie/version.go +++ b/lib/trie/version.go @@ -6,16 +6,17 @@ package trie import ( "errors" "fmt" + "math" "strings" "github.com/ChainSafe/gossamer/lib/common" ) const ( - // NoMaxValueSize is the numeric representation used to indicate that there is no max value size. - NoMaxValueSize = -1 - // V1MaxValueSize is the maximum size of a value to be hashed in state trie version 1. - V1MaxValueSize = 32 + // NoMaxInlineValueSize is the numeric representation used to indicate that there is no max value size. + NoMaxInlineValueSize = math.MaxInt + // V1MaxInlineValueSize is the maximum size of a value to be hashed in state trie version 1. + V1MaxInlineValueSize = 32 ) // Version is the state trie version which dictates how a @@ -56,9 +57,15 @@ func (v Version) String() string { } } -// ShouldHashValue returns true if the value should be hashed based on trie version -func (v Version) ShouldHashValue(value []byte) bool { - return v == V1 && len(value) > V1MaxValueSize +func (v Version) maxInlineValue() int { + switch v { + case V0: + return NoMaxInlineValueSize + case V1: + return V1MaxInlineValueSize + default: + panic(fmt.Sprintf("unknown version %d", v)) + } } // Root returns the root hash of the trie built using the given entries @@ -66,7 +73,7 @@ func (v Version) Root(entries Entries) (common.Hash, error) { t := NewEmptyTrie() for _, kv := range entries { - err := t.Put(kv.Key, kv.Value, v) + err := t.Put(kv.Key, kv.Value) if err != nil { return common.EmptyHash, err } diff --git a/lib/trie/version_test.go b/lib/trie/version_test.go index dfb7a5c85c..5f894277e3 100644 --- a/lib/trie/version_test.go +++ b/lib/trie/version_test.go @@ -117,47 +117,6 @@ func Test_ParseVersion(t *testing.T) { } } -func Test_ShouldHashValue(t *testing.T) { - t.Parallel() - - testCases := map[string]struct { - version Version - value []byte - shouldHash bool - }{ - "v0_small_value": { - version: V0, - value: []byte("smallvalue"), - shouldHash: false, - }, - "v0_large_value": { - version: V0, - value: []byte("newvaluewithmorethan32byteslength"), - shouldHash: false, - }, - "v1_small_value": { - version: V1, - value: []byte("smallvalue"), - shouldHash: false, - }, - "v1_large_value": { - version: V1, - value: []byte("newvaluewithmorethan32byteslength"), - shouldHash: true, - }, - } - - for name, testCase := range testCases { - testCase := testCase - t.Run(name, func(t *testing.T) { - t.Parallel() - - shouldHash := testCase.version.ShouldHashValue(testCase.value) - assert.Equal(t, testCase.shouldHash, shouldHash) - }) - } -} - func Test_Version_Root(t *testing.T) { t.Parallel() From f0478c2d009a992182e12f5072743568eb8bd741 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 7 Nov 2023 14:31:03 -0300 Subject: [PATCH 097/128] Remove DB --- dot/state/storage.go | 2 -- dot/state/tries.go | 8 -------- lib/trie/trie.go | 5 ----- 3 files changed, 15 deletions(-) diff --git a/dot/state/storage.go b/dot/state/storage.go index 05e19eb328..02bc495527 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -48,8 +48,6 @@ func NewStorageState(db database.Database, blockState *BlockState, tries *Tries) (*StorageState, error) { storageTable := database.NewTable(db, storagePrefix) - tries.SetDB(storageTable) - return &StorageState{ blockState: blockState, tries: tries, diff --git a/dot/state/tries.go b/dot/state/tries.go index 042dd6282e..34a8484a7d 100644 --- a/dot/state/tries.go +++ b/dot/state/tries.go @@ -8,7 +8,6 @@ import ( "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/trie" - "github.com/ChainSafe/gossamer/lib/trie/db" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) @@ -52,13 +51,6 @@ func NewTries() (tries *Tries) { } } -// SetDB is to set the same db for every trie in the collection -func (t *Tries) SetDB(db db.Database) { - for _, trie := range t.rootToTrie { - trie.SetDB(db) - } -} - // SetEmptyTrie sets the empty trie in the tries. // Note the empty trie is the same for the v0 and the v1 // state trie versions. diff --git a/lib/trie/trie.go b/lib/trie/trie.go index c0d77c7d20..688bd12005 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -62,11 +62,6 @@ func (t *Trie) Equal(other *Trie) bool { reflect.DeepEqual(t.childTries, other.childTries) && reflect.DeepEqual(t.deltas, other.deltas) } -// SetDB is to set the db that this trie will use primary to store v1 value nodes -func (t *Trie) SetDB(db db.Database) { - t.db = db -} - // Snapshot creates a copy of the trie. // Note it does not deep copy the trie, but will // copy on write as modifications are done on this new trie. From b04de53e7319e8d6fbf8c4427ae803f8c1e54736 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 7 Nov 2023 17:32:06 -0300 Subject: [PATCH 098/128] Use trie version to calculate hash only --- cmd/gossamer/commands/import_state.go | 2 - cmd/gossamer/commands/import_state_test.go | 26 ----- dot/core/helpers_test.go | 4 +- dot/core/service.go | 5 +- dot/digest/helpers_test.go | 2 +- dot/helpers_test.go | 2 +- dot/import_test.go | 3 +- dot/node_integration_test.go | 2 +- dot/rpc/helpers_test.go | 2 +- .../modules/childstate_integration_test.go | 2 +- dot/rpc/modules/childstate_test.go | 2 +- dot/rpc/modules/helpers_test.go | 2 +- dot/rpc/modules/state_integration_test.go | 3 +- dot/rpc/modules/system_integration_test.go | 2 +- dot/state/base_test.go | 6 +- dot/state/helpers_test.go | 2 +- dot/state/service.go | 2 +- dot/state/service_integration_test.go | 8 +- dot/state/storage.go | 6 +- dot/state/storage_test.go | 14 +-- dot/state/tries.go | 4 +- dot/state/tries_test.go | 2 +- dot/sync/chain_sync.go | 3 +- dot/sync/chain_sync_test.go | 12 +- dot/sync/syncer_integration_test.go | 4 +- dot/utils_integration_test.go | 12 +- internal/trie/node/branch_encode.go | 30 ++--- internal/trie/node/branch_encode_test.go | 103 +----------------- internal/trie/node/encode.go | 18 ++- internal/trie/node/encode_decode_test.go | 3 +- internal/trie/node/encode_test.go | 96 ++++++---------- internal/trie/node/hash.go | 16 +-- internal/trie/node/hash_test.go | 9 +- internal/trie/node/header.go | 8 +- internal/trie/node/header_test.go | 30 ++--- lib/babe/helpers_test.go | 4 +- lib/grandpa/helpers_integration_test.go | 2 +- lib/runtime/interfaces.go | 2 +- lib/runtime/storage/trie.go | 8 +- lib/runtime/storage/trie_test.go | 4 +- lib/runtime/wazero/imports.go | 22 +++- lib/runtime/wazero/imports_test.go | 41 ++++--- lib/runtime/wazero/instance_test.go | 22 ++-- lib/trie/child_storage.go | 6 +- lib/trie/child_storage_test.go | 6 +- lib/trie/database.go | 12 +- lib/trie/database_test.go | 14 +-- lib/trie/genesis.go | 2 +- lib/trie/proof/generate.go | 4 +- lib/trie/proof/helpers_test.go | 3 +- lib/trie/proof/proof_test.go | 3 +- lib/trie/trie.go | 19 +--- lib/trie/trie_endtoend_test.go | 73 +++++++------ lib/trie/trie_test.go | 31 +----- lib/trie/version.go | 15 ++- 55 files changed, 298 insertions(+), 442 deletions(-) diff --git a/cmd/gossamer/commands/import_state.go b/cmd/gossamer/commands/import_state.go index 850101f81d..e7b5991a62 100644 --- a/cmd/gossamer/commands/import_state.go +++ b/cmd/gossamer/commands/import_state.go @@ -7,14 +7,12 @@ import ( "fmt" "github.com/ChainSafe/gossamer/dot" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" "github.com/spf13/cobra" ) func init() { ImportStateCmd.Flags().String("state-file", "", "Path to JSON file consisting of key-value pairs") - ImportStateCmd.Flags().String("state-version", trie.DefaultStateVersion.String(), "State version v0 or v1") ImportStateCmd.Flags().String("header-file", "", "Path to JSON file of block header corresponding to the given state") ImportStateCmd.Flags().Uint64("first-slot", 0, "The first BABE slot of the network") } diff --git a/cmd/gossamer/commands/import_state_test.go b/cmd/gossamer/commands/import_state_test.go index e7a35ed3d4..6406023a42 100644 --- a/cmd/gossamer/commands/import_state_test.go +++ b/cmd/gossamer/commands/import_state_test.go @@ -30,23 +30,12 @@ func TestImportStateInvalidFirstSlot(t *testing.T) { assert.ErrorContains(t, err, "invalid argument \"wrong\"") } -func TestImportStateEmptyStateFile(t *testing.T) { - rootCmd, err := NewRootCommand() - require.NoError(t, err) - rootCmd.AddCommand(ImportStateCmd) - - rootCmd.SetArgs([]string{ImportStateCmd.Name(), "--state-version", "v0", "--state-file", ""}) - err = rootCmd.Execute() - assert.ErrorContains(t, err, "state-file must be specified") -} - func TestImportStateEmptyHeaderFile(t *testing.T) { rootCmd, err := NewRootCommand() require.NoError(t, err) rootCmd.AddCommand(ImportStateCmd) rootCmd.SetArgs([]string{ImportStateCmd.Name(), - "--state-version", "v0", "--state-file", "test", "--header-file", "", }) @@ -54,27 +43,12 @@ func TestImportStateEmptyHeaderFile(t *testing.T) { assert.ErrorContains(t, err, "header-file must be specified") } -func TestImportStateInvalidStateVersion(t *testing.T) { - rootCmd, err := NewRootCommand() - require.NoError(t, err) - rootCmd.AddCommand(ImportStateCmd) - - rootCmd.SetArgs([]string{ImportStateCmd.Name(), - "--state-version", "v999", - "--state-file", "test", - "--header-file", "test", - }) - err = rootCmd.Execute() - assert.ErrorContains(t, err, "failed to parse state-version: parsing version failed: \"v999\" must be one of [v0, v1]") -} - func TestImportStateErrorImportingState(t *testing.T) { rootCmd, err := NewRootCommand() require.NoError(t, err) rootCmd.AddCommand(ImportStateCmd) rootCmd.SetArgs([]string{ImportStateCmd.Name(), - "--state-version", "v0", "--state-file", "test", "--header-file", "test", }) diff --git a/dot/core/helpers_test.go b/dot/core/helpers_test.go index 53a2af4d78..f25ed6c7ad 100644 --- a/dot/core/helpers_test.go +++ b/dot/core/helpers_test.go @@ -57,7 +57,7 @@ func createTestService(t *testing.T, genesisFilePath string, require.NoError(t, err) genesisHeader := &types.Header{ - StateRoot: genesisTrie.MustHash(), + StateRoot: genesisTrie.MustHash(trie.NoMaxInlineValueSize), Number: 0, } @@ -271,7 +271,7 @@ func newWestendLocalWithTrieAndHeader(t *testing.T) ( require.NoError(t, err) parentHash := common.NewHash([]byte{0}) - stateRoot := genesisTrie.MustHash() + stateRoot := genesisTrie.MustHash(trie.NoMaxInlineValueSize) extrinsicRoot := trie.EmptyHash const number = 0 digest := types.NewDigest() diff --git a/dot/core/service.go b/dot/core/service.go index 07204a59e8..217dff9e36 100644 --- a/dot/core/service.go +++ b/dot/core/service.go @@ -20,6 +20,7 @@ import ( rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage" wazero_runtime "github.com/ChainSafe/gossamer/lib/runtime/wazero" "github.com/ChainSafe/gossamer/lib/transaction" + "github.com/ChainSafe/gossamer/lib/trie" cscale "github.com/centrifuge/go-substrate-rpc-client/v4/scale" ctypes "github.com/centrifuge/go-substrate-rpc-client/v4/types" @@ -123,7 +124,7 @@ func (s *Service) StorageRoot() (common.Hash, error) { return common.Hash{}, err } - return ts.Root() + return ts.Root(trie.NoMaxInlineValueSize) } // HandleBlockImport handles a block that was imported via the network @@ -226,7 +227,7 @@ func (s *Service) handleBlock(block *types.Block, state *rtstorage.TrieState) er } logger.Debugf("imported block %s and stored state trie with root %s", - block.Header.Hash(), state.MustRoot()) + block.Header.Hash(), state.MustRoot(trie.NoMaxInlineValueSize)) parentRuntimeInstance, err := s.blockState.GetRuntime(block.Header.ParentHash) if err != nil { diff --git a/dot/digest/helpers_test.go b/dot/digest/helpers_test.go index 10298488a9..4f4fc81f9a 100644 --- a/dot/digest/helpers_test.go +++ b/dot/digest/helpers_test.go @@ -28,7 +28,7 @@ func newWestendDevGenesisWithTrieAndHeader(t *testing.T) ( require.NoError(t, err) parentHash := common.NewHash([]byte{0}) - stateRoot := genesisTrie.MustHash() + stateRoot := genesisTrie.MustHash(trie.NoMaxInlineValueSize) extrinsicRoot := trie.EmptyHash const number = 0 digest := types.NewDigest() diff --git a/dot/helpers_test.go b/dot/helpers_test.go index 88a22bbb37..2889f70af4 100644 --- a/dot/helpers_test.go +++ b/dot/helpers_test.go @@ -42,7 +42,7 @@ func newWestendDevGenesisWithTrieAndHeader(t *testing.T) ( require.NoError(t, err) parentHash := common.NewHash([]byte{0}) - stateRoot := genesisTrie.MustHash() + stateRoot := genesisTrie.MustHash(trie.NoMaxInlineValueSize) extrinsicRoot := trie.EmptyHash const number = 0 digest := types.NewDigest() diff --git a/dot/import_test.go b/dot/import_test.go index fa53b453ce..c5a5fcf72e 100644 --- a/dot/import_test.go +++ b/dot/import_test.go @@ -12,6 +12,7 @@ import ( "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -174,7 +175,7 @@ func Test_newTrieFromPairs(t *testing.T) { if tt.want.IsEmpty() { assert.Nil(t, got) } else { - assert.Equal(t, tt.want, got.MustHash()) + assert.Equal(t, tt.want, got.MustHash(trie.NoMaxInlineValueSize)) } }) } diff --git a/dot/node_integration_test.go b/dot/node_integration_test.go index 19e048b92e..63026a8688 100644 --- a/dot/node_integration_test.go +++ b/dot/node_integration_test.go @@ -388,7 +388,7 @@ func TestInitNode_LoadStorageRoot(t *testing.T) { expected, err := trie.LoadFromMap(gen.GenesisFields().Raw["top"]) require.NoError(t, err) - expectedRoot, err := expected.Hash() + expectedRoot, err := expected.Hash(trie.NoMaxInlineValueSize) require.NoError(t, err) coreServiceInterface := node.ServiceRegistry.Get(&core.Service{}) diff --git a/dot/rpc/helpers_test.go b/dot/rpc/helpers_test.go index 0e981800d2..ef962f0f46 100644 --- a/dot/rpc/helpers_test.go +++ b/dot/rpc/helpers_test.go @@ -28,7 +28,7 @@ func newWestendDevGenesisWithTrieAndHeader(t *testing.T) ( require.NoError(t, err) parentHash := common.NewHash([]byte{0}) - stateRoot := genesisTrie.MustHash() + stateRoot := genesisTrie.MustHash(trie.NoMaxInlineValueSize) extrinsicRoot := trie.EmptyHash const number = 0 digest := types.NewDigest() diff --git a/dot/rpc/modules/childstate_integration_test.go b/dot/rpc/modules/childstate_integration_test.go index c9e7481051..74c7ae53e0 100644 --- a/dot/rpc/modules/childstate_integration_test.go +++ b/dot/rpc/modules/childstate_integration_test.go @@ -255,7 +255,7 @@ func setupChildStateStorage(t *testing.T) (*ChildStateModule, common.Hash) { err = tr.SetChild([]byte(":child_storage_key"), childTr) require.NoError(t, err) - stateRoot, err := tr.Root() + stateRoot, err := tr.Root(trie.NoMaxInlineValueSize) require.NoError(t, err) bb, err := st.Block.BestBlock() diff --git a/dot/rpc/modules/childstate_test.go b/dot/rpc/modules/childstate_test.go index c84df630ce..9fe353fe49 100644 --- a/dot/rpc/modules/childstate_test.go +++ b/dot/rpc/modules/childstate_test.go @@ -35,7 +35,7 @@ func createTestTrieState(t *testing.T) (*trie.Trie, common.Hash) { err := tr.SetChild([]byte(":child_storage_key"), childTr) require.NoError(t, err) - stateRoot, err := tr.Root() + stateRoot, err := tr.Root(trie.NoMaxInlineValueSize) require.NoError(t, err) return tr.Trie(), stateRoot diff --git a/dot/rpc/modules/helpers_test.go b/dot/rpc/modules/helpers_test.go index fad1310ce0..bce60ba019 100644 --- a/dot/rpc/modules/helpers_test.go +++ b/dot/rpc/modules/helpers_test.go @@ -35,7 +35,7 @@ func newWestendLocalGenesisWithTrieAndHeader(t *testing.T) ( require.NoError(t, err) parentHash := common.NewHash([]byte{0}) - stateRoot := genesisTrie.MustHash() + stateRoot := genesisTrie.MustHash(trie.NoMaxInlineValueSize) extrinsicRoot := trie.EmptyHash const number = 0 digest := types.NewDigest() diff --git a/dot/rpc/modules/state_integration_test.go b/dot/rpc/modules/state_integration_test.go index 04cd189554..7347aab6c5 100644 --- a/dot/rpc/modules/state_integration_test.go +++ b/dot/rpc/modules/state_integration_test.go @@ -17,6 +17,7 @@ import ( "github.com/ChainSafe/gossamer/dot/rpc/modules/mocks" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/trie" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -574,7 +575,7 @@ func setupStateModule(t *testing.T) (*StateModule, *common.Hash, *common.Hash) { err = ts.SetChildStorage([]byte(`:child1`), []byte(`:key1`), []byte(`:childValue1`)) require.NoError(t, err) - sr1, err := ts.Root() + sr1, err := ts.Root(trie.NoMaxInlineValueSize) require.NoError(t, err) err = chain.Storage.StoreTrie(ts, nil) require.NoError(t, err) diff --git a/dot/rpc/modules/system_integration_test.go b/dot/rpc/modules/system_integration_test.go index 453ac8bb72..ad9ae3ae8c 100644 --- a/dot/rpc/modules/system_integration_test.go +++ b/dot/rpc/modules/system_integration_test.go @@ -330,7 +330,7 @@ func setupSystemModule(t *testing.T) *SystemModule { Header: types.Header{ Number: 3, ParentHash: chain.Block.BestBlockHash(), - StateRoot: ts.MustRoot(), + StateRoot: ts.MustRoot(trie.NoMaxInlineValueSize), Digest: digest, }, Body: types.Body{}, diff --git a/dot/state/base_test.go b/dot/state/base_test.go index a6bedcde74..637ee44bb9 100644 --- a/dot/state/base_test.go +++ b/dot/state/base_test.go @@ -29,15 +29,15 @@ func TestTrie_StoreAndLoadFromDB(t *testing.T) { err := tt.WriteDirty(db) require.NoError(t, err) - encroot, err := tt.Hash() + encroot, err := tt.Hash(trie.NoMaxInlineValueSize) require.NoError(t, err) - expected := tt.MustHash() + expected := tt.MustHash(trie.NoMaxInlineValueSize) tt = trie.NewEmptyTrie() err = tt.Load(db, encroot) require.NoError(t, err) - require.Equal(t, expected, tt.MustHash()) + require.Equal(t, expected, tt.MustHash(trie.NoMaxInlineValueSize)) } func TestStoreAndLoadGenesisData(t *testing.T) { diff --git a/dot/state/helpers_test.go b/dot/state/helpers_test.go index 44abfc4de5..10029e9f01 100644 --- a/dot/state/helpers_test.go +++ b/dot/state/helpers_test.go @@ -102,7 +102,7 @@ func newWestendDevGenesisWithTrieAndHeader(t *testing.T) ( require.NoError(t, err) parentHash := common.NewHash([]byte{0}) - stateRoot := genesisTrie.MustHash() + stateRoot := genesisTrie.MustHash(trie.NoMaxInlineValueSize) extrinsicRoot := trie.EmptyHash const number = 0 digest := types.NewDigest() diff --git a/dot/state/service.go b/dot/state/service.go index eb17ca25ea..a5b405ee19 100644 --- a/dot/state/service.go +++ b/dot/state/service.go @@ -301,7 +301,7 @@ func (s *Service) Import(header *types.Header, t *trie.Trie, firstSlot uint64) e return err } - root := t.MustHash() + root := t.MustHash(trie.NoMaxInlineValueSize) if root != header.StateRoot { return fmt.Errorf("trie state root does not equal header state root") } diff --git a/dot/state/service_integration_test.go b/dot/state/service_integration_test.go index d5b51186e7..cf041c64ee 100644 --- a/dot/state/service_integration_test.go +++ b/dot/state/service_integration_test.go @@ -90,7 +90,7 @@ func TestService_Initialise(t *testing.T) { require.NoError(t, err) genesisHeaderPtr := types.NewHeader(common.NewHash([]byte{77}), - genTrie.MustHash(), trie.EmptyHash, 0, types.NewDigest()) + genTrie.MustHash(trie.NoMaxInlineValueSize), trie.EmptyHash, 0, types.NewDigest()) err = state.Initialise(&genData, genesisHeaderPtr, genTrieCopy) require.NoError(t, err) @@ -287,7 +287,7 @@ func TestService_PruneStorage(t *testing.T) { copiedTrie := trieState.Trie().DeepCopy() var rootHash common.Hash - rootHash, err = copiedTrie.Hash() + rootHash, err = copiedTrie.Hash(trie.NoMaxInlineValueSize) require.NoError(t, err) prunedArr = append(prunedArr, prunedBlock{hash: block.Header.StateRoot, dbKey: rootHash[:]}) @@ -400,7 +400,7 @@ func TestService_Import(t *testing.T) { require.NoError(t, err) header := &types.Header{ Number: 77, - StateRoot: tr.MustHash(), + StateRoot: tr.MustHash(trie.NoMaxInlineValueSize), Digest: digest, } @@ -440,7 +440,7 @@ func generateBlockWithRandomTrie(t *testing.T, serv *Service, err = trieState.Put(key, value) require.NoError(t, err) - trieStateRoot, err := trieState.Root() + trieStateRoot, err := trieState.Root(trie.NoMaxInlineValueSize) require.NoError(t, err) if parent == nil { diff --git a/dot/state/storage.go b/dot/state/storage.go index 02bc495527..ed8f8377a0 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -59,7 +59,7 @@ func NewStorageState(db database.Database, blockState *BlockState, // StoreTrie stores the given trie in the StorageState and writes it to the database func (s *StorageState) StoreTrie(ts *rtstorage.TrieState, header *types.Header) error { - root := ts.MustRoot() + root := ts.MustRoot(trie.NoMaxInlineValueSize) s.tries.softSet(root, ts.Trie()) @@ -106,7 +106,7 @@ func (s *StorageState) TrieState(root *common.Hash) (*rtstorage.TrieState, error } s.tries.softSet(*root, t) - } else if t.MustHash() != *root { + } else if t.MustHash(trie.NoMaxInlineValueSize) != *root { panic("trie does not have expected root") } @@ -125,7 +125,7 @@ func (s *StorageState) LoadFromDB(root common.Hash) (*trie.Trie, error) { return nil, err } - s.tries.softSet(t.MustHash(), t) + s.tries.softSet(t.MustHash(trie.NoMaxInlineValueSize), t) return t, nil } diff --git a/dot/state/storage_test.go b/dot/state/storage_test.go index 9f745a8891..a654e3ea82 100644 --- a/dot/state/storage_test.go +++ b/dot/state/storage_test.go @@ -35,7 +35,7 @@ func TestStorage_StoreAndLoadTrie(t *testing.T) { ts, err := storage.TrieState(&trie.EmptyHash) require.NoError(t, err) - root, err := ts.Root() + root, err := ts.Root(trie.NoMaxInlineValueSize) require.NoError(t, err) err = storage.StoreTrie(ts, nil) require.NoError(t, err) @@ -59,7 +59,7 @@ func TestStorage_GetStorageByBlockHash(t *testing.T) { value := []byte("testvalue") ts.Put(key, value) - root, err := ts.Root() + root, err := ts.Root(trie.NoMaxInlineValueSize) require.NoError(t, err) err = storage.StoreTrie(ts, nil) require.NoError(t, err) @@ -91,7 +91,7 @@ func TestStorage_TrieState(t *testing.T) { require.NoError(t, err) ts.Put([]byte("noot"), []byte("washere")) - root, err := ts.Root() + root, err := ts.Root(trie.NoMaxInlineValueSize) require.NoError(t, err) err = storage.StoreTrie(ts, nil) require.NoError(t, err) @@ -102,7 +102,7 @@ func TestStorage_TrieState(t *testing.T) { storage.blockState.tries.delete(root) ts3, err := storage.TrieState(&root) require.NoError(t, err) - require.Equal(t, ts.Trie().MustHash(), ts3.Trie().MustHash()) + require.Equal(t, ts.Trie().MustHash(trie.NoMaxInlineValueSize), ts3.Trie().MustHash(trie.NoMaxInlineValueSize)) } func TestStorage_LoadFromDB(t *testing.T) { @@ -125,7 +125,7 @@ func TestStorage_LoadFromDB(t *testing.T) { ts.Put(kv.key, kv.value) } - root, err := ts.Root() + root, err := ts.Root(trie.NoMaxInlineValueSize) require.NoError(t, err) // Write trie to disk. @@ -205,13 +205,13 @@ func TestGetStorageChildAndGetStorageFromChild(t *testing.T) { trieState := runtime.NewTrieState(&genTrie) - header := types.NewHeader(blockState.GenesisHash(), trieState.MustRoot(), + header := types.NewHeader(blockState.GenesisHash(), trieState.MustRoot(trie.NoMaxInlineValueSize), common.Hash{}, 1, types.NewDigest()) err = storage.StoreTrie(trieState, header) require.NoError(t, err) - rootHash, err := genTrie.Hash() + rootHash, err := genTrie.Hash(trie.NoMaxInlineValueSize) require.NoError(t, err) _, err = storage.GetStorageChild(&rootHash, []byte("keyToChild")) diff --git a/dot/state/tries.go b/dot/state/tries.go index 34a8484a7d..d7c87bc625 100644 --- a/dot/state/tries.go +++ b/dot/state/tries.go @@ -59,8 +59,8 @@ func (t *Tries) SetEmptyTrie() { } // SetTrie sets the trie at its root hash in the tries map. -func (t *Tries) SetTrie(trie *trie.Trie) { - t.softSet(trie.MustHash(), trie) +func (t *Tries) SetTrie(tr *trie.Trie) { + t.softSet(tr.MustHash(trie.NoMaxInlineValueSize), tr) } // softSet sets the given trie at the given root hash diff --git a/dot/state/tries_test.go b/dot/state/tries_test.go index 4bdc84e1d2..40ef5626bd 100644 --- a/dot/state/tries_test.go +++ b/dot/state/tries_test.go @@ -59,7 +59,7 @@ func Test_Tries_SetTrie(t *testing.T) { expectedTries := &Tries{ rootToTrie: map[common.Hash]*trie.Trie{ - tr.MustHash(): tr, + tr.MustHash(trie.NoMaxInlineValueSize): tr, }, triesGauge: triesGauge, setCounter: setCounter, diff --git a/dot/sync/chain_sync.go b/dot/sync/chain_sync.go index 6457a72e0b..68da2fa83c 100644 --- a/dot/sync/chain_sync.go +++ b/dot/sync/chain_sync.go @@ -26,6 +26,7 @@ import ( "github.com/ChainSafe/gossamer/lib/blocktree" "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/common/variadic" + "github.com/ChainSafe/gossamer/lib/trie" ) var _ ChainSync = (*chainSync)(nil) @@ -950,7 +951,7 @@ func (cs *chainSync) handleBlock(block *types.Block, announceImportedBlock bool) return err } - root := ts.MustRoot() + root := ts.MustRoot(trie.NoMaxInlineValueSize) if !bytes.Equal(parent.StateRoot[:], root[:]) { panic("parent state root does not match snapshot state root") } diff --git a/dot/sync/chain_sync_test.go b/dot/sync/chain_sync_test.go index 0344ab43f6..fb932a8c60 100644 --- a/dot/sync/chain_sync_test.go +++ b/dot/sync/chain_sync_test.go @@ -65,9 +65,10 @@ func Test_chainSync_onBlockAnnounce(t *testing.T) { errTest := errors.New("test error") emptyTrieState := storage.NewTrieState(nil) - block1AnnounceHeader := types.NewHeader(common.Hash{}, emptyTrieState.MustRoot(), + block1AnnounceHeader := types.NewHeader(common.Hash{}, emptyTrieState.MustRoot(trie.NoMaxInlineValueSize), common.Hash{}, 1, scale.VaryingDataTypeSlice{}) - block2AnnounceHeader := types.NewHeader(block1AnnounceHeader.Hash(), emptyTrieState.MustRoot(), + block2AnnounceHeader := types.NewHeader(block1AnnounceHeader.Hash(), + emptyTrieState.MustRoot(trie.NoMaxInlineValueSize), common.Hash{}, 2, scale.VaryingDataTypeSlice{}) testCases := map[string]struct { @@ -242,9 +243,10 @@ func Test_chainSync_onBlockAnnounceHandshake_tipModeNeedToCatchup(t *testing.T) const somePeer = peer.ID("abc") emptyTrieState := storage.NewTrieState(nil) - block1AnnounceHeader := types.NewHeader(common.Hash{}, emptyTrieState.MustRoot(), + block1AnnounceHeader := types.NewHeader(common.Hash{}, emptyTrieState.MustRoot(trie.NoMaxInlineValueSize), common.Hash{}, 1, scale.VaryingDataTypeSlice{}) - block2AnnounceHeader := types.NewHeader(block1AnnounceHeader.Hash(), emptyTrieState.MustRoot(), + block2AnnounceHeader := types.NewHeader(block1AnnounceHeader.Hash(), + emptyTrieState.MustRoot(trie.NoMaxInlineValueSize), common.Hash{}, 130, scale.VaryingDataTypeSlice{}) blockStateMock := NewMockBlockState(ctrl) @@ -1249,7 +1251,7 @@ func createSuccesfullBlockResponse(t *testing.T, parentHeader common.Hash, response.BlockData = make([]*types.BlockData, numBlocks) emptyTrieState := storage.NewTrieState(nil) - tsRoot := emptyTrieState.MustRoot() + tsRoot := emptyTrieState.MustRoot(trie.NoMaxInlineValueSize) firstHeader := types.NewHeader(parentHeader, tsRoot, common.Hash{}, uint(startingAt), scale.VaryingDataTypeSlice{}) diff --git a/dot/sync/syncer_integration_test.go b/dot/sync/syncer_integration_test.go index b12ed27363..9333486bce 100644 --- a/dot/sync/syncer_integration_test.go +++ b/dot/sync/syncer_integration_test.go @@ -99,7 +99,7 @@ func newTestSyncer(t *testing.T) *Service { stateSrvc.Block.StoreRuntime(block.Header.Hash(), instance) logger.Debugf("imported block %s and stored state trie with root %s", - block.Header.Hash(), ts.MustRoot()) + block.Header.Hash(), ts.MustRoot(trie.NoMaxInlineValueSize)) return nil }).AnyTimes() cfg.BlockImportHandler = blockImportHandler @@ -137,7 +137,7 @@ func newWestendDevGenesisWithTrieAndHeader(t *testing.T) ( require.NoError(t, err) parentHash := common.NewHash([]byte{0}) - stateRoot := genesisTrie.MustHash() + stateRoot := genesisTrie.MustHash(trie.NoMaxInlineValueSize) extrinsicRoot := trie.EmptyHash const number = 0 digest := types.NewDigest() diff --git a/dot/utils_integration_test.go b/dot/utils_integration_test.go index 7f4e0a67a3..27e0d7c738 100644 --- a/dot/utils_integration_test.go +++ b/dot/utils_integration_test.go @@ -36,13 +36,13 @@ func TestTrieSnapshot(t *testing.T) { newTrie := tri.Snapshot() // Get the Trie root hash for all the 3 tries. - tHash, err := tri.Hash() + tHash, err := tri.Hash(trie.NoMaxInlineValueSize) require.NoError(t, err) - dcTrieHash, err := deepCopyTrie.Hash() + dcTrieHash, err := deepCopyTrie.Hash(trie.NoMaxInlineValueSize) require.NoError(t, err) - newTrieHash, err := newTrie.Hash() + newTrieHash, err := newTrie.Hash(trie.NoMaxInlineValueSize) require.NoError(t, err) // Root hash for the 3 tries should be equal. @@ -54,13 +54,13 @@ func TestTrieSnapshot(t *testing.T) { newTrie.Put(key, value) // Get the updated root hash of all tries. - tHash, err = tri.Hash() + tHash, err = tri.Hash(trie.NoMaxInlineValueSize) require.NoError(t, err) - dcTrieHash, err = deepCopyTrie.Hash() + dcTrieHash, err = deepCopyTrie.Hash(trie.NoMaxInlineValueSize) require.NoError(t, err) - newTrieHash, err = newTrie.Hash() + newTrieHash, err = newTrie.Hash(trie.NoMaxInlineValueSize) require.NoError(t, err) // Only the current trie should have a different root hash since it is updated. diff --git a/internal/trie/node/branch_encode.go b/internal/trie/node/branch_encode.go index 75d00cc6d8..d9b8e7b5a6 100644 --- a/internal/trie/node/branch_encode.go +++ b/internal/trie/node/branch_encode.go @@ -18,10 +18,10 @@ type encodingAsyncResult struct { err error } -func runEncodeChild(child *Node, index int, +func runEncodeChild(child *Node, index, maxInlineValue int, results chan<- encodingAsyncResult, rateLimit <-chan struct{}) { buffer := bytes.NewBuffer(nil) - err := encodeChild(child, buffer) + err := encodeChild(child, maxInlineValue, buffer) results <- encodingAsyncResult{ index: index, @@ -44,7 +44,7 @@ var parallelEncodingRateLimit = make(chan struct{}, parallelLimit) // goroutines IF they are less than the parallelLimit number of goroutines already // running. This is designed to limit the total number of goroutines in order to // avoid using too much memory on the stack. -func encodeChildrenOpportunisticParallel(children []*Node, buffer io.Writer) (err error) { +func encodeChildrenOpportunisticParallel(children []*Node, maxInlineValue int, buffer io.Writer) (err error) { // Buffered channels since children might be encoded in this // goroutine or another one. resultsCh := make(chan encodingAsyncResult, ChildrenCapacity) @@ -56,7 +56,7 @@ func encodeChildrenOpportunisticParallel(children []*Node, buffer io.Writer) (er } if child.Kind() == Leaf { - runEncodeChild(child, i, resultsCh, nil) + runEncodeChild(child, i, maxInlineValue, resultsCh, nil) continue } @@ -65,11 +65,11 @@ func encodeChildrenOpportunisticParallel(children []*Node, buffer io.Writer) (er case parallelEncodingRateLimit <- struct{}{}: // We have a goroutine available to encode // the branch in parallel. - go runEncodeChild(child, i, resultsCh, parallelEncodingRateLimit) + go runEncodeChild(child, i, maxInlineValue, resultsCh, parallelEncodingRateLimit) default: // we reached the maximum parallel goroutines // so encode this branch in this goroutine - runEncodeChild(child, i, resultsCh, nil) + runEncodeChild(child, i, maxInlineValue, resultsCh, nil) } } @@ -116,24 +116,10 @@ func encodeChildrenOpportunisticParallel(children []*Node, buffer io.Writer) (er return err } -func encodeChildrenSequentially(children []*Node, buffer io.Writer) (err error) { - for i, child := range children { - if child == nil { - continue - } - - err = encodeChild(child, buffer) - if err != nil { - return fmt.Errorf("encoding child at index %d: %w", i, err) - } - } - return nil -} - // encodeChild computes the Merkle value of the node // and then SCALE encodes it to the given buffer. -func encodeChild(child *Node, buffer io.Writer) (err error) { - merkleValue, err := child.CalculateMerkleValue() +func encodeChild(child *Node, maxInlineValue int, buffer io.Writer) (err error) { + merkleValue, err := child.CalculateMerkleValue(maxInlineValue) if err != nil { return fmt.Errorf("computing %s Merkle value: %w", child.Kind(), err) } diff --git a/internal/trie/node/branch_encode_test.go b/internal/trie/node/branch_encode_test.go index 7ef2ec4c0f..6698ca07ad 100644 --- a/internal/trie/node/branch_encode_test.go +++ b/internal/trie/node/branch_encode_test.go @@ -6,6 +6,7 @@ package node import ( "bytes" "io" + "math" "testing" "github.com/golang/mock/gomock" @@ -23,7 +24,7 @@ func Benchmark_encodeChildrenOpportunisticParallel(b *testing.B) { b.Run("", func(b *testing.B) { for i := 0; i < b.N; i++ { - _ = encodeChildrenOpportunisticParallel(children, io.Discard) + _ = encodeChildrenOpportunisticParallel(children, math.MaxInt, io.Discard) } }) } @@ -139,7 +140,7 @@ func Test_encodeChildrenOpportunisticParallel(t *testing.T) { previousCall = call } - err := encodeChildrenOpportunisticParallel(testCase.children, buffer) + err := encodeChildrenOpportunisticParallel(testCase.children, math.MaxInt, buffer) if testCase.wrappedErr != nil { assert.ErrorIs(t, err, testCase.wrappedErr) @@ -164,7 +165,7 @@ func Test_encodeChildrenOpportunisticParallel(t *testing.T) { // Note this may run in parallel or not depending on other tests // running in parallel. - err := encodeChildrenOpportunisticParallel(children, buffer) + err := encodeChildrenOpportunisticParallel(children, math.MaxInt, buffer) require.NoError(t, err) expectedBytes := []byte{ @@ -180,100 +181,6 @@ func Test_encodeChildrenOpportunisticParallel(t *testing.T) { }) } -func Test_encodeChildrenSequentially(t *testing.T) { - t.Parallel() - - testCases := map[string]struct { - children []*Node - writes []writeCall - wrappedErr error - errMessage string - }{ - "no_children": {}, - "first_child_not_nil": { - children: []*Node{ - {PartialKey: []byte{1}, StorageValue: []byte{2}}, - }, - writes: []writeCall{ - {written: []byte{16}}, - {written: []byte{65, 1, 4, 2}}, - }, - }, - "last_child_not_nil": { - children: []*Node{ - nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, - {PartialKey: []byte{1}, StorageValue: []byte{2}}, - }, - writes: []writeCall{ - {written: []byte{16}}, - {written: []byte{65, 1, 4, 2}}, - }, - }, - "first_two_children_not_nil": { - children: []*Node{ - {PartialKey: []byte{1}, StorageValue: []byte{2}}, - {PartialKey: []byte{3}, StorageValue: []byte{4}}, - }, - writes: []writeCall{ - {written: []byte{16}}, - {written: []byte{65, 1, 4, 2}}, - {written: []byte{16}}, - {written: []byte{65, 3, 4, 4}}, - }, - }, - "encoding_error": { - children: []*Node{ - nil, nil, nil, nil, - nil, nil, nil, nil, - nil, nil, nil, - {PartialKey: []byte{1}, StorageValue: []byte{2}}, - nil, nil, nil, nil, - }, - writes: []writeCall{ - { - written: []byte{16}, - err: errTest, - }, - }, - wrappedErr: errTest, - errMessage: "encoding child at index 11: " + - "scale encoding Merkle value: test error", - }, - } - - for name, testCase := range testCases { - testCase := testCase - t.Run(name, func(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - - buffer := NewMockWriter(ctrl) - var previousCall *gomock.Call - for _, write := range testCase.writes { - call := buffer.EXPECT(). - Write(write.written). - Return(write.n, write.err) - - if previousCall != nil { - call.After(previousCall) - } - previousCall = call - } - - err := encodeChildrenSequentially(testCase.children, buffer) - - if testCase.wrappedErr != nil { - assert.ErrorIs(t, err, testCase.wrappedErr) - assert.EqualError(t, err, testCase.errMessage) - } else { - require.NoError(t, err) - } - }) - } -} - func Test_encodeChild(t *testing.T) { t.Parallel() @@ -349,7 +256,7 @@ func Test_encodeChild(t *testing.T) { previousCall = call } - err := encodeChild(testCase.child, buffer) + err := encodeChild(testCase.child, math.MaxInt, buffer) if testCase.wrappedErr != nil { assert.ErrorIs(t, err, testCase.wrappedErr) diff --git a/internal/trie/node/encode.go b/internal/trie/node/encode.go index 87821dd534..1e5810ea34 100644 --- a/internal/trie/node/encode.go +++ b/internal/trie/node/encode.go @@ -4,7 +4,6 @@ package node import ( - "errors" "fmt" "github.com/ChainSafe/gossamer/internal/trie/codec" @@ -12,14 +11,12 @@ import ( "github.com/ChainSafe/gossamer/pkg/scale" ) -var ErrEncodeHashedValueTooShort = errors.New("hashed storage value too short") - // Encode encodes the node to the buffer given. // The encoding format is documented in the README.md // of this package, and specified in the Polkadot spec at // https://spec.polkadot.network/#sect-state-storage -func (n *Node) Encode(buffer Buffer) (err error) { - err = encodeHeader(n, buffer) +func (n *Node) Encode(buffer Buffer, maxInlineValue int) (err error) { + err = encodeHeader(n, maxInlineValue, buffer) if err != nil { return fmt.Errorf("cannot encode header: %w", err) } @@ -49,11 +46,12 @@ func (n *Node) Encode(buffer Buffer) (err error) { // even if it is empty. Do not encode if the branch is without value. // Note leaves and branches with value cannot have a `nil` storage value. if n.StorageValue != nil { - if n.IsHashedValue { - if len(n.StorageValue) != common.HashLength { - return fmt.Errorf("%w: expected %d, got: %d", ErrEncodeHashedValueTooShort, common.HashLength, len(n.StorageValue)) + if len(n.StorageValue) > maxInlineValue { + hashedValue, err := common.Blake2bHash(n.StorageValue) + if err != nil { + return fmt.Errorf("hashing storage value: %w", err) } - _, err := buffer.Write(n.StorageValue) + _, err = buffer.Write(hashedValue.ToBytes()) if err != nil { return fmt.Errorf("encoding hashed storage value: %w", err) } @@ -67,7 +65,7 @@ func (n *Node) Encode(buffer Buffer) (err error) { } if nodeIsBranch { - err = encodeChildrenOpportunisticParallel(n.Children, buffer) + err = encodeChildrenOpportunisticParallel(n.Children, maxInlineValue, buffer) if err != nil { return fmt.Errorf("cannot encode children of branch: %w", err) } diff --git a/internal/trie/node/encode_decode_test.go b/internal/trie/node/encode_decode_test.go index 71974ff907..b736dcccab 100644 --- a/internal/trie/node/encode_decode_test.go +++ b/internal/trie/node/encode_decode_test.go @@ -5,6 +5,7 @@ package node import ( "bytes" + "math" "testing" "github.com/stretchr/testify/assert" @@ -118,7 +119,7 @@ func Test_Branch_Encode_Decode(t *testing.T) { buffer := bytes.NewBuffer(nil) - err := testCase.branchToEncode.Encode(buffer) + err := testCase.branchToEncode.Encode(buffer, math.MaxInt) require.NoError(t, err) nodeVariant, partialKeyLength, err := decodeHeader(buffer) diff --git a/internal/trie/node/encode_test.go b/internal/trie/node/encode_test.go index 1593950cbd..63fb98d008 100644 --- a/internal/trie/node/encode_test.go +++ b/internal/trie/node/encode_test.go @@ -5,6 +5,7 @@ package node import ( "errors" + "math" "testing" "github.com/ChainSafe/gossamer/lib/common" @@ -24,17 +25,19 @@ var errTest = errors.New("test error") func Test_Node_Encode(t *testing.T) { t.Parallel() - hashedValue, err := common.Blake2bHash([]byte("test")) - assert.NoError(t, err) + largeValue := []byte("newvaluewithmorethan32byteslength") + hashedLargeValue := common.MustBlake2bHash(largeValue).ToBytes() testCases := map[string]struct { - node *Node - writes []writeCall - wrappedErr error - errMessage string + node *Node + maxInlineValueSize int + writes []writeCall + wrappedErr error + errMessage string }{ "nil_node": { - node: nil, + node: nil, + maxInlineValueSize: math.MaxInt, writes: []writeCall{ { written: []byte{emptyVariant.bits}, @@ -45,6 +48,7 @@ func Test_Node_Encode(t *testing.T) { node: &Node{ PartialKey: make([]byte, 1), }, + maxInlineValueSize: math.MaxInt, writes: []writeCall{ { written: []byte{leafVariant.bits | 1}, @@ -59,6 +63,7 @@ func Test_Node_Encode(t *testing.T) { PartialKey: []byte{1, 2, 3}, StorageValue: []byte{1}, }, + maxInlineValueSize: math.MaxInt, writes: []writeCall{ { written: []byte{leafVariant.bits | 3}, // partial key length 3 @@ -76,6 +81,7 @@ func Test_Node_Encode(t *testing.T) { PartialKey: []byte{1, 2, 3}, StorageValue: []byte{4, 5, 6}, }, + maxInlineValueSize: math.MaxInt, writes: []writeCall{ { written: []byte{leafVariant.bits | 3}, // partial key length 3 @@ -96,6 +102,7 @@ func Test_Node_Encode(t *testing.T) { PartialKey: []byte{1, 2, 3}, StorageValue: []byte{4, 5, 6}, }, + maxInlineValueSize: math.MaxInt, writes: []writeCall{ { written: []byte{leafVariant.bits | 3}, // partial key length 3 @@ -110,6 +117,7 @@ func Test_Node_Encode(t *testing.T) { PartialKey: []byte{1, 2, 3}, StorageValue: []byte{}, }, + maxInlineValueSize: math.MaxInt, writes: []writeCall{ {written: []byte{leafVariant.bits | 3}}, // partial key length 3 {written: []byte{0x01, 0x23}}, // partial key @@ -117,26 +125,27 @@ func Test_Node_Encode(t *testing.T) { {written: []byte{}}, // node storage value }, }, - "leaf_with_hashed_value_success": { + "leaf_with_value_gt_max_success": { node: &Node{ - PartialKey: []byte{1, 2, 3}, - StorageValue: hashedValue.ToBytes(), - IsHashedValue: true, + PartialKey: []byte{1, 2, 3}, + StorageValue: largeValue, }, + maxInlineValueSize: 32, writes: []writeCall{ { written: []byte{leafWithHashedValueVariant.bits | 3}, }, {written: []byte{0x01, 0x23}}, - {written: hashedValue.ToBytes()}, + {written: hashedLargeValue}, }, }, - "leaf_with_hashed_value_fail": { + "leaf_with_value_gt_max_fail": { node: &Node{ PartialKey: []byte{1, 2, 3}, - StorageValue: hashedValue.ToBytes(), + StorageValue: largeValue, IsHashedValue: true, }, + maxInlineValueSize: 32, writes: []writeCall{ { written: []byte{leafWithHashedValueVariant.bits | 3}, @@ -145,28 +154,13 @@ func Test_Node_Encode(t *testing.T) { written: []byte{0x01, 0x23}, }, { - written: hashedValue.ToBytes(), + written: hashedLargeValue, err: errTest, }, }, wrappedErr: errTest, errMessage: "encoding hashed storage value: test error", }, - "leaf_with_hashed_value_fail_too_short": { - node: &Node{ - PartialKey: []byte{1, 2, 3}, - StorageValue: []byte("tooshort"), - IsHashedValue: true, - }, - writes: []writeCall{ - { - written: []byte{leafWithHashedValueVariant.bits | 3}, - }, - {written: []byte{0x01, 0x23}}, - }, - wrappedErr: ErrEncodeHashedValueTooShort, - errMessage: "hashed storage value too short: expected 32, got: 8", - }, "branch_header_encoding_error": { node: &Node{ Children: make([]*Node, ChildrenCapacity), @@ -187,6 +181,7 @@ func Test_Node_Encode(t *testing.T) { PartialKey: []byte{1, 2, 3}, StorageValue: []byte{100}, }, + maxInlineValueSize: math.MaxInt, writes: []writeCall{ { // header written: []byte{branchWithValueVariant.bits | 3}, // partial key length 3 @@ -208,6 +203,7 @@ func Test_Node_Encode(t *testing.T) { nil, nil, nil, {PartialKey: []byte{11}, StorageValue: []byte{1}}, }, }, + maxInlineValueSize: math.MaxInt, writes: []writeCall{ { // header written: []byte{branchWithValueVariant.bits | 3}, // partial key length 3 @@ -232,6 +228,7 @@ func Test_Node_Encode(t *testing.T) { nil, nil, nil, {PartialKey: []byte{11}, StorageValue: []byte{1}}, }, }, + maxInlineValueSize: math.MaxInt, writes: []writeCall{ { // header written: []byte{branchWithValueVariant.bits | 3}, // partial key length 3 @@ -259,6 +256,7 @@ func Test_Node_Encode(t *testing.T) { nil, nil, nil, {PartialKey: []byte{11}, StorageValue: []byte{1}}, }, }, + maxInlineValueSize: math.MaxInt, writes: []writeCall{ { // header written: []byte{branchWithValueVariant.bits | 3}, // partial key length 3 @@ -291,6 +289,7 @@ func Test_Node_Encode(t *testing.T) { nil, nil, nil, {PartialKey: []byte{11}, StorageValue: []byte{1}}, }, }, + maxInlineValueSize: math.MaxInt, writes: []writeCall{ { // header written: []byte{branchWithValueVariant.bits | 3}, // partial key length 3 @@ -320,6 +319,7 @@ func Test_Node_Encode(t *testing.T) { nil, nil, nil, {PartialKey: []byte{11}, StorageValue: []byte{1}}, }, }, + maxInlineValueSize: math.MaxInt, writes: []writeCall{ { // header written: []byte{branchVariant.bits | 3}, // partial key length 3 @@ -338,16 +338,16 @@ func Test_Node_Encode(t *testing.T) { }, }, }, - "branch_with_hashed_value_success": { + "branch_with_value_gt_max_success": { node: &Node{ - PartialKey: []byte{1, 2, 3}, - StorageValue: hashedValue.ToBytes(), - IsHashedValue: true, + PartialKey: []byte{1, 2, 3}, + StorageValue: largeValue, Children: []*Node{ nil, nil, nil, {PartialKey: []byte{9}, StorageValue: []byte{1}}, nil, nil, nil, {PartialKey: []byte{11}, StorageValue: []byte{1}}, }, }, + maxInlineValueSize: 32, writes: []writeCall{ { // header written: []byte{branchWithHashedValueVariant.bits | 3}, // partial key length 3 @@ -359,7 +359,7 @@ func Test_Node_Encode(t *testing.T) { written: []byte{136, 0}, }, { - written: hashedValue.ToBytes(), + written: hashedLargeValue, }, { // first children written: []byte{16, 65, 9, 4, 1}, @@ -369,30 +369,6 @@ func Test_Node_Encode(t *testing.T) { }, }, }, - "branch_with_hashed_value_fail_too_short": { - node: &Node{ - PartialKey: []byte{1, 2, 3}, - StorageValue: []byte("tooshort"), - IsHashedValue: true, - Children: []*Node{ - nil, nil, nil, {PartialKey: []byte{9}, StorageValue: []byte{1}}, - nil, nil, nil, {PartialKey: []byte{11}, StorageValue: []byte{1}}, - }, - }, - writes: []writeCall{ - { // header - written: []byte{branchWithHashedValueVariant.bits | 3}, // partial key length 3 - }, - { // key LE - written: []byte{0x01, 0x23}, - }, - { // children bitmap - written: []byte{136, 0}, - }, - }, - wrappedErr: ErrEncodeHashedValueTooShort, - errMessage: "hashed storage value too short: expected 32, got: 8", - }, } for name, testCase := range testCases { @@ -414,7 +390,7 @@ func Test_Node_Encode(t *testing.T) { previousCall = call } - err := testCase.node.Encode(buffer) + err := testCase.node.Encode(buffer, testCase.maxInlineValueSize) if testCase.wrappedErr != nil { assert.ErrorIs(t, err, testCase.wrappedErr) diff --git a/internal/trie/node/hash.go b/internal/trie/node/hash.go index 595e06fc0f..1dfbca098d 100644 --- a/internal/trie/node/hash.go +++ b/internal/trie/node/hash.go @@ -55,12 +55,12 @@ func hashEncoding(encoding []byte, writer io.Writer) (err error) { } // CalculateMerkleValue returns the Merkle value of the non-root node. -func (n *Node) CalculateMerkleValue() (merkleValue []byte, err error) { +func (n *Node) CalculateMerkleValue(maxInlineValue int) (merkleValue []byte, err error) { if !n.Dirty && n.MerkleValue != nil { return n.MerkleValue, nil } - _, merkleValue, err = n.EncodeAndHash() + _, merkleValue, err = n.EncodeAndHash(maxInlineValue) if err != nil { return nil, fmt.Errorf("encoding and hashing node: %w", err) } @@ -69,13 +69,13 @@ func (n *Node) CalculateMerkleValue() (merkleValue []byte, err error) { } // CalculateRootMerkleValue returns the Merkle value of the root node. -func (n *Node) CalculateRootMerkleValue() (merkleValue []byte, err error) { +func (n *Node) CalculateRootMerkleValue(maxInlineValue int) (merkleValue []byte, err error) { const rootMerkleValueLength = 32 if !n.Dirty && len(n.MerkleValue) == rootMerkleValueLength { return n.MerkleValue, nil } - _, merkleValue, err = n.EncodeAndHashRoot() + _, merkleValue, err = n.EncodeAndHashRoot(maxInlineValue) if err != nil { return nil, fmt.Errorf("encoding and hashing root node: %w", err) } @@ -89,9 +89,9 @@ func (n *Node) CalculateRootMerkleValue() (merkleValue []byte, err error) { // TODO change this function to write to an encoding writer // and a merkle value writer, such that buffer sync pools can be used // by the caller. -func (n *Node) EncodeAndHash() (encoding, merkleValue []byte, err error) { +func (n *Node) EncodeAndHash(maxInlineValue int) (encoding, merkleValue []byte, err error) { encodingBuffer := bytes.NewBuffer(nil) - err = n.Encode(encodingBuffer) + err = n.Encode(encodingBuffer, maxInlineValue) if err != nil { return nil, nil, fmt.Errorf("encoding node: %w", err) } @@ -115,9 +115,9 @@ func (n *Node) EncodeAndHash() (encoding, merkleValue []byte, err error) { // TODO change this function to write to an encoding writer // and a merkle value writer, such that buffer sync pools can be used // by the caller. -func (n *Node) EncodeAndHashRoot() (encoding, merkleValue []byte, err error) { +func (n *Node) EncodeAndHashRoot(maxInlineValue int) (encoding, merkleValue []byte, err error) { encodingBuffer := bytes.NewBuffer(nil) - err = n.Encode(encodingBuffer) + err = n.Encode(encodingBuffer, maxInlineValue) if err != nil { return nil, nil, fmt.Errorf("encoding node: %w", err) } diff --git a/internal/trie/node/hash_test.go b/internal/trie/node/hash_test.go index 7c19b9d5de..980ee891c5 100644 --- a/internal/trie/node/hash_test.go +++ b/internal/trie/node/hash_test.go @@ -5,6 +5,7 @@ package node import ( "io" + "math" "testing" "github.com/golang/mock/gomock" @@ -197,7 +198,7 @@ func Test_Node_CalculateMerkleValue(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - merkleValue, err := testCase.node.CalculateMerkleValue() + merkleValue, err := testCase.node.CalculateMerkleValue(math.MaxInt) assert.ErrorIs(t, err, testCase.errWrapped) if testCase.errWrapped != nil { @@ -259,7 +260,7 @@ func Test_Node_CalculateRootMerkleValue(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - merkleValue, err := testCase.node.CalculateRootMerkleValue() + merkleValue, err := testCase.node.CalculateRootMerkleValue(math.MaxInt) assert.ErrorIs(t, err, testCase.errWrapped) if testCase.errWrapped != nil { @@ -346,7 +347,7 @@ func Test_Node_EncodeAndHash(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - encoding, hash, err := testCase.node.EncodeAndHash() + encoding, hash, err := testCase.node.EncodeAndHash(math.MaxInt) assert.ErrorIs(t, err, testCase.errWrapped) if testCase.errWrapped != nil { @@ -400,7 +401,7 @@ func Test_Node_EncodeAndHashRoot(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - encoding, hash, err := testCase.node.EncodeAndHashRoot() + encoding, hash, err := testCase.node.EncodeAndHashRoot(math.MaxInt) assert.ErrorIs(t, err, testCase.errWrapped) if testCase.errWrapped != nil { diff --git a/internal/trie/node/header.go b/internal/trie/node/header.go index 806fe5a020..30f910be5e 100644 --- a/internal/trie/node/header.go +++ b/internal/trie/node/header.go @@ -10,7 +10,7 @@ import ( ) // encodeHeader writes the encoded header for the node. -func encodeHeader(node *Node, writer io.Writer) (err error) { +func encodeHeader(node *Node, maxInlineValueSize int, writer io.Writer) (err error) { if node == nil { _, err = writer.Write([]byte{emptyVariant.bits}) return err @@ -21,17 +21,19 @@ func encodeHeader(node *Node, writer io.Writer) (err error) { panic(fmt.Sprintf("partial key length is too big: %d", partialKeyLength)) } + isHashedValue := len(node.StorageValue) > maxInlineValueSize + // Merge variant byte and partial key length together var nodeVariant variant if node.Kind() == Leaf { - if node.IsHashedValue { + if isHashedValue { nodeVariant = leafWithHashedValueVariant } else { nodeVariant = leafVariant } } else if node.StorageValue == nil { nodeVariant = branchVariant - } else if node.IsHashedValue { + } else if isHashedValue { nodeVariant = branchWithHashedValueVariant } else { nodeVariant = branchWithValueVariant diff --git a/internal/trie/node/header_test.go b/internal/trie/node/header_test.go index 6690ba1167..672fd2b45f 100644 --- a/internal/trie/node/header_test.go +++ b/internal/trie/node/header_test.go @@ -10,7 +10,6 @@ import ( "sort" "testing" - "github.com/ChainSafe/gossamer/lib/common" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -19,14 +18,14 @@ import ( func Test_encodeHeader(t *testing.T) { t.Parallel() - hashedValue, err := common.Blake2bHash([]byte("test")) - assert.NoError(t, err) + largeValue := []byte("newvaluewithmorethan32byteslength") testCases := map[string]struct { - node *Node - writes []writeCall - errWrapped error - errMessage string + node *Node + writes []writeCall + maxInlineValueSize int + errWrapped error + errMessage string }{ "branch_with_no_key": { node: &Node{ @@ -47,10 +46,10 @@ func Test_encodeHeader(t *testing.T) { }, "branch_with_hashed_value": { node: &Node{ - StorageValue: hashedValue.ToBytes(), - IsHashedValue: true, - Children: make([]*Node, ChildrenCapacity), + StorageValue: largeValue, + Children: make([]*Node, ChildrenCapacity), }, + maxInlineValueSize: 32, writes: []writeCall{ {written: []byte{branchWithHashedValueVariant.bits}}, }, @@ -126,9 +125,9 @@ func Test_encodeHeader(t *testing.T) { }, "leaf_with_hashed_value": { node: &Node{ - StorageValue: hashedValue.ToBytes(), - IsHashedValue: true, + StorageValue: largeValue, }, + maxInlineValueSize: 32, writes: []writeCall{ {written: []byte{leafWithHashedValueVariant.bits}}, }, @@ -138,6 +137,7 @@ func Test_encodeHeader(t *testing.T) { writes: []writeCall{ {written: []byte{leafVariant.bits}}, }, + maxInlineValueSize: 32, }, "leaf_with_key_of_length_30": { node: &Node{ @@ -240,7 +240,7 @@ func Test_encodeHeader(t *testing.T) { previousCall = call } - err := encodeHeader(testCase.node, writer) + err := encodeHeader(testCase.node, testCase.maxInlineValueSize, writer) assert.ErrorIs(t, err, testCase.errWrapped) if testCase.errWrapped != nil { @@ -258,7 +258,7 @@ func Test_encodeHeader(t *testing.T) { } assert.PanicsWithValue(t, "partial key length is too big: 65536", func() { - _ = encodeHeader(node, io.Discard) + _ = encodeHeader(node, 0, io.Discard) }) }) } @@ -293,7 +293,7 @@ func Test_encodeHeader_At_Maximum(t *testing.T) { PartialKey: make([]byte, keyLength), } - err := encodeHeader(node, buffer) + err := encodeHeader(node, math.MaxInt, buffer) require.NoError(t, err) assert.Equal(t, expectedBytes, buffer.Bytes()) diff --git a/lib/babe/helpers_test.go b/lib/babe/helpers_test.go index d7b5804b2c..0aa192bcb9 100644 --- a/lib/babe/helpers_test.go +++ b/lib/babe/helpers_test.go @@ -375,7 +375,7 @@ func newWestendLocalGenesisWithTrieAndHeader(t *testing.T) ( require.NoError(t, err) genesisHeader = *types.NewHeader(common.NewHash([]byte{0}), - genesisTrie.MustHash(), trie.EmptyHash, 0, types.NewDigest()) + genesisTrie.MustHash(trie.NoMaxInlineValueSize), trie.EmptyHash, 0, types.NewDigest()) return gen, genesisTrie, genesisHeader } @@ -394,7 +394,7 @@ func newWestendDevGenesisWithTrieAndHeader(t *testing.T) ( require.NoError(t, err) genesisHeader = *types.NewHeader(common.NewHash([]byte{0}), - genesisTrie.MustHash(), trie.EmptyHash, 0, types.NewDigest()) + genesisTrie.MustHash(trie.NoMaxInlineValueSize), trie.EmptyHash, 0, types.NewDigest()) return gen, genesisTrie, genesisHeader } diff --git a/lib/grandpa/helpers_integration_test.go b/lib/grandpa/helpers_integration_test.go index ed65b1b582..b0105ff546 100644 --- a/lib/grandpa/helpers_integration_test.go +++ b/lib/grandpa/helpers_integration_test.go @@ -203,7 +203,7 @@ func newWestendDevGenesisWithTrieAndHeader(t *testing.T) ( assert.NoError(t, err) parentHash := common.NewHash([]byte{0}) - stateRoot := genesisTrie.MustHash() + stateRoot := genesisTrie.MustHash(trie.NoMaxInlineValueSize) extrinsicRoot := trie.EmptyHash const number = 0 digest := types.NewDigest() diff --git a/lib/runtime/interfaces.go b/lib/runtime/interfaces.go index 2f813a21f6..131b23cad4 100644 --- a/lib/runtime/interfaces.go +++ b/lib/runtime/interfaces.go @@ -13,7 +13,7 @@ import ( type Storage interface { Put(key []byte, value []byte) (err error) Get(key []byte) []byte - Root() (common.Hash, error) + Root(maxInlineValueSize int) (common.Hash, error) SetChild(keyToChild []byte, child *trie.Trie) error SetChildStorage(keyToChild, key, value []byte) error GetChildStorage(keyToChild, key []byte) ([]byte, error) diff --git a/lib/runtime/storage/trie.go b/lib/runtime/storage/trie.go index b88e8fb56a..2a21377222 100644 --- a/lib/runtime/storage/trie.go +++ b/lib/runtime/storage/trie.go @@ -84,13 +84,13 @@ func (s *TrieState) Get(key []byte) []byte { } // MustRoot returns the trie's root hash. It panics if it fails to compute the root. -func (s *TrieState) MustRoot() common.Hash { - return s.t.MustHash() +func (s *TrieState) MustRoot(maxInlineValue int) common.Hash { + return s.t.MustHash(maxInlineValue) } // Root returns the trie's root hash -func (s *TrieState) Root() (common.Hash, error) { - return s.t.Hash() +func (s *TrieState) Root(maxInlineValue int) (common.Hash, error) { + return s.t.Hash(maxInlineValue) } // Has returns whether or not a key exists diff --git a/lib/runtime/storage/trie_test.go b/lib/runtime/storage/trie_test.go index 1e1075f550..e913b89435 100644 --- a/lib/runtime/storage/trie_test.go +++ b/lib/runtime/storage/trie_test.go @@ -104,8 +104,8 @@ func TestTrieState_Root(t *testing.T) { ts.Put([]byte(tc), []byte(tc)) } - expected := ts.MustRoot() - require.Equal(t, expected, ts.MustRoot()) + expected := ts.MustRoot(trie.NoMaxInlineValueSize) + require.Equal(t, expected, ts.MustRoot(trie.NoMaxInlineValueSize)) } ts := &TrieState{t: trie.NewEmptyTrie()} diff --git a/lib/runtime/wazero/imports.go b/lib/runtime/wazero/imports.go index 5ee7400dd1..bfa745029b 100644 --- a/lib/runtime/wazero/imports.go +++ b/lib/runtime/wazero/imports.go @@ -1243,7 +1243,7 @@ func ext_default_child_storage_root_version_1( return 0 } - childRoot, err := child.Hash() + childRoot, err := trie.V0.Hash(child) if err != nil { logger.Errorf("failed to encode child root: %s", err) return 0 @@ -1259,7 +1259,7 @@ func ext_default_child_storage_root_version_1( //export ext_default_child_storage_root_version_2 func ext_default_child_storage_root_version_2(ctx context.Context, m api.Module, childStorageKey uint64, - stateVersion uint32) (ptrSize uint64) { //skipcq: RVV-B0012 + version uint32) (ptrSize uint64) { //skipcq: RVV-B0012 rtCtx := ctx.Value(runtimeContextKey).(*runtime.Context) if rtCtx == nil { panic("nil runtime context") @@ -1271,7 +1271,17 @@ func ext_default_child_storage_root_version_2(ctx context.Context, m api.Module, return mustWrite(m, rtCtx.Allocator, emptyByteVectorEncoded) } - childRoot, err := child.Hash() + // TODO: fix this to get the right version + /*stateVersionBytes, _ := m.Memory().Read(version, 4) + stateVersion, err := trie.ParseVersion(binary.LittleEndian.Uint32(stateVersionBytes)) + if err != nil { + logger.Errorf("failed parsing state version: %s", err) + return 0 + }*/ + + stateVersion := trie.V1 + + childRoot, err := stateVersion.Hash(child) if err != nil { logger.Errorf("failed to encode child root: %s", err) return mustWrite(m, rtCtx.Allocator, emptyByteVectorEncoded) @@ -2237,7 +2247,7 @@ func ext_storage_root_version_1(ctx context.Context, m api.Module) uint64 { } storage := rtCtx.Storage - root, err := storage.Root() + root, err := storage.Root(trie.V0.MaxInlineValue()) if err != nil { logger.Errorf("failed to get storage root: %s", err) panic(err) @@ -2261,13 +2271,13 @@ func ext_storage_root_version_2(ctx context.Context, m api.Module, version uint3 storage := rtCtx.Storage stateVersionBytes, _ := m.Memory().Read(version, 4) - _, err := trie.ParseVersion(binary.LittleEndian.Uint32(stateVersionBytes)) + stateVersion, err := trie.ParseVersion(binary.LittleEndian.Uint32(stateVersionBytes)) if err != nil { logger.Errorf("failed parsing state version: %s", err) return mustWrite(m, rtCtx.Allocator, emptyByteVectorEncoded) } - root, err := storage.Root() + root, err := storage.Root(stateVersion.MaxInlineValue()) if err != nil { logger.Errorf("failed to get storage root: %s", err) panic(err) diff --git a/lib/runtime/wazero/imports_test.go b/lib/runtime/wazero/imports_test.go index 703546cd70..dd050c7b72 100644 --- a/lib/runtime/wazero/imports_test.go +++ b/lib/runtime/wazero/imports_test.go @@ -567,7 +567,7 @@ func Test_ext_trie_blake2_256_root_version_1(t *testing.T) { tt.Put([]byte("noot"), []byte("was")) tt.Put([]byte("here"), []byte("??")) - expected := tt.MustHash() + expected := tt.MustHash(trie.NoMaxInlineValueSize) require.Equal(t, expected[:], hash) } @@ -579,9 +579,11 @@ func Test_ext_trie_blake2_256_root_version_2(t *testing.T) { require.NoError(t, err) encInput[0] = encInput[0] >> 1 - stateVersion := uint32(trie.V1) + stateVersion := trie.V1 + + stateVersionInt := uint32(stateVersion) stateVersionBytes := make([]byte, 4) - binary.LittleEndian.PutUint32(stateVersionBytes, stateVersion) + binary.LittleEndian.PutUint32(stateVersionBytes, stateVersionInt) data := append(encInput, stateVersionBytes...) @@ -596,7 +598,7 @@ func Test_ext_trie_blake2_256_root_version_2(t *testing.T) { tt.Put([]byte("dimartiro"), []byte("was")) tt.Put([]byte("here"), []byte("??")) - expected := tt.MustHash() + expected := tt.MustHash(stateVersion.MaxInlineValue()) require.Equal(t, expected[:], hash) } @@ -647,10 +649,12 @@ func Test_ext_trie_blake2_256_verify_proof_version_1(t *testing.T) { memdb, err := database.NewPebble(tmp, true) require.NoError(t, err) + stateVersion := trie.V0 //Since this is Test_ext_trie_blake2_256_verify_proof_version_1 + otherTrie := trie.NewEmptyTrie() otherTrie.Put([]byte("simple"), []byte("cat")) - otherHash, err := otherTrie.Hash() + otherHash, err := stateVersion.Hash(otherTrie) require.NoError(t, err) tr := trie.NewEmptyTrie() @@ -663,7 +667,7 @@ func Test_ext_trie_blake2_256_verify_proof_version_1(t *testing.T) { err = tr.WriteDirty(memdb) require.NoError(t, err) - hash, err := tr.Hash() + hash, err := stateVersion.Hash(tr) require.NoError(t, err) keys := [][]byte{ @@ -739,10 +743,15 @@ func Test_ext_trie_blake2_256_verify_proof_version_2(t *testing.T) { memdb, err := database.NewPebble(tmp, true) require.NoError(t, err) + stateVersion := trie.V1 + stateVersionInt := uint32(trie.V1) + stateVersionBytes := make([]byte, 4) + binary.LittleEndian.PutUint32(stateVersionBytes, stateVersionInt) + otherTrie := trie.NewEmptyTrie() otherTrie.Put([]byte("simple"), []byte("cat")) - otherHash, err := otherTrie.Hash() + otherHash, err := stateVersion.Hash(otherTrie) require.NoError(t, err) tr := trie.NewEmptyTrie() @@ -755,7 +764,7 @@ func Test_ext_trie_blake2_256_verify_proof_version_2(t *testing.T) { err = tr.WriteDirty(memdb) require.NoError(t, err) - hash, err := tr.Hash() + hash, err := stateVersion.Hash(tr) require.NoError(t, err) keys := [][]byte{ @@ -815,10 +824,6 @@ func Test_ext_trie_blake2_256_verify_proof_version_2(t *testing.T) { require.NoError(t, err) args = append(args, valueEnc...) - stateVersion := uint32(trie.V1) - stateVersionBytes := make([]byte, 4) - binary.LittleEndian.PutUint32(stateVersionBytes, stateVersion) - args = append(args, stateVersionBytes...) res, err := inst.Exec("rtm_ext_trie_blake2_256_verify_proof_version_2", args) @@ -1283,7 +1288,9 @@ func Test_ext_default_child_storage_root_version_1(t *testing.T) { child, err := inst.Context.Storage.GetChild(testChildKey) require.NoError(t, err) - rootHash, err := child.Hash() + stateVersion := trie.V0 + + rootHash, err := stateVersion.Hash(child) require.NoError(t, err) encChildKey, err := scale.Marshal(testChildKey) @@ -1306,6 +1313,8 @@ func Test_ext_default_child_storage_root_version_1(t *testing.T) { func Test_ext_default_child_storage_root_version_2(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) + stateVersion := trie.V1 + err := inst.Context.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) require.NoError(t, err) @@ -1315,7 +1324,7 @@ func Test_ext_default_child_storage_root_version_2(t *testing.T) { child, err := inst.Context.Storage.GetChild(testChildKey) require.NoError(t, err) - rootHash, err := child.Hash() + rootHash, err := stateVersion.Hash(child) require.NoError(t, err) encChildKey, err := scale.Marshal(testChildKey) @@ -1323,8 +1332,8 @@ func Test_ext_default_child_storage_root_version_2(t *testing.T) { encKey, err := scale.Marshal(testKey) require.NoError(t, err) - stateVersion := uint32(trie.V1) - encVersion, err := scale.Marshal(&stateVersion) + stateVersionInt := uint32(stateVersion) + encVersion, err := scale.Marshal(&stateVersionInt) require.NoError(t, err) data := append(encChildKey, encKey...) diff --git a/lib/runtime/wazero/instance_test.go b/lib/runtime/wazero/instance_test.go index 511dfc3ed8..8a45a2df3d 100644 --- a/lib/runtime/wazero/instance_test.go +++ b/lib/runtime/wazero/instance_test.go @@ -230,7 +230,7 @@ func TestWestendRuntime_ValidateTransaction(t *testing.T) { genesisHeader := &types.Header{ Number: 0, - StateRoot: genTrie.MustHash(), + StateRoot: trie.V0.MustHash(genTrie), // Get right state version from runtime } extHex := runtime.NewTestExtrinsic(t, rt, genesisHeader.Hash(), genesisHeader.Hash(), @@ -435,7 +435,7 @@ func TestInstance_BadSignature_WestendBlock8077850(t *testing.T) { genesisHeader := &types.Header{ Number: 0, - StateRoot: genTrie.MustHash(), + StateRoot: trie.V0.MustHash(genTrie), // Use right version from runtime } header := &types.Header{ @@ -461,7 +461,7 @@ func TestInstance_BadSignature_WestendBlock8077850(t *testing.T) { genesisHeader := &types.Header{ Number: 0, - StateRoot: genTrie.MustHash(), + StateRoot: trie.V0.MustHash(genTrie), // Use right version from runtime } header := &types.Header{ @@ -634,7 +634,7 @@ func TestInstance_ApplyExtrinsic_WestendRuntime(t *testing.T) { genesisHeader := &types.Header{ Number: 0, - StateRoot: genTrie.MustHash(), + StateRoot: trie.V0.MustHash(genTrie), // Use right version from runtime } header := &types.Header{ ParentHash: genesisHeader.Hash(), @@ -675,7 +675,7 @@ func TestInstance_ExecuteBlock_PolkadotRuntime_PolkadotBlock1(t *testing.T) { require.NoError(t, err) expectedGenesisRoot := common.MustHexToHash("0x29d0d972cd27cbc511e9589fcb7a4506d5eb6a9e8df205f00472e5ab354a4e17") - require.Equal(t, expectedGenesisRoot, genTrie.MustHash()) + require.Equal(t, expectedGenesisRoot, trie.V0.MustHash(genTrie)) // set state to genesis state genState := storage.NewTrieState(&genTrie) @@ -725,7 +725,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1(t *testing.T) { require.NoError(t, err) expectedGenesisRoot := common.MustHexToHash("0xb0006203c3a6e6bd2c6a17b1d4ae8ca49a31da0f4579da950b127774b44aef6b") - require.Equal(t, expectedGenesisRoot, genTrie.MustHash()) + require.Equal(t, expectedGenesisRoot, trie.V0.MustHash(genTrie)) // set state to genesis state genState := storage.NewTrieState(&genTrie) @@ -771,7 +771,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1(t *testing.T) { func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock3784(t *testing.T) { gossTrie3783 := newTrieFromPairs(t, "../test_data/kusama/block3783.out") expectedRoot := common.MustHexToHash("0x948338bc0976aee78879d559a1f42385407e5a481b05a91d2a9386aa7507e7a0") - require.Equal(t, expectedRoot, gossTrie3783.MustHash()) + require.Equal(t, expectedRoot, trie.V0.MustHash(*gossTrie3783)) // set state to genesis state state3783 := storage.NewTrieState(gossTrie3783) @@ -817,7 +817,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock3784(t *testing.T) { func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock901442(t *testing.T) { ksmTrie901441 := newTrieFromPairs(t, "../test_data/kusama/block901441.out") expectedRoot := common.MustHexToHash("0x3a2ef7ee032f5810160bb8f3ffe3e3377bb6f2769ee9f79a5425973347acd504") - require.Equal(t, expectedRoot, ksmTrie901441.MustHash()) + require.Equal(t, expectedRoot, trie.V0.MustHash(*ksmTrie901441)) // set state to genesis state state901441 := storage.NewTrieState(ksmTrie901441) @@ -863,7 +863,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock901442(t *testing.T) { func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1377831(t *testing.T) { ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block1377830.out") expectedRoot := common.MustHexToHash("0xe4de6fecda9e9e35f937d159665cf984bc1a68048b6c78912de0aeb6bd7f7e99") - require.Equal(t, expectedRoot, ksmTrie.MustHash()) + require.Equal(t, expectedRoot, trie.V0.MustHash(*ksmTrie)) // set state to genesis state state := storage.NewTrieState(ksmTrie) @@ -909,7 +909,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1377831(t *testing.T) { func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1482003(t *testing.T) { ksmTrie := newTrieFromPairs(t, "../test_data/kusama/block1482002.out") expectedRoot := common.MustHexToHash("0x09f9ca28df0560c2291aa16b56e15e07d1e1927088f51356d522722aa90ca7cb") - require.Equal(t, expectedRoot, ksmTrie.MustHash()) + require.Equal(t, expectedRoot, trie.V0.MustHash(*ksmTrie)) // set state to genesis state state := storage.NewTrieState(ksmTrie) @@ -956,7 +956,7 @@ func TestInstance_ExecuteBlock_KusamaRuntime_KusamaBlock1482003(t *testing.T) { func TestInstance_ExecuteBlock_PolkadotBlock1089328(t *testing.T) { dotTrie := newTrieFromPairs(t, "../test_data/polkadot/block1089327.json") expectedRoot := common.MustHexToHash("0x87ed9ebe7fb645d3b5b0255cc16e78ed022d9fbb52486105436e15a74557535b") - require.Equal(t, expectedRoot, dotTrie.MustHash()) + require.Equal(t, expectedRoot, trie.V0.MustHash(*dotTrie)) // set state to genesis state state := storage.NewTrieState(dotTrie) diff --git a/lib/trie/child_storage.go b/lib/trie/child_storage.go index 00dd28d20b..2f3cd008da 100644 --- a/lib/trie/child_storage.go +++ b/lib/trie/child_storage.go @@ -19,7 +19,7 @@ var ErrChildTrieDoesNotExist = errors.New("child trie does not exist") // A child trie is added as a node (K, V) in the main trie. K is the child storage key // associated to the child trie, and V is the root hash of the child trie. func (t *Trie) SetChild(keyToChild []byte, child *Trie) error { - childHash, err := child.Hash() + childHash, err := child.Hash(NoMaxInlineValueSize) if err != nil { return err } @@ -62,7 +62,7 @@ func (t *Trie) PutIntoChild(keyToChild, key, value []byte) error { } } - origChildHash, err := child.Hash() + origChildHash, err := child.Hash(NoMaxInlineValueSize) if err != nil { return err } @@ -116,7 +116,7 @@ func (t *Trie) ClearFromChild(keyToChild, key []byte) error { return fmt.Errorf("%w at key 0x%x%x", ErrChildTrieDoesNotExist, ChildStorageKeyPrefix, keyToChild) } - origChildHash, err := child.Hash() + origChildHash, err := child.Hash(NoMaxInlineValueSize) if err != nil { return err } diff --git a/lib/trie/child_storage_test.go b/lib/trie/child_storage_test.go index c77d8f7a58..ad5d9f099d 100644 --- a/lib/trie/child_storage_test.go +++ b/lib/trie/child_storage_test.go @@ -90,7 +90,7 @@ func TestPutAndGetFromChild(t *testing.T) { func TestChildTrieHashAfterClear(t *testing.T) { trieThatHoldsAChildTrie := NewEmptyTrie() - originalEmptyHash := trieThatHoldsAChildTrie.MustHash() + originalEmptyHash := DefaultStateVersion.MustHash(*trieThatHoldsAChildTrie) keyToChild := []byte("crowdloan") keyInChild := []byte("account-alice") @@ -103,7 +103,7 @@ func TestChildTrieHashAfterClear(t *testing.T) { // the parent trie hash SHOULT NOT BE EQUAL to the original // empty hash since it contains a value - require.NotEqual(t, originalEmptyHash, trieThatHoldsAChildTrie.MustHash()) + require.NotEqual(t, originalEmptyHash, DefaultStateVersion.MustHash(*trieThatHoldsAChildTrie)) // ensure the value is inside the child trie valueStored, err := trieThatHoldsAChildTrie.GetFromChild(keyToChild, keyInChild) @@ -116,6 +116,6 @@ func TestChildTrieHashAfterClear(t *testing.T) { // the parent trie hash SHOULD BE EQUAL to the original // empty hash since now it does not have any other value in it - require.Equal(t, originalEmptyHash, trieThatHoldsAChildTrie.MustHash()) + require.Equal(t, originalEmptyHash, DefaultStateVersion.MustHash(*trieThatHoldsAChildTrie)) } diff --git a/lib/trie/database.go b/lib/trie/database.go index e4335dd2cd..b8e8a5b76c 100644 --- a/lib/trie/database.go +++ b/lib/trie/database.go @@ -61,7 +61,7 @@ func (t *Trie) loadNode(db db.DBGetter, n *Node) error { if len(merkleValue) < 32 { // node has already been loaded inline // just set its encoding - _, err := child.CalculateMerkleValue() + _, err := child.CalculateMerkleValue(NoMaxInlineValueSize) if err != nil { return fmt.Errorf("merkle value: %w", err) } @@ -109,7 +109,7 @@ func (t *Trie) loadNode(db db.DBGetter, n *Node) error { return fmt.Errorf("failed to load child trie with root hash=%s: %w", rootHash, err) } - hash, err := childTrie.Hash() + hash, err := childTrie.Hash(NoMaxInlineValueSize) if err != nil { return fmt.Errorf("cannot hash chilld trie at key 0x%x: %w", key, err) } @@ -287,9 +287,9 @@ func (t *Trie) writeDirtyNode(db db.DBPutter, n *Node) (err error) { var encoding, merkleValue []byte if n == t.root { - encoding, merkleValue, err = n.EncodeAndHashRoot() + encoding, merkleValue, err = n.EncodeAndHashRoot(V0.MaxInlineValue()) //TODO: solve this with right version } else { - encoding, merkleValue, err = n.EncodeAndHash() + encoding, merkleValue, err = n.EncodeAndHash(V0.MaxInlineValue()) //TODO: solve this with right version } if err != nil { return fmt.Errorf( @@ -364,9 +364,9 @@ func (t *Trie) getInsertedNodeHashesAtNode(n *Node, nodeHashes map[common.Hash]s var merkleValue []byte if n == t.root { - merkleValue, err = n.CalculateRootMerkleValue() + merkleValue, err = n.CalculateRootMerkleValue(NoMaxInlineValueSize) } else { - merkleValue, err = n.CalculateMerkleValue() + merkleValue, err = n.CalculateMerkleValue(NoMaxInlineValueSize) } if err != nil { return fmt.Errorf("calculating Merkle value: %w", err) diff --git a/lib/trie/database_test.go b/lib/trie/database_test.go index 2a43b5826d..a0cde1701c 100644 --- a/lib/trie/database_test.go +++ b/lib/trie/database_test.go @@ -24,7 +24,7 @@ func Test_Trie_Store_Load(t *testing.T) { const size = 1000 trie, _ := makeSeededTrie(t, size) - rootHash := trie.MustHash() + rootHash := V0.MustHash(*trie) db := newTestDB(t) err := trie.WriteDirty(db) @@ -65,7 +65,7 @@ func Test_Trie_WriteDirty_Put(t *testing.T) { err := trie.WriteDirty(db) require.NoError(t, err) - rootHash := trie.MustHash() + rootHash := DefaultStateVersion.MustHash(*trie) valueFromDB, err := GetFromDB(db, rootHash, key) require.NoError(t, err) assert.Equalf(t, value, valueFromDB, "for key=%x", key) @@ -85,7 +85,7 @@ func Test_Trie_WriteDirty_Put(t *testing.T) { err = trie.WriteDirty(db) require.NoError(t, err) - rootHash := trie.MustHash() + rootHash := DefaultStateVersion.MustHash(*trie) // Verify the trie in database is also modified. trieFromDB := NewEmptyTrie() @@ -119,7 +119,7 @@ func Test_Trie_WriteDirty_Delete(t *testing.T) { deletedKeys[string(keyToDelete)] = struct{}{} } - rootHash := trie.MustHash() + rootHash := DefaultStateVersion.MustHash(*trie) trieFromDB := NewEmptyTrie() err = trieFromDB.Load(db, rootHash) @@ -157,7 +157,7 @@ func Test_Trie_WriteDirty_ClearPrefix(t *testing.T) { require.NoError(t, err) } - rootHash := trie.MustHash() + rootHash := DefaultStateVersion.MustHash(*trie) trieFromDB := NewEmptyTrie() err = trieFromDB.Load(db, rootHash) @@ -277,7 +277,7 @@ func Test_GetFromDB(t *testing.T) { err := trie.WriteDirty(db) require.NoError(t, err) - root := trie.MustHash() + root := DefaultStateVersion.MustHash(*trie) for keyString, expectedValue := range keyValues { key := []byte(keyString) @@ -325,7 +325,7 @@ func Test_Trie_PutChild_Store_Load(t *testing.T) { require.NoError(t, err) trieFromDB := NewEmptyTrie() - err = trieFromDB.Load(db, trie.MustHash()) + err = trieFromDB.Load(db, DefaultStateVersion.MustHash(*trie)) require.NoError(t, err) assert.Equal(t, trie.childTries, trieFromDB.childTries) diff --git a/lib/trie/genesis.go b/lib/trie/genesis.go index c4e6e21737..d2205208b1 100644 --- a/lib/trie/genesis.go +++ b/lib/trie/genesis.go @@ -12,7 +12,7 @@ import ( // GenesisBlock creates a genesis block from the trie. func (t *Trie) GenesisBlock() (genesisHeader types.Header, err error) { - rootHash, err := t.Hash() + rootHash, err := t.Hash(NoMaxInlineValueSize) if err != nil { return genesisHeader, fmt.Errorf("root hashing trie: %w", err) } diff --git a/lib/trie/proof/generate.go b/lib/trie/proof/generate.go index 861a8787d6..fed898c3cf 100644 --- a/lib/trie/proof/generate.go +++ b/lib/trie/proof/generate.go @@ -88,7 +88,7 @@ func walkRoot(root *node.Node, fullKey []byte) ( // Note we do not use sync.Pool buffers since we would have // to copy it so it persists in encodedProofNodes. encodingBuffer := bytes.NewBuffer(nil) - err = root.Encode(encodingBuffer) + err = root.Encode(encodingBuffer, trie.NoMaxInlineValueSize) if err != nil { return nil, fmt.Errorf("encode node: %w", err) } @@ -133,7 +133,7 @@ func walk(parent *node.Node, fullKey []byte) ( // Note we do not use sync.Pool buffers since we would have // to copy it so it persists in encodedProofNodes. encodingBuffer := bytes.NewBuffer(nil) - err = parent.Encode(encodingBuffer) + err = parent.Encode(encodingBuffer, trie.NoMaxInlineValueSize) if err != nil { return nil, fmt.Errorf("encode node: %w", err) } diff --git a/lib/trie/proof/helpers_test.go b/lib/trie/proof/helpers_test.go index de3d6fe25a..30d6cb488a 100644 --- a/lib/trie/proof/helpers_test.go +++ b/lib/trie/proof/helpers_test.go @@ -5,6 +5,7 @@ package proof import ( "bytes" + "math" "math/rand" "testing" @@ -23,7 +24,7 @@ func padRightChildren(slice []*node.Node) (paddedSlice []*node.Node) { func encodeNode(t *testing.T, node node.Node) (encoded []byte) { t.Helper() buffer := bytes.NewBuffer(nil) - err := node.Encode(buffer) + err := node.Encode(buffer, math.MaxInt) require.NoError(t, err) return buffer.Bytes() } diff --git a/lib/trie/proof/proof_test.go b/lib/trie/proof/proof_test.go index 954c48cb7f..fbc0b45ed9 100644 --- a/lib/trie/proof/proof_test.go +++ b/lib/trie/proof/proof_test.go @@ -6,6 +6,7 @@ package proof import ( "encoding/hex" "fmt" + "math" "testing" "github.com/ChainSafe/gossamer/internal/database" @@ -32,7 +33,7 @@ func Test_Generate_Verify(t *testing.T) { trie.Put([]byte(key), []byte(value)) } - rootHash, err := trie.Hash() + rootHash, err := trie.Hash(math.MaxInt) require.NoError(t, err) db, err := database.NewPebble("", true) diff --git a/lib/trie/trie.go b/lib/trie/trie.go index 688bd12005..99152be798 100644 --- a/lib/trie/trie.go +++ b/lib/trie/trie.go @@ -139,13 +139,6 @@ func (t *Trie) registerDeletedNodeHash(node *Node, // since the last trie snapshot. nodeHash := common.NewHash(node.MerkleValue) pendingDeltas.RecordDeleted(nodeHash) - - // If this node contains a hashed value we have to remove the value node from the db too - if node.IsHashedValue { - // TODO: fix this! we could have an issue since we are using the same shared value node for N hashed nodes - // One way to fix it is having a reference counter for each value node - pendingDeltas.RecordDeleted(common.NewHash(node.StorageValue)) - } } return nil @@ -196,8 +189,8 @@ func (t *Trie) RootNode() *Node { // MustHash returns the hashed root of the trie. // It panics if it fails to hash the root node. -func (t *Trie) MustHash() common.Hash { - h, err := t.Hash() +func (t *Trie) MustHash(maxInlineValue int) common.Hash { + h, err := t.Hash(maxInlineValue) if err != nil { panic(err) } @@ -206,12 +199,12 @@ func (t *Trie) MustHash() common.Hash { } // Hash returns the hashed root of the trie. -func (t *Trie) Hash() (rootHash common.Hash, err error) { +func (t *Trie) Hash(maxInlineValue int) (rootHash common.Hash, err error) { if t.root == nil { return EmptyHash, nil } - merkleValue, err := t.root.CalculateRootMerkleValue() + merkleValue, err := t.root.CalculateRootMerkleValue(maxInlineValue) if err != nil { return rootHash, err } @@ -1413,12 +1406,12 @@ func (t *Trie) ensureMerkleValueIsCalculated(parent *Node) (err error) { } if parent == t.root { - _, err = parent.CalculateRootMerkleValue() + _, err = parent.CalculateRootMerkleValue(NoMaxInlineValueSize) if err != nil { return fmt.Errorf("calculating Merkle value of root node: %w", err) } } else { - _, err = parent.CalculateMerkleValue() + _, err = parent.CalculateMerkleValue(NoMaxInlineValueSize) if err != nil { return fmt.Errorf("calculating Merkle value of node: %w", err) } diff --git a/lib/trie/trie_endtoend_test.go b/lib/trie/trie_endtoend_test.go index eb73aecaa7..ab7a58b022 100644 --- a/lib/trie/trie_endtoend_test.go +++ b/lib/trie/trie_endtoend_test.go @@ -349,13 +349,13 @@ func TestDelete(t *testing.T) { ssTrie := trie.Snapshot() // Get the Trie root hash for all the 3 tries. - tHash, err := trie.Hash() + tHash, err := DefaultStateVersion.Hash(trie) require.NoError(t, err) - dcTrieHash, err := dcTrie.Hash() + dcTrieHash, err := DefaultStateVersion.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err := ssTrie.Hash() + ssTrieHash, err := DefaultStateVersion.Hash(ssTrie) require.NoError(t, err) // Root hash for all the 3 tries should be equal. @@ -376,13 +376,13 @@ func TestDelete(t *testing.T) { } // Get the updated root hash of all tries. - tHash, err = trie.Hash() + tHash, err = DefaultStateVersion.Hash(trie) require.NoError(t, err) - dcTrieHash, err = dcTrie.Hash() + dcTrieHash, err = DefaultStateVersion.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err = ssTrie.Hash() + ssTrieHash, err = DefaultStateVersion.Hash(ssTrie) require.NoError(t, err) // Only the current trie should have a different root hash since it is updated. @@ -432,13 +432,13 @@ func TestClearPrefix(t *testing.T) { ssTrie := trie.Snapshot() // Get the Trie root hash for all the 3 tries. - tHash, err := trie.Hash() + tHash, err := DefaultStateVersion.Hash(trie) require.NoError(t, err) - dcTrieHash, err := dcTrie.Hash() + dcTrieHash, err := DefaultStateVersion.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err := ssTrie.Hash() + ssTrieHash, err := DefaultStateVersion.Hash(ssTrie) require.NoError(t, err) // Root hash for all the 3 tries should be equal. @@ -464,13 +464,13 @@ func TestClearPrefix(t *testing.T) { } // Get the updated root hash of all tries. - tHash, err = trie.Hash() + tHash, err = DefaultStateVersion.Hash(trie) require.NoError(t, err) - dcTrieHash, err = dcTrie.Hash() + dcTrieHash, err = DefaultStateVersion.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err = ssTrie.Hash() + ssTrieHash, err = DefaultStateVersion.Hash(ssTrie) require.NoError(t, err) // Only the current trie should have a different root hash since it is updated. @@ -489,13 +489,13 @@ func TestClearPrefix_Small(t *testing.T) { ssTrie := trie.Snapshot() // Get the Trie root hash for all the 3 tries. - tHash, err := trie.Hash() + tHash, err := DefaultStateVersion.Hash(trie) require.NoError(t, err) - dcTrieHash, err := dcTrie.Hash() + dcTrieHash, err := DefaultStateVersion.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err := ssTrie.Hash() + ssTrieHash, err := DefaultStateVersion.Hash(ssTrie) require.NoError(t, err) // Root hash for all the 3 tries should be equal. @@ -522,13 +522,13 @@ func TestClearPrefix_Small(t *testing.T) { require.Equal(t, expectedRoot, ssTrie.root) // Get the updated root hash of all tries. - tHash, err = trie.Hash() + tHash, err = DefaultStateVersion.Hash(trie) require.NoError(t, err) - dcTrieHash, err = dcTrie.Hash() + dcTrieHash, err = DefaultStateVersion.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err = ssTrie.Hash() + ssTrieHash, err = DefaultStateVersion.Hash(ssTrie) require.NoError(t, err) require.Equal(t, tHash, dcTrieHash) @@ -600,7 +600,10 @@ func TestTrie_ClearPrefixVsDelete(t *testing.T) { trieClearPrefix.ClearPrefix(prefix) - require.Equal(t, trieClearPrefix.MustHash(), trieDelete.MustHash()) + trieClearPrefixHash := DefaultStateVersion.MustHash(*trieClearPrefix) + trieDeleteHash := DefaultStateVersion.MustHash(*trieDelete) + + require.Equal(t, trieClearPrefixHash, trieDeleteHash) } } } @@ -633,8 +636,12 @@ func TestSnapshot(t *testing.T) { newTrie := parentTrie.Snapshot() newTrie.Put(tests[0].key, tests[0].value) - require.Equal(t, expectedTrie.MustHash(), newTrie.MustHash()) - require.NotEqual(t, parentTrie.MustHash(), newTrie.MustHash()) + expectedTrieHash := DefaultStateVersion.MustHash(*expectedTrie) + newTrieHash := DefaultStateVersion.MustHash(*newTrie) + parentTrieHash := DefaultStateVersion.MustHash(*parentTrie) + + require.Equal(t, expectedTrieHash, newTrieHash) + require.NotEqual(t, parentTrieHash, newTrieHash) } func Test_Trie_NextKey_Random(t *testing.T) { @@ -686,7 +693,7 @@ func Benchmark_Trie_Hash(b *testing.B) { } b.StartTimer() - _, err := trie.Hash() + _, err := DefaultStateVersion.Hash(trie) b.StopTimer() require.NoError(b, err) @@ -767,9 +774,11 @@ func TestTrie_ConcurrentSnapshotWrites(t *testing.T) { finishWg.Wait() for i := 0; i < workers; i++ { - assert.Equal(t, - expectedTries[i].MustHash(), - snapshotedTries[i].MustHash()) + assert.Equal( + t, + DefaultStateVersion.MustHash(*expectedTries[i]), + DefaultStateVersion.MustHash(*snapshotedTries[i]), + ) } } @@ -950,13 +959,13 @@ func TestTrie_ClearPrefixLimitSnapshot(t *testing.T) { ssTrie := trieClearPrefix.Snapshot() // Get the Trie root hash for all the 3 tries. - tHash, err := trieClearPrefix.Hash() + tHash, err := DefaultStateVersion.Hash(trieClearPrefix) require.NoError(t, err) - dcTrieHash, err := dcTrie.Hash() + dcTrieHash, err := DefaultStateVersion.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err := ssTrie.Hash() + ssTrieHash, err := DefaultStateVersion.Hash(ssTrie) require.NoError(t, err) // Root hash for all the 3 tries should be equal. @@ -992,13 +1001,13 @@ func TestTrie_ClearPrefixLimitSnapshot(t *testing.T) { } // Get the updated root hash of all tries. - tHash, err = trieClearPrefix.Hash() + tHash, err = DefaultStateVersion.Hash(trieClearPrefix) require.NoError(t, err) - dcTrieHash, err = dcTrie.Hash() + dcTrieHash, err = DefaultStateVersion.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err = ssTrie.Hash() + ssTrieHash, err = DefaultStateVersion.Hash(ssTrie) require.NoError(t, err) // If node got deleted then root hash must be updated else it has same root hash. @@ -1033,7 +1042,7 @@ func Test_encodeRoot_fuzz(t *testing.T) { assert.Equal(t, value, retrievedValue) } buffer := bytes.NewBuffer(nil) - err := trie.root.Encode(buffer) + err := trie.root.Encode(buffer, V0.MaxInlineValue()) require.NoError(t, err) require.NotEmpty(t, buffer.Bytes()) } diff --git a/lib/trie/trie_test.go b/lib/trie/trie_test.go index 2666583daf..3e19a939a1 100644 --- a/lib/trie/trie_test.go +++ b/lib/trie/trie_test.go @@ -276,12 +276,6 @@ func Test_Trie_registerDeletedNodeHash(t *testing.T) { StorageValue: []byte{2}, } - someSmallNodeWithHashedValue := &Node{ - PartialKey: []byte{1}, - StorageValue: common.MustBlake2bHash([]byte("hash")).ToBytes(), - IsHashedValue: true, - } - testCases := map[string]struct { trie Trie node *Node @@ -327,27 +321,6 @@ func Test_Trie_registerDeletedNodeHash(t *testing.T) { pendingDeltas: newDeltas(), expectedPendingDeltas: newDeltas("0x98fcd66ba312c29ef193052fd0c14c6e38b158bd5c0235064594cacc1ab5965d"), }, - "clean_v1_node_with_hashed_subvalue": { - node: someSmallNodeWithHashedValue, - trie: Trie{root: someSmallNodeWithHashedValue}, - pendingDeltas: newDeltas(), - expectedPendingDeltas: newDeltas( - "0x4269e2a9cdf14dbb1f94ea10e5e65be796e940f7043bcb71276682712e6730d5", - "0x97edaa69596438136dcd128553e904bc03f526426f727d270b69841fb6cf50d3", - ), - expectedTrie: Trie{ - root: &Node{ - PartialKey: []byte{1}, - StorageValue: common.MustBlake2bHash([]byte("hash")).ToBytes(), - IsHashedValue: true, - MerkleValue: []byte{ - 0x42, 0x69, 0xe2, 0xa9, 0xcd, 0xf1, 0x4d, 0xbb, - 0x1f, 0x94, 0xea, 0x10, 0xe5, 0xe6, 0x5b, 0xe7, - 0x96, 0xe9, 0x40, 0xf7, 0x04, 0x3b, 0xcb, 0x71, - 0x27, 0x66, 0x82, 0x71, 0x2e, 0x67, 0x30, 0xd5}, - }, - }, - }, } for name, testCase := range testCases { @@ -490,7 +463,7 @@ func Test_Trie_MustHash(t *testing.T) { var trie Trie - hash := trie.MustHash() + hash := DefaultStateVersion.MustHash(trie) expectedHash := common.Hash{ 0x3, 0x17, 0xa, 0x2e, 0x75, 0x97, 0xb7, 0xb7, @@ -587,7 +560,7 @@ func Test_Trie_Hash(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - hash, err := testCase.trie.Hash() + hash, err := DefaultStateVersion.Hash(&testCase.trie) assert.ErrorIs(t, err, testCase.errWrapped) if testCase.errWrapped != nil { diff --git a/lib/trie/version.go b/lib/trie/version.go index a760fa0d94..9691c72d63 100644 --- a/lib/trie/version.go +++ b/lib/trie/version.go @@ -57,7 +57,8 @@ func (v Version) String() string { } } -func (v Version) maxInlineValue() int { +// MaxInlineValueSize returns the maximum size of a value to be inlined in the trie node +func (v Version) MaxInlineValue() int { switch v { case V0: return NoMaxInlineValueSize @@ -79,7 +80,17 @@ func (v Version) Root(entries Entries) (common.Hash, error) { } } - return t.Hash() + return t.Hash(v.MaxInlineValue()) +} + +// Root returns the root hash of the trie built using the given entries +func (v Version) Hash(t *Trie) (common.Hash, error) { + return t.Hash(v.MaxInlineValue()) +} + +// Root returns the root hash of the trie built using the given entries +func (v Version) MustHash(t Trie) common.Hash { + return t.MustHash(v.MaxInlineValue()) } // ParseVersion parses a state trie version string. From a16d3deb568c2b0139ed5de9435f46560441e7b4 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 7 Nov 2023 17:34:15 -0300 Subject: [PATCH 099/128] Fix import state command --- cmd/gossamer/commands/import_state.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/gossamer/commands/import_state.go b/cmd/gossamer/commands/import_state.go index e7b5991a62..a12f137ba9 100644 --- a/cmd/gossamer/commands/import_state.go +++ b/cmd/gossamer/commands/import_state.go @@ -12,6 +12,7 @@ import ( ) func init() { + ImportStateCmd.Flags().String("chain", "", "Chain id used to load default configuration for specified chain") ImportStateCmd.Flags().String("state-file", "", "Path to JSON file consisting of key-value pairs") ImportStateCmd.Flags().String("header-file", "", "Path to JSON file of block header corresponding to the given state") ImportStateCmd.Flags().Uint64("first-slot", 0, "The first BABE slot of the network") @@ -25,7 +26,7 @@ var ImportStateCmd = &cobra.Command{ in the form of key-value pairs to be imported. Input can be generated by using the RPC function state_getPairs. Example: - gossamer import-state --state-file state.json --state-version v1 --header-file header.json --first-slot `, //nolint:lll + gossamer import-state --state-file state.json --header-file header.json --first-slot `, RunE: func(cmd *cobra.Command, args []string) error { return execImportState(cmd) }, From eda5f6c7b0e6790d45c2a463211689a049ee57a2 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 7 Nov 2023 17:37:30 -0300 Subject: [PATCH 100/128] Use default state version in helpers test --- dot/core/helpers_test.go | 4 ++-- lib/runtime/genesis.go | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/dot/core/helpers_test.go b/dot/core/helpers_test.go index f25ed6c7ad..a94e57fe09 100644 --- a/dot/core/helpers_test.go +++ b/dot/core/helpers_test.go @@ -57,7 +57,7 @@ func createTestService(t *testing.T, genesisFilePath string, require.NoError(t, err) genesisHeader := &types.Header{ - StateRoot: genesisTrie.MustHash(trie.NoMaxInlineValueSize), + StateRoot: trie.DefaultStateVersion.MustHash(genesisTrie), Number: 0, } @@ -271,7 +271,7 @@ func newWestendLocalWithTrieAndHeader(t *testing.T) ( require.NoError(t, err) parentHash := common.NewHash([]byte{0}) - stateRoot := genesisTrie.MustHash(trie.NoMaxInlineValueSize) + stateRoot := trie.DefaultStateVersion.MustHash(genesisTrie) extrinsicRoot := trie.EmptyHash const number = 0 digest := types.NewDigest() diff --git a/lib/runtime/genesis.go b/lib/runtime/genesis.go index 54fc27f824..5912886bf1 100644 --- a/lib/runtime/genesis.go +++ b/lib/runtime/genesis.go @@ -26,8 +26,6 @@ func NewTrieFromGenesis(gen genesis.Genesis) (tr trie.Trie, err error) { ErrGenesisTopNotFound, gen.Name) } - // TODO: I'll set it to V0 since our goal is to work on westend first but we have to revisit it in the future - // to get the version from the runtime tr, err = trie.LoadFromMap(keyValues) if err != nil { return tr, fmt.Errorf("loading genesis top key values into trie: %w", err) From a3db05c9a86b36bcec8d58b3affacc05a17c7e8f Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 7 Nov 2023 17:54:51 -0300 Subject: [PATCH 101/128] Use default state version in helpers test --- dot/digest/helpers_test.go | 2 +- dot/helpers_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dot/digest/helpers_test.go b/dot/digest/helpers_test.go index 4f4fc81f9a..dc9a661001 100644 --- a/dot/digest/helpers_test.go +++ b/dot/digest/helpers_test.go @@ -28,7 +28,7 @@ func newWestendDevGenesisWithTrieAndHeader(t *testing.T) ( require.NoError(t, err) parentHash := common.NewHash([]byte{0}) - stateRoot := genesisTrie.MustHash(trie.NoMaxInlineValueSize) + stateRoot := trie.DefaultStateVersion.MustHash(genesisTrie) extrinsicRoot := trie.EmptyHash const number = 0 digest := types.NewDigest() diff --git a/dot/helpers_test.go b/dot/helpers_test.go index 2889f70af4..fa6265d493 100644 --- a/dot/helpers_test.go +++ b/dot/helpers_test.go @@ -42,7 +42,7 @@ func newWestendDevGenesisWithTrieAndHeader(t *testing.T) ( require.NoError(t, err) parentHash := common.NewHash([]byte{0}) - stateRoot := genesisTrie.MustHash(trie.NoMaxInlineValueSize) + stateRoot := trie.DefaultStateVersion.MustHash(genesisTrie) extrinsicRoot := trie.EmptyHash const number = 0 digest := types.NewDigest() From 50f9616d9c36bbc0a0bfad3b112841bc1801d031 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 7 Nov 2023 17:56:42 -0300 Subject: [PATCH 102/128] Revert import integration tests change --- dot/import_integration_test.go | 40 +++++++++++++++++++++++++++++++++ dot/import_test.go | 41 ---------------------------------- 2 files changed, 40 insertions(+), 41 deletions(-) diff --git a/dot/import_integration_test.go b/dot/import_integration_test.go index ca235afef6..534f50b4cf 100644 --- a/dot/import_integration_test.go +++ b/dot/import_integration_test.go @@ -18,6 +18,46 @@ import ( "github.com/stretchr/testify/require" ) +func Test_newTrieFromPairs(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + filename string + want common.Hash + err error + }{ + { + name: "no_arguments", + err: errors.New("read .: is a directory"), + want: common.Hash{}, + }, + { + name: "working example", + filename: setupStateFile(t), + want: common.MustHexToHash("0x09f9ca28df0560c2291aa16b56e15e07d1e1927088f51356d522722aa90ca7cb"), + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := newTrieFromPairs(tt.filename) + if tt.err != nil { + assert.EqualError(t, err, tt.err.Error()) + } else { + assert.NoError(t, err) + } + if tt.want.IsEmpty() { + assert.Nil(t, got) + } else { + assert.Equal(t, tt.want, got.MustHash()) + } + }) + } +} + func TestNewHeaderFromFile(t *testing.T) { fp := setupHeaderFile(t) header, err := newHeaderFromFile(fp) diff --git a/dot/import_test.go b/dot/import_test.go index c5a5fcf72e..38b54f8db7 100644 --- a/dot/import_test.go +++ b/dot/import_test.go @@ -12,7 +12,6 @@ import ( "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -140,43 +139,3 @@ func Test_newHeaderFromFile(t *testing.T) { }) } } - -func Test_newTrieFromPairs(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - filename string - want common.Hash - err error - }{ - { - name: "no_arguments", - err: errors.New("read .: is a directory"), - want: common.Hash{}, - }, - { - name: "working example", - filename: setupStateFile(t), - want: common.MustHexToHash("0x09f9ca28df0560c2291aa16b56e15e07d1e1927088f51356d522722aa90ca7cb"), - }, - } - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - got, err := newTrieFromPairs(tt.filename) - if tt.err != nil { - assert.EqualError(t, err, tt.err.Error()) - } else { - assert.NoError(t, err) - } - if tt.want.IsEmpty() { - assert.Nil(t, got) - } else { - assert.Equal(t, tt.want, got.MustHash(trie.NoMaxInlineValueSize)) - } - }) - } -} From ffa71576ccaee82a1246a5cb07a7b17035c7e8a8 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 7 Nov 2023 18:10:48 -0300 Subject: [PATCH 103/128] Lint --- dot/import_integration_test.go | 3 ++- dot/node_integration_test.go | 3 ++- dot/rpc/helpers_test.go | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/dot/import_integration_test.go b/dot/import_integration_test.go index 534f50b4cf..dda734f49a 100644 --- a/dot/import_integration_test.go +++ b/dot/import_integration_test.go @@ -13,6 +13,7 @@ import ( "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/internal/log" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/pkg/scale" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -52,7 +53,7 @@ func Test_newTrieFromPairs(t *testing.T) { if tt.want.IsEmpty() { assert.Nil(t, got) } else { - assert.Equal(t, tt.want, got.MustHash()) + assert.Equal(t, tt.want, trie.DefaultStateVersion.MustHash(*got)) } }) } diff --git a/dot/node_integration_test.go b/dot/node_integration_test.go index 63026a8688..736e155d46 100644 --- a/dot/node_integration_test.go +++ b/dot/node_integration_test.go @@ -388,7 +388,8 @@ func TestInitNode_LoadStorageRoot(t *testing.T) { expected, err := trie.LoadFromMap(gen.GenesisFields().Raw["top"]) require.NoError(t, err) - expectedRoot, err := expected.Hash(trie.NoMaxInlineValueSize) + // TODO: get trie state version from runtime + expectedRoot, err := trie.V0.Hash(&expected) require.NoError(t, err) coreServiceInterface := node.ServiceRegistry.Get(&core.Service{}) diff --git a/dot/rpc/helpers_test.go b/dot/rpc/helpers_test.go index ef962f0f46..5f7ac8c8da 100644 --- a/dot/rpc/helpers_test.go +++ b/dot/rpc/helpers_test.go @@ -28,7 +28,7 @@ func newWestendDevGenesisWithTrieAndHeader(t *testing.T) ( require.NoError(t, err) parentHash := common.NewHash([]byte{0}) - stateRoot := genesisTrie.MustHash(trie.NoMaxInlineValueSize) + stateRoot := trie.DefaultStateVersion.MustHash(genesisTrie) extrinsicRoot := trie.EmptyHash const number = 0 digest := types.NewDigest() From c72deb87938fa04e4c13a7f08331b6a380869637 Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 13 Nov 2023 11:06:42 -0300 Subject: [PATCH 104/128] Change default version to 0 --- dot/core/helpers_test.go | 4 +-- dot/digest/helpers_test.go | 2 +- dot/helpers_test.go | 2 +- dot/rpc/helpers_test.go | 2 +- lib/trie/child_storage_test.go | 6 ++-- lib/trie/database.go | 6 ++-- lib/trie/database_test.go | 12 +++---- lib/trie/trie_endtoend_test.go | 64 +++++++++++++++++----------------- lib/trie/trie_test.go | 4 +-- 9 files changed, 52 insertions(+), 50 deletions(-) diff --git a/dot/core/helpers_test.go b/dot/core/helpers_test.go index a94e57fe09..4c47371437 100644 --- a/dot/core/helpers_test.go +++ b/dot/core/helpers_test.go @@ -57,7 +57,7 @@ func createTestService(t *testing.T, genesisFilePath string, require.NoError(t, err) genesisHeader := &types.Header{ - StateRoot: trie.DefaultStateVersion.MustHash(genesisTrie), + StateRoot: trie.V0.MustHash(genesisTrie), Number: 0, } @@ -271,7 +271,7 @@ func newWestendLocalWithTrieAndHeader(t *testing.T) ( require.NoError(t, err) parentHash := common.NewHash([]byte{0}) - stateRoot := trie.DefaultStateVersion.MustHash(genesisTrie) + stateRoot := trie.V0.MustHash(genesisTrie) extrinsicRoot := trie.EmptyHash const number = 0 digest := types.NewDigest() diff --git a/dot/digest/helpers_test.go b/dot/digest/helpers_test.go index dc9a661001..f603664a62 100644 --- a/dot/digest/helpers_test.go +++ b/dot/digest/helpers_test.go @@ -28,7 +28,7 @@ func newWestendDevGenesisWithTrieAndHeader(t *testing.T) ( require.NoError(t, err) parentHash := common.NewHash([]byte{0}) - stateRoot := trie.DefaultStateVersion.MustHash(genesisTrie) + stateRoot := trie.V0.MustHash(genesisTrie) extrinsicRoot := trie.EmptyHash const number = 0 digest := types.NewDigest() diff --git a/dot/helpers_test.go b/dot/helpers_test.go index fa6265d493..1da49be908 100644 --- a/dot/helpers_test.go +++ b/dot/helpers_test.go @@ -42,7 +42,7 @@ func newWestendDevGenesisWithTrieAndHeader(t *testing.T) ( require.NoError(t, err) parentHash := common.NewHash([]byte{0}) - stateRoot := trie.DefaultStateVersion.MustHash(genesisTrie) + stateRoot := trie.V0.MustHash(genesisTrie) extrinsicRoot := trie.EmptyHash const number = 0 digest := types.NewDigest() diff --git a/dot/rpc/helpers_test.go b/dot/rpc/helpers_test.go index 5f7ac8c8da..af7835518a 100644 --- a/dot/rpc/helpers_test.go +++ b/dot/rpc/helpers_test.go @@ -28,7 +28,7 @@ func newWestendDevGenesisWithTrieAndHeader(t *testing.T) ( require.NoError(t, err) parentHash := common.NewHash([]byte{0}) - stateRoot := trie.DefaultStateVersion.MustHash(genesisTrie) + stateRoot := trie.V0.MustHash(genesisTrie) extrinsicRoot := trie.EmptyHash const number = 0 digest := types.NewDigest() diff --git a/lib/trie/child_storage_test.go b/lib/trie/child_storage_test.go index ad5d9f099d..d75dda0eaf 100644 --- a/lib/trie/child_storage_test.go +++ b/lib/trie/child_storage_test.go @@ -90,7 +90,7 @@ func TestPutAndGetFromChild(t *testing.T) { func TestChildTrieHashAfterClear(t *testing.T) { trieThatHoldsAChildTrie := NewEmptyTrie() - originalEmptyHash := DefaultStateVersion.MustHash(*trieThatHoldsAChildTrie) + originalEmptyHash := V0.MustHash(*trieThatHoldsAChildTrie) keyToChild := []byte("crowdloan") keyInChild := []byte("account-alice") @@ -103,7 +103,7 @@ func TestChildTrieHashAfterClear(t *testing.T) { // the parent trie hash SHOULT NOT BE EQUAL to the original // empty hash since it contains a value - require.NotEqual(t, originalEmptyHash, DefaultStateVersion.MustHash(*trieThatHoldsAChildTrie)) + require.NotEqual(t, originalEmptyHash, V0.MustHash(*trieThatHoldsAChildTrie)) // ensure the value is inside the child trie valueStored, err := trieThatHoldsAChildTrie.GetFromChild(keyToChild, keyInChild) @@ -116,6 +116,6 @@ func TestChildTrieHashAfterClear(t *testing.T) { // the parent trie hash SHOULD BE EQUAL to the original // empty hash since now it does not have any other value in it - require.Equal(t, originalEmptyHash, DefaultStateVersion.MustHash(*trieThatHoldsAChildTrie)) + require.Equal(t, originalEmptyHash, V0.MustHash(*trieThatHoldsAChildTrie)) } diff --git a/lib/trie/database.go b/lib/trie/database.go index b8e8a5b76c..667ccec5df 100644 --- a/lib/trie/database.go +++ b/lib/trie/database.go @@ -286,10 +286,12 @@ func (t *Trie) writeDirtyNode(db db.DBPutter, n *Node) (err error) { } var encoding, merkleValue []byte + // TODO: I'm sure we don't need to store the encoded now, we can try storing the (key,value) only but it needs + // some refactor and testing. In the meantime we can store the encoded node using the v0 encoding if n == t.root { - encoding, merkleValue, err = n.EncodeAndHashRoot(V0.MaxInlineValue()) //TODO: solve this with right version + encoding, merkleValue, err = n.EncodeAndHashRoot(V0.MaxInlineValue()) } else { - encoding, merkleValue, err = n.EncodeAndHash(V0.MaxInlineValue()) //TODO: solve this with right version + encoding, merkleValue, err = n.EncodeAndHash(V0.MaxInlineValue()) } if err != nil { return fmt.Errorf( diff --git a/lib/trie/database_test.go b/lib/trie/database_test.go index a0cde1701c..0e2b6bb4b5 100644 --- a/lib/trie/database_test.go +++ b/lib/trie/database_test.go @@ -65,7 +65,7 @@ func Test_Trie_WriteDirty_Put(t *testing.T) { err := trie.WriteDirty(db) require.NoError(t, err) - rootHash := DefaultStateVersion.MustHash(*trie) + rootHash := V0.MustHash(*trie) valueFromDB, err := GetFromDB(db, rootHash, key) require.NoError(t, err) assert.Equalf(t, value, valueFromDB, "for key=%x", key) @@ -85,7 +85,7 @@ func Test_Trie_WriteDirty_Put(t *testing.T) { err = trie.WriteDirty(db) require.NoError(t, err) - rootHash := DefaultStateVersion.MustHash(*trie) + rootHash := V0.MustHash(*trie) // Verify the trie in database is also modified. trieFromDB := NewEmptyTrie() @@ -119,7 +119,7 @@ func Test_Trie_WriteDirty_Delete(t *testing.T) { deletedKeys[string(keyToDelete)] = struct{}{} } - rootHash := DefaultStateVersion.MustHash(*trie) + rootHash := V0.MustHash(*trie) trieFromDB := NewEmptyTrie() err = trieFromDB.Load(db, rootHash) @@ -157,7 +157,7 @@ func Test_Trie_WriteDirty_ClearPrefix(t *testing.T) { require.NoError(t, err) } - rootHash := DefaultStateVersion.MustHash(*trie) + rootHash := V0.MustHash(*trie) trieFromDB := NewEmptyTrie() err = trieFromDB.Load(db, rootHash) @@ -277,7 +277,7 @@ func Test_GetFromDB(t *testing.T) { err := trie.WriteDirty(db) require.NoError(t, err) - root := DefaultStateVersion.MustHash(*trie) + root := V0.MustHash(*trie) for keyString, expectedValue := range keyValues { key := []byte(keyString) @@ -325,7 +325,7 @@ func Test_Trie_PutChild_Store_Load(t *testing.T) { require.NoError(t, err) trieFromDB := NewEmptyTrie() - err = trieFromDB.Load(db, DefaultStateVersion.MustHash(*trie)) + err = trieFromDB.Load(db, V0.MustHash(*trie)) require.NoError(t, err) assert.Equal(t, trie.childTries, trieFromDB.childTries) diff --git a/lib/trie/trie_endtoend_test.go b/lib/trie/trie_endtoend_test.go index ab7a58b022..6871d9a06a 100644 --- a/lib/trie/trie_endtoend_test.go +++ b/lib/trie/trie_endtoend_test.go @@ -349,13 +349,13 @@ func TestDelete(t *testing.T) { ssTrie := trie.Snapshot() // Get the Trie root hash for all the 3 tries. - tHash, err := DefaultStateVersion.Hash(trie) + tHash, err := V0.Hash(trie) require.NoError(t, err) - dcTrieHash, err := DefaultStateVersion.Hash(dcTrie) + dcTrieHash, err := V0.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err := DefaultStateVersion.Hash(ssTrie) + ssTrieHash, err := V0.Hash(ssTrie) require.NoError(t, err) // Root hash for all the 3 tries should be equal. @@ -376,13 +376,13 @@ func TestDelete(t *testing.T) { } // Get the updated root hash of all tries. - tHash, err = DefaultStateVersion.Hash(trie) + tHash, err = V0.Hash(trie) require.NoError(t, err) - dcTrieHash, err = DefaultStateVersion.Hash(dcTrie) + dcTrieHash, err = V0.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err = DefaultStateVersion.Hash(ssTrie) + ssTrieHash, err = V0.Hash(ssTrie) require.NoError(t, err) // Only the current trie should have a different root hash since it is updated. @@ -432,13 +432,13 @@ func TestClearPrefix(t *testing.T) { ssTrie := trie.Snapshot() // Get the Trie root hash for all the 3 tries. - tHash, err := DefaultStateVersion.Hash(trie) + tHash, err := V0.Hash(trie) require.NoError(t, err) - dcTrieHash, err := DefaultStateVersion.Hash(dcTrie) + dcTrieHash, err := V0.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err := DefaultStateVersion.Hash(ssTrie) + ssTrieHash, err := V0.Hash(ssTrie) require.NoError(t, err) // Root hash for all the 3 tries should be equal. @@ -464,13 +464,13 @@ func TestClearPrefix(t *testing.T) { } // Get the updated root hash of all tries. - tHash, err = DefaultStateVersion.Hash(trie) + tHash, err = V0.Hash(trie) require.NoError(t, err) - dcTrieHash, err = DefaultStateVersion.Hash(dcTrie) + dcTrieHash, err = V0.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err = DefaultStateVersion.Hash(ssTrie) + ssTrieHash, err = V0.Hash(ssTrie) require.NoError(t, err) // Only the current trie should have a different root hash since it is updated. @@ -489,13 +489,13 @@ func TestClearPrefix_Small(t *testing.T) { ssTrie := trie.Snapshot() // Get the Trie root hash for all the 3 tries. - tHash, err := DefaultStateVersion.Hash(trie) + tHash, err := V0.Hash(trie) require.NoError(t, err) - dcTrieHash, err := DefaultStateVersion.Hash(dcTrie) + dcTrieHash, err := V0.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err := DefaultStateVersion.Hash(ssTrie) + ssTrieHash, err := V0.Hash(ssTrie) require.NoError(t, err) // Root hash for all the 3 tries should be equal. @@ -522,13 +522,13 @@ func TestClearPrefix_Small(t *testing.T) { require.Equal(t, expectedRoot, ssTrie.root) // Get the updated root hash of all tries. - tHash, err = DefaultStateVersion.Hash(trie) + tHash, err = V0.Hash(trie) require.NoError(t, err) - dcTrieHash, err = DefaultStateVersion.Hash(dcTrie) + dcTrieHash, err = V0.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err = DefaultStateVersion.Hash(ssTrie) + ssTrieHash, err = V0.Hash(ssTrie) require.NoError(t, err) require.Equal(t, tHash, dcTrieHash) @@ -600,8 +600,8 @@ func TestTrie_ClearPrefixVsDelete(t *testing.T) { trieClearPrefix.ClearPrefix(prefix) - trieClearPrefixHash := DefaultStateVersion.MustHash(*trieClearPrefix) - trieDeleteHash := DefaultStateVersion.MustHash(*trieDelete) + trieClearPrefixHash := V0.MustHash(*trieClearPrefix) + trieDeleteHash := V0.MustHash(*trieDelete) require.Equal(t, trieClearPrefixHash, trieDeleteHash) } @@ -636,9 +636,9 @@ func TestSnapshot(t *testing.T) { newTrie := parentTrie.Snapshot() newTrie.Put(tests[0].key, tests[0].value) - expectedTrieHash := DefaultStateVersion.MustHash(*expectedTrie) - newTrieHash := DefaultStateVersion.MustHash(*newTrie) - parentTrieHash := DefaultStateVersion.MustHash(*parentTrie) + expectedTrieHash := V0.MustHash(*expectedTrie) + newTrieHash := V0.MustHash(*newTrie) + parentTrieHash := V0.MustHash(*parentTrie) require.Equal(t, expectedTrieHash, newTrieHash) require.NotEqual(t, parentTrieHash, newTrieHash) @@ -693,7 +693,7 @@ func Benchmark_Trie_Hash(b *testing.B) { } b.StartTimer() - _, err := DefaultStateVersion.Hash(trie) + _, err := V0.Hash(trie) b.StopTimer() require.NoError(b, err) @@ -776,8 +776,8 @@ func TestTrie_ConcurrentSnapshotWrites(t *testing.T) { for i := 0; i < workers; i++ { assert.Equal( t, - DefaultStateVersion.MustHash(*expectedTries[i]), - DefaultStateVersion.MustHash(*snapshotedTries[i]), + V0.MustHash(*expectedTries[i]), + V0.MustHash(*snapshotedTries[i]), ) } } @@ -959,13 +959,13 @@ func TestTrie_ClearPrefixLimitSnapshot(t *testing.T) { ssTrie := trieClearPrefix.Snapshot() // Get the Trie root hash for all the 3 tries. - tHash, err := DefaultStateVersion.Hash(trieClearPrefix) + tHash, err := V0.Hash(trieClearPrefix) require.NoError(t, err) - dcTrieHash, err := DefaultStateVersion.Hash(dcTrie) + dcTrieHash, err := V0.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err := DefaultStateVersion.Hash(ssTrie) + ssTrieHash, err := V0.Hash(ssTrie) require.NoError(t, err) // Root hash for all the 3 tries should be equal. @@ -1001,13 +1001,13 @@ func TestTrie_ClearPrefixLimitSnapshot(t *testing.T) { } // Get the updated root hash of all tries. - tHash, err = DefaultStateVersion.Hash(trieClearPrefix) + tHash, err = V0.Hash(trieClearPrefix) require.NoError(t, err) - dcTrieHash, err = DefaultStateVersion.Hash(dcTrie) + dcTrieHash, err = V0.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err = DefaultStateVersion.Hash(ssTrie) + ssTrieHash, err = V0.Hash(ssTrie) require.NoError(t, err) // If node got deleted then root hash must be updated else it has same root hash. diff --git a/lib/trie/trie_test.go b/lib/trie/trie_test.go index 3e19a939a1..6dfbb19064 100644 --- a/lib/trie/trie_test.go +++ b/lib/trie/trie_test.go @@ -463,7 +463,7 @@ func Test_Trie_MustHash(t *testing.T) { var trie Trie - hash := DefaultStateVersion.MustHash(trie) + hash := V0.MustHash(trie) expectedHash := common.Hash{ 0x3, 0x17, 0xa, 0x2e, 0x75, 0x97, 0xb7, 0xb7, @@ -560,7 +560,7 @@ func Test_Trie_Hash(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - hash, err := DefaultStateVersion.Hash(&testCase.trie) + hash, err := V0.Hash(&testCase.trie) assert.ErrorIs(t, err, testCase.errWrapped) if testCase.errWrapped != nil { From 93c29f77b6bc1cd933decc6429e95ec263771384 Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 13 Nov 2023 11:14:32 -0300 Subject: [PATCH 105/128] Fix Test_newTrieFromPairs --- dot/import_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dot/import_integration_test.go b/dot/import_integration_test.go index dda734f49a..b26ae60ec5 100644 --- a/dot/import_integration_test.go +++ b/dot/import_integration_test.go @@ -53,7 +53,7 @@ func Test_newTrieFromPairs(t *testing.T) { if tt.want.IsEmpty() { assert.Nil(t, got) } else { - assert.Equal(t, tt.want, trie.DefaultStateVersion.MustHash(*got)) + assert.Equal(t, tt.want, trie.V0.MustHash(*got)) } }) } From 9f66a11ef340ad0adb4f4749018df7a1d8999136 Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 13 Nov 2023 11:23:23 -0300 Subject: [PATCH 106/128] Add one more test case --- lib/trie/version_test.go | 41 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/lib/trie/version_test.go b/lib/trie/version_test.go index 5f894277e3..033a9cc0d8 100644 --- a/lib/trie/version_test.go +++ b/lib/trie/version_test.go @@ -117,6 +117,47 @@ func Test_ParseVersion(t *testing.T) { } } +func Test_Version_MaxInlineValue(t *testing.T) { + t.Parallel() + + testCases := map[string]struct { + version Version + max int + panicMessage string + }{ + "v0": { + version: V0, + max: NoMaxInlineValueSize, + }, + "v1": { + version: V1, + max: V1MaxInlineValueSize, + }, + "invalid": { + version: Version(99), + max: 0, + panicMessage: "unknown version 99", + }, + } + + for name, testCase := range testCases { + testCase := testCase + t.Run(name, func(t *testing.T) { + t.Parallel() + + if testCase.panicMessage != "" { + assert.PanicsWithValue(t, testCase.panicMessage, func() { + _ = testCase.version.String() + }) + return + } + + maxInline := testCase.version.MaxInlineValue() + assert.Equal(t, testCase.max, maxInline) + }) + } +} + func Test_Version_Root(t *testing.T) { t.Parallel() From a47970a472267de578e37d5bbae332bd79b20c17 Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 13 Nov 2023 11:30:11 -0300 Subject: [PATCH 107/128] Fix version test --- lib/trie/version_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/trie/version_test.go b/lib/trie/version_test.go index 033a9cc0d8..c092a4cb6c 100644 --- a/lib/trie/version_test.go +++ b/lib/trie/version_test.go @@ -147,7 +147,7 @@ func Test_Version_MaxInlineValue(t *testing.T) { if testCase.panicMessage != "" { assert.PanicsWithValue(t, testCase.panicMessage, func() { - _ = testCase.version.String() + _ = testCase.version.MaxInlineValue() }) return } From 297067462971b6f0ddf5681a348ecbd45b1cc1f4 Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 13 Nov 2023 11:44:32 -0300 Subject: [PATCH 108/128] Fix deepsource checks --- lib/runtime/wazero/imports_test.go | 15 +++++++++++---- lib/trie/db/db.go | 2 +- lib/trie/version.go | 6 +++--- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/lib/runtime/wazero/imports_test.go b/lib/runtime/wazero/imports_test.go index dd050c7b72..a140f8961d 100644 --- a/lib/runtime/wazero/imports_test.go +++ b/lib/runtime/wazero/imports_test.go @@ -585,7 +585,9 @@ func Test_ext_trie_blake2_256_root_version_2(t *testing.T) { stateVersionBytes := make([]byte, 4) binary.LittleEndian.PutUint32(stateVersionBytes, stateVersionInt) - data := append(encInput, stateVersionBytes...) + data := []byte{} + data = append(data, encInput...) + data = append(data, stateVersionBytes...) res, err := inst.Exec("rtm_ext_trie_blake2_256_root_version_2", data) require.NoError(t, err) @@ -631,7 +633,9 @@ func Test_ext_trie_blake2_256_ordered_root_version_2(t *testing.T) { stateVersionBytes := make([]byte, 4) binary.LittleEndian.PutUint32(stateVersionBytes, stateVersion) - data := append(encValues, stateVersionBytes...) + data := []byte{} + data = append(data, encValues...) + data = append(data, stateVersionBytes...) res, err := inst.Exec("rtm_ext_trie_blake2_256_ordered_root_version_2", data) require.NoError(t, err) @@ -649,7 +653,8 @@ func Test_ext_trie_blake2_256_verify_proof_version_1(t *testing.T) { memdb, err := database.NewPebble(tmp, true) require.NoError(t, err) - stateVersion := trie.V0 //Since this is Test_ext_trie_blake2_256_verify_proof_version_1 + // Since this is Test_ext_trie_blake2_256_verify_proof_version_1, we use trie.V0 + stateVersion := trie.V0 otherTrie := trie.NewEmptyTrie() otherTrie.Put([]byte("simple"), []byte("cat")) @@ -1336,7 +1341,9 @@ func Test_ext_default_child_storage_root_version_2(t *testing.T) { encVersion, err := scale.Marshal(&stateVersionInt) require.NoError(t, err) - data := append(encChildKey, encKey...) + data := []byte{} + data = append(data, encChildKey...) + data = append(data, encKey...) data = append(data, encVersion...) ret, err := inst.Exec("rtm_ext_default_child_storage_root_version_2", data) diff --git a/lib/trie/db/db.go b/lib/trie/db/db.go index 02579d4906..dd38b113fb 100644 --- a/lib/trie/db/db.go +++ b/lib/trie/db/db.go @@ -82,7 +82,7 @@ func (mdb *MemoryDB) Get(key []byte) ([]byte, error) { return nil, nil } -func (mdb *MemoryDB) Put(key []byte, value []byte) error { +func (mdb *MemoryDB) Put(key, value []byte) error { if len(key) != common.HashLength { return fmt.Errorf("expected %d bytes length key, given %d (%x)", common.HashLength, len(key), key) } diff --git a/lib/trie/version.go b/lib/trie/version.go index 9691c72d63..a2f5e0a9cf 100644 --- a/lib/trie/version.go +++ b/lib/trie/version.go @@ -57,7 +57,7 @@ func (v Version) String() string { } } -// MaxInlineValueSize returns the maximum size of a value to be inlined in the trie node +// MaxInlineValue returns the maximum size of a value to be inlined in the trie node func (v Version) MaxInlineValue() int { switch v { case V0: @@ -83,12 +83,12 @@ func (v Version) Root(entries Entries) (common.Hash, error) { return t.Hash(v.MaxInlineValue()) } -// Root returns the root hash of the trie built using the given entries +// Hash returns the root hash of the trie built using the given entries func (v Version) Hash(t *Trie) (common.Hash, error) { return t.Hash(v.MaxInlineValue()) } -// Root returns the root hash of the trie built using the given entries +// MustHash returns the root hash of the trie built using the given entries or panics if it fails func (v Version) MustHash(t Trie) common.Hash { return t.MustHash(v.MaxInlineValue()) } From 34f4e6bedae22219f2ad61fb1a6355b93848423d Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 13 Nov 2023 12:00:27 -0300 Subject: [PATCH 109/128] Fix deepsource checks --- lib/runtime/wazero/imports_test.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/runtime/wazero/imports_test.go b/lib/runtime/wazero/imports_test.go index a140f8961d..f6f13380d8 100644 --- a/lib/runtime/wazero/imports_test.go +++ b/lib/runtime/wazero/imports_test.go @@ -585,8 +585,7 @@ func Test_ext_trie_blake2_256_root_version_2(t *testing.T) { stateVersionBytes := make([]byte, 4) binary.LittleEndian.PutUint32(stateVersionBytes, stateVersionInt) - data := []byte{} - data = append(data, encInput...) + data := append([]byte{}, encInput...) data = append(data, stateVersionBytes...) res, err := inst.Exec("rtm_ext_trie_blake2_256_root_version_2", data) @@ -633,8 +632,7 @@ func Test_ext_trie_blake2_256_ordered_root_version_2(t *testing.T) { stateVersionBytes := make([]byte, 4) binary.LittleEndian.PutUint32(stateVersionBytes, stateVersion) - data := []byte{} - data = append(data, encValues...) + data := append([]byte{}, encValues...) data = append(data, stateVersionBytes...) res, err := inst.Exec("rtm_ext_trie_blake2_256_ordered_root_version_2", data) @@ -1341,8 +1339,7 @@ func Test_ext_default_child_storage_root_version_2(t *testing.T) { encVersion, err := scale.Marshal(&stateVersionInt) require.NoError(t, err) - data := []byte{} - data = append(data, encChildKey...) + data := append([]byte{}, encChildKey...) data = append(data, encKey...) data = append(data, encVersion...) From 39bcb4698151995d6463dcf4d8f8e3f04d590208 Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 13 Nov 2023 13:50:25 -0300 Subject: [PATCH 110/128] Replace max int to constant --- internal/trie/node/branch_encode_test.go | 9 ++++---- internal/trie/node/encode_decode_test.go | 3 +-- internal/trie/node/encode_test.go | 26 +++++++++++++----------- internal/trie/node/hash_test.go | 9 ++++---- lib/trie/proof/helpers_test.go | 4 ++-- lib/trie/proof/proof_test.go | 9 ++++---- 6 files changed, 29 insertions(+), 31 deletions(-) diff --git a/internal/trie/node/branch_encode_test.go b/internal/trie/node/branch_encode_test.go index 6698ca07ad..e21b315423 100644 --- a/internal/trie/node/branch_encode_test.go +++ b/internal/trie/node/branch_encode_test.go @@ -6,7 +6,6 @@ package node import ( "bytes" "io" - "math" "testing" "github.com/golang/mock/gomock" @@ -24,7 +23,7 @@ func Benchmark_encodeChildrenOpportunisticParallel(b *testing.B) { b.Run("", func(b *testing.B) { for i := 0; i < b.N; i++ { - _ = encodeChildrenOpportunisticParallel(children, math.MaxInt, io.Discard) + _ = encodeChildrenOpportunisticParallel(children, NoMaxInlineValueSize, io.Discard) } }) } @@ -140,7 +139,7 @@ func Test_encodeChildrenOpportunisticParallel(t *testing.T) { previousCall = call } - err := encodeChildrenOpportunisticParallel(testCase.children, math.MaxInt, buffer) + err := encodeChildrenOpportunisticParallel(testCase.children, NoMaxInlineValueSize, buffer) if testCase.wrappedErr != nil { assert.ErrorIs(t, err, testCase.wrappedErr) @@ -165,7 +164,7 @@ func Test_encodeChildrenOpportunisticParallel(t *testing.T) { // Note this may run in parallel or not depending on other tests // running in parallel. - err := encodeChildrenOpportunisticParallel(children, math.MaxInt, buffer) + err := encodeChildrenOpportunisticParallel(children, NoMaxInlineValueSize, buffer) require.NoError(t, err) expectedBytes := []byte{ @@ -256,7 +255,7 @@ func Test_encodeChild(t *testing.T) { previousCall = call } - err := encodeChild(testCase.child, math.MaxInt, buffer) + err := encodeChild(testCase.child, NoMaxInlineValueSize, buffer) if testCase.wrappedErr != nil { assert.ErrorIs(t, err, testCase.wrappedErr) diff --git a/internal/trie/node/encode_decode_test.go b/internal/trie/node/encode_decode_test.go index b736dcccab..8eed629bae 100644 --- a/internal/trie/node/encode_decode_test.go +++ b/internal/trie/node/encode_decode_test.go @@ -5,7 +5,6 @@ package node import ( "bytes" - "math" "testing" "github.com/stretchr/testify/assert" @@ -119,7 +118,7 @@ func Test_Branch_Encode_Decode(t *testing.T) { buffer := bytes.NewBuffer(nil) - err := testCase.branchToEncode.Encode(buffer, math.MaxInt) + err := testCase.branchToEncode.Encode(buffer, NoMaxInlineValueSize) require.NoError(t, err) nodeVariant, partialKeyLength, err := decodeHeader(buffer) diff --git a/internal/trie/node/encode_test.go b/internal/trie/node/encode_test.go index 63fb98d008..aea9d02196 100644 --- a/internal/trie/node/encode_test.go +++ b/internal/trie/node/encode_test.go @@ -14,6 +14,8 @@ import ( "github.com/stretchr/testify/require" ) +const NoMaxInlineValueSize = math.MaxInt + type writeCall struct { written []byte n int // number of bytes @@ -37,7 +39,7 @@ func Test_Node_Encode(t *testing.T) { }{ "nil_node": { node: nil, - maxInlineValueSize: math.MaxInt, + maxInlineValueSize: NoMaxInlineValueSize, writes: []writeCall{ { written: []byte{emptyVariant.bits}, @@ -48,7 +50,7 @@ func Test_Node_Encode(t *testing.T) { node: &Node{ PartialKey: make([]byte, 1), }, - maxInlineValueSize: math.MaxInt, + maxInlineValueSize: NoMaxInlineValueSize, writes: []writeCall{ { written: []byte{leafVariant.bits | 1}, @@ -63,7 +65,7 @@ func Test_Node_Encode(t *testing.T) { PartialKey: []byte{1, 2, 3}, StorageValue: []byte{1}, }, - maxInlineValueSize: math.MaxInt, + maxInlineValueSize: NoMaxInlineValueSize, writes: []writeCall{ { written: []byte{leafVariant.bits | 3}, // partial key length 3 @@ -81,7 +83,7 @@ func Test_Node_Encode(t *testing.T) { PartialKey: []byte{1, 2, 3}, StorageValue: []byte{4, 5, 6}, }, - maxInlineValueSize: math.MaxInt, + maxInlineValueSize: NoMaxInlineValueSize, writes: []writeCall{ { written: []byte{leafVariant.bits | 3}, // partial key length 3 @@ -102,7 +104,7 @@ func Test_Node_Encode(t *testing.T) { PartialKey: []byte{1, 2, 3}, StorageValue: []byte{4, 5, 6}, }, - maxInlineValueSize: math.MaxInt, + maxInlineValueSize: NoMaxInlineValueSize, writes: []writeCall{ { written: []byte{leafVariant.bits | 3}, // partial key length 3 @@ -117,7 +119,7 @@ func Test_Node_Encode(t *testing.T) { PartialKey: []byte{1, 2, 3}, StorageValue: []byte{}, }, - maxInlineValueSize: math.MaxInt, + maxInlineValueSize: NoMaxInlineValueSize, writes: []writeCall{ {written: []byte{leafVariant.bits | 3}}, // partial key length 3 {written: []byte{0x01, 0x23}}, // partial key @@ -181,7 +183,7 @@ func Test_Node_Encode(t *testing.T) { PartialKey: []byte{1, 2, 3}, StorageValue: []byte{100}, }, - maxInlineValueSize: math.MaxInt, + maxInlineValueSize: NoMaxInlineValueSize, writes: []writeCall{ { // header written: []byte{branchWithValueVariant.bits | 3}, // partial key length 3 @@ -203,7 +205,7 @@ func Test_Node_Encode(t *testing.T) { nil, nil, nil, {PartialKey: []byte{11}, StorageValue: []byte{1}}, }, }, - maxInlineValueSize: math.MaxInt, + maxInlineValueSize: NoMaxInlineValueSize, writes: []writeCall{ { // header written: []byte{branchWithValueVariant.bits | 3}, // partial key length 3 @@ -228,7 +230,7 @@ func Test_Node_Encode(t *testing.T) { nil, nil, nil, {PartialKey: []byte{11}, StorageValue: []byte{1}}, }, }, - maxInlineValueSize: math.MaxInt, + maxInlineValueSize: NoMaxInlineValueSize, writes: []writeCall{ { // header written: []byte{branchWithValueVariant.bits | 3}, // partial key length 3 @@ -256,7 +258,7 @@ func Test_Node_Encode(t *testing.T) { nil, nil, nil, {PartialKey: []byte{11}, StorageValue: []byte{1}}, }, }, - maxInlineValueSize: math.MaxInt, + maxInlineValueSize: NoMaxInlineValueSize, writes: []writeCall{ { // header written: []byte{branchWithValueVariant.bits | 3}, // partial key length 3 @@ -289,7 +291,7 @@ func Test_Node_Encode(t *testing.T) { nil, nil, nil, {PartialKey: []byte{11}, StorageValue: []byte{1}}, }, }, - maxInlineValueSize: math.MaxInt, + maxInlineValueSize: NoMaxInlineValueSize, writes: []writeCall{ { // header written: []byte{branchWithValueVariant.bits | 3}, // partial key length 3 @@ -319,7 +321,7 @@ func Test_Node_Encode(t *testing.T) { nil, nil, nil, {PartialKey: []byte{11}, StorageValue: []byte{1}}, }, }, - maxInlineValueSize: math.MaxInt, + maxInlineValueSize: NoMaxInlineValueSize, writes: []writeCall{ { // header written: []byte{branchVariant.bits | 3}, // partial key length 3 diff --git a/internal/trie/node/hash_test.go b/internal/trie/node/hash_test.go index 980ee891c5..adbe06326e 100644 --- a/internal/trie/node/hash_test.go +++ b/internal/trie/node/hash_test.go @@ -5,7 +5,6 @@ package node import ( "io" - "math" "testing" "github.com/golang/mock/gomock" @@ -198,7 +197,7 @@ func Test_Node_CalculateMerkleValue(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - merkleValue, err := testCase.node.CalculateMerkleValue(math.MaxInt) + merkleValue, err := testCase.node.CalculateMerkleValue(NoMaxInlineValueSize) assert.ErrorIs(t, err, testCase.errWrapped) if testCase.errWrapped != nil { @@ -260,7 +259,7 @@ func Test_Node_CalculateRootMerkleValue(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - merkleValue, err := testCase.node.CalculateRootMerkleValue(math.MaxInt) + merkleValue, err := testCase.node.CalculateRootMerkleValue(NoMaxInlineValueSize) assert.ErrorIs(t, err, testCase.errWrapped) if testCase.errWrapped != nil { @@ -347,7 +346,7 @@ func Test_Node_EncodeAndHash(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - encoding, hash, err := testCase.node.EncodeAndHash(math.MaxInt) + encoding, hash, err := testCase.node.EncodeAndHash(NoMaxInlineValueSize) assert.ErrorIs(t, err, testCase.errWrapped) if testCase.errWrapped != nil { @@ -401,7 +400,7 @@ func Test_Node_EncodeAndHashRoot(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - encoding, hash, err := testCase.node.EncodeAndHashRoot(math.MaxInt) + encoding, hash, err := testCase.node.EncodeAndHashRoot(NoMaxInlineValueSize) assert.ErrorIs(t, err, testCase.errWrapped) if testCase.errWrapped != nil { diff --git a/lib/trie/proof/helpers_test.go b/lib/trie/proof/helpers_test.go index 30d6cb488a..4da55557b7 100644 --- a/lib/trie/proof/helpers_test.go +++ b/lib/trie/proof/helpers_test.go @@ -5,12 +5,12 @@ package proof import ( "bytes" - "math" "math/rand" "testing" "github.com/ChainSafe/gossamer/internal/trie/node" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/pkg/scale" "github.com/stretchr/testify/require" ) @@ -24,7 +24,7 @@ func padRightChildren(slice []*node.Node) (paddedSlice []*node.Node) { func encodeNode(t *testing.T, node node.Node) (encoded []byte) { t.Helper() buffer := bytes.NewBuffer(nil) - err := node.Encode(buffer, math.MaxInt) + err := node.Encode(buffer, trie.V0.MaxInlineValue()) require.NoError(t, err) return buffer.Bytes() } diff --git a/lib/trie/proof/proof_test.go b/lib/trie/proof/proof_test.go index fbc0b45ed9..a45e4ffde3 100644 --- a/lib/trie/proof/proof_test.go +++ b/lib/trie/proof/proof_test.go @@ -6,7 +6,6 @@ package proof import ( "encoding/hex" "fmt" - "math" "testing" "github.com/ChainSafe/gossamer/internal/database" @@ -26,19 +25,19 @@ func Test_Generate_Verify(t *testing.T) { "doguinho", } - trie := trie.NewEmptyTrie() + tr := trie.NewEmptyTrie() for i, key := range keys { value := fmt.Sprintf("%x-%d", key, i) - trie.Put([]byte(key), []byte(value)) + tr.Put([]byte(key), []byte(value)) } - rootHash, err := trie.Hash(math.MaxInt) + rootHash, err := trie.V0.Hash(tr) require.NoError(t, err) db, err := database.NewPebble("", true) require.NoError(t, err) - err = trie.WriteDirty(db) + err = tr.WriteDirty(db) require.NoError(t, err) for i, key := range keys { From 0e4ab540804e59c0f55f42bb6dc17b3403d7314e Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 13 Nov 2023 16:22:51 -0300 Subject: [PATCH 111/128] Use default state version in e2e tests --- lib/trie/trie_endtoend_test.go | 66 +++++++++++++++++----------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/lib/trie/trie_endtoend_test.go b/lib/trie/trie_endtoend_test.go index 6871d9a06a..1cccd342be 100644 --- a/lib/trie/trie_endtoend_test.go +++ b/lib/trie/trie_endtoend_test.go @@ -349,13 +349,13 @@ func TestDelete(t *testing.T) { ssTrie := trie.Snapshot() // Get the Trie root hash for all the 3 tries. - tHash, err := V0.Hash(trie) + tHash, err := DefaultStateVersion.Hash(trie) require.NoError(t, err) - dcTrieHash, err := V0.Hash(dcTrie) + dcTrieHash, err := DefaultStateVersion.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err := V0.Hash(ssTrie) + ssTrieHash, err := DefaultStateVersion.Hash(ssTrie) require.NoError(t, err) // Root hash for all the 3 tries should be equal. @@ -376,13 +376,13 @@ func TestDelete(t *testing.T) { } // Get the updated root hash of all tries. - tHash, err = V0.Hash(trie) + tHash, err = DefaultStateVersion.Hash(trie) require.NoError(t, err) - dcTrieHash, err = V0.Hash(dcTrie) + dcTrieHash, err = DefaultStateVersion.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err = V0.Hash(ssTrie) + ssTrieHash, err = DefaultStateVersion.Hash(ssTrie) require.NoError(t, err) // Only the current trie should have a different root hash since it is updated. @@ -432,13 +432,13 @@ func TestClearPrefix(t *testing.T) { ssTrie := trie.Snapshot() // Get the Trie root hash for all the 3 tries. - tHash, err := V0.Hash(trie) + tHash, err := DefaultStateVersion.Hash(trie) require.NoError(t, err) - dcTrieHash, err := V0.Hash(dcTrie) + dcTrieHash, err := DefaultStateVersion.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err := V0.Hash(ssTrie) + ssTrieHash, err := DefaultStateVersion.Hash(ssTrie) require.NoError(t, err) // Root hash for all the 3 tries should be equal. @@ -464,13 +464,13 @@ func TestClearPrefix(t *testing.T) { } // Get the updated root hash of all tries. - tHash, err = V0.Hash(trie) + tHash, err = DefaultStateVersion.Hash(trie) require.NoError(t, err) - dcTrieHash, err = V0.Hash(dcTrie) + dcTrieHash, err = DefaultStateVersion.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err = V0.Hash(ssTrie) + ssTrieHash, err = DefaultStateVersion.Hash(ssTrie) require.NoError(t, err) // Only the current trie should have a different root hash since it is updated. @@ -489,13 +489,13 @@ func TestClearPrefix_Small(t *testing.T) { ssTrie := trie.Snapshot() // Get the Trie root hash for all the 3 tries. - tHash, err := V0.Hash(trie) + tHash, err := DefaultStateVersion.Hash(trie) require.NoError(t, err) - dcTrieHash, err := V0.Hash(dcTrie) + dcTrieHash, err := DefaultStateVersion.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err := V0.Hash(ssTrie) + ssTrieHash, err := DefaultStateVersion.Hash(ssTrie) require.NoError(t, err) // Root hash for all the 3 tries should be equal. @@ -522,13 +522,13 @@ func TestClearPrefix_Small(t *testing.T) { require.Equal(t, expectedRoot, ssTrie.root) // Get the updated root hash of all tries. - tHash, err = V0.Hash(trie) + tHash, err = DefaultStateVersion.Hash(trie) require.NoError(t, err) - dcTrieHash, err = V0.Hash(dcTrie) + dcTrieHash, err = DefaultStateVersion.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err = V0.Hash(ssTrie) + ssTrieHash, err = DefaultStateVersion.Hash(ssTrie) require.NoError(t, err) require.Equal(t, tHash, dcTrieHash) @@ -600,8 +600,8 @@ func TestTrie_ClearPrefixVsDelete(t *testing.T) { trieClearPrefix.ClearPrefix(prefix) - trieClearPrefixHash := V0.MustHash(*trieClearPrefix) - trieDeleteHash := V0.MustHash(*trieDelete) + trieClearPrefixHash := DefaultStateVersion.MustHash(*trieClearPrefix) + trieDeleteHash := DefaultStateVersion.MustHash(*trieDelete) require.Equal(t, trieClearPrefixHash, trieDeleteHash) } @@ -636,9 +636,9 @@ func TestSnapshot(t *testing.T) { newTrie := parentTrie.Snapshot() newTrie.Put(tests[0].key, tests[0].value) - expectedTrieHash := V0.MustHash(*expectedTrie) - newTrieHash := V0.MustHash(*newTrie) - parentTrieHash := V0.MustHash(*parentTrie) + expectedTrieHash := DefaultStateVersion.MustHash(*expectedTrie) + newTrieHash := DefaultStateVersion.MustHash(*newTrie) + parentTrieHash := DefaultStateVersion.MustHash(*parentTrie) require.Equal(t, expectedTrieHash, newTrieHash) require.NotEqual(t, parentTrieHash, newTrieHash) @@ -693,7 +693,7 @@ func Benchmark_Trie_Hash(b *testing.B) { } b.StartTimer() - _, err := V0.Hash(trie) + _, err := DefaultStateVersion.Hash(trie) b.StopTimer() require.NoError(b, err) @@ -776,8 +776,8 @@ func TestTrie_ConcurrentSnapshotWrites(t *testing.T) { for i := 0; i < workers; i++ { assert.Equal( t, - V0.MustHash(*expectedTries[i]), - V0.MustHash(*snapshotedTries[i]), + DefaultStateVersion.MustHash(*expectedTries[i]), + DefaultStateVersion.MustHash(*snapshotedTries[i]), ) } } @@ -959,13 +959,13 @@ func TestTrie_ClearPrefixLimitSnapshot(t *testing.T) { ssTrie := trieClearPrefix.Snapshot() // Get the Trie root hash for all the 3 tries. - tHash, err := V0.Hash(trieClearPrefix) + tHash, err := DefaultStateVersion.Hash(trieClearPrefix) require.NoError(t, err) - dcTrieHash, err := V0.Hash(dcTrie) + dcTrieHash, err := DefaultStateVersion.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err := V0.Hash(ssTrie) + ssTrieHash, err := DefaultStateVersion.Hash(ssTrie) require.NoError(t, err) // Root hash for all the 3 tries should be equal. @@ -1001,13 +1001,13 @@ func TestTrie_ClearPrefixLimitSnapshot(t *testing.T) { } // Get the updated root hash of all tries. - tHash, err = V0.Hash(trieClearPrefix) + tHash, err = DefaultStateVersion.Hash(trieClearPrefix) require.NoError(t, err) - dcTrieHash, err = V0.Hash(dcTrie) + dcTrieHash, err = DefaultStateVersion.Hash(dcTrie) require.NoError(t, err) - ssTrieHash, err = V0.Hash(ssTrie) + ssTrieHash, err = DefaultStateVersion.Hash(ssTrie) require.NoError(t, err) // If node got deleted then root hash must be updated else it has same root hash. @@ -1042,7 +1042,7 @@ func Test_encodeRoot_fuzz(t *testing.T) { assert.Equal(t, value, retrievedValue) } buffer := bytes.NewBuffer(nil) - err := trie.root.Encode(buffer, V0.MaxInlineValue()) + err := trie.root.Encode(buffer, DefaultStateVersion.MaxInlineValue()) require.NoError(t, err) require.NotEmpty(t, buffer.Bytes()) } From 8a7cc08bad6a20c86ee72d8004bcf06d72efce6a Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 13 Nov 2023 16:45:09 -0300 Subject: [PATCH 112/128] Change constant --- lib/trie/proof/helpers_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/trie/proof/helpers_test.go b/lib/trie/proof/helpers_test.go index 4da55557b7..58ec6bb417 100644 --- a/lib/trie/proof/helpers_test.go +++ b/lib/trie/proof/helpers_test.go @@ -24,7 +24,7 @@ func padRightChildren(slice []*node.Node) (paddedSlice []*node.Node) { func encodeNode(t *testing.T, node node.Node) (encoded []byte) { t.Helper() buffer := bytes.NewBuffer(nil) - err := node.Encode(buffer, trie.V0.MaxInlineValue()) + err := node.Encode(buffer, trie.NoMaxInlineValueSize) require.NoError(t, err) return buffer.Bytes() } From 8c49535bc3f22723ae5b48577663b087d5e43d4c Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 13 Nov 2023 16:45:19 -0300 Subject: [PATCH 113/128] Simplify database interface --- lib/trie/proof/database_mocks_test.go | 14 -------------- lib/trie/proof/generate.go | 1 - 2 files changed, 15 deletions(-) diff --git a/lib/trie/proof/database_mocks_test.go b/lib/trie/proof/database_mocks_test.go index ec07c4c1ee..69262dc315 100644 --- a/lib/trie/proof/database_mocks_test.go +++ b/lib/trie/proof/database_mocks_test.go @@ -47,17 +47,3 @@ func (mr *MockDatabaseMockRecorder) Get(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockDatabase)(nil).Get), arg0) } - -// Put mocks base method. -func (m *MockDatabase) Put(arg0, arg1 []byte) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Put", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// Put indicates an expected call of Put. -func (mr *MockDatabaseMockRecorder) Put(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*MockDatabase)(nil).Put), arg0, arg1) -} diff --git a/lib/trie/proof/generate.go b/lib/trie/proof/generate.go index fed898c3cf..3548ea1142 100644 --- a/lib/trie/proof/generate.go +++ b/lib/trie/proof/generate.go @@ -24,7 +24,6 @@ var ( // for proof generation. type Database interface { db.DBGetter - db.DBPutter } // Generate generates and deduplicates the encoded proof nodes From bd218868d56c53cc60f5b319c8bf4e56a38508f5 Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 13 Nov 2023 17:16:17 -0300 Subject: [PATCH 114/128] Removes harcoded state trie version --- dot/digest/helpers_test.go | 3 +++ dot/helpers_test.go | 3 +++ dot/import_integration_test.go | 18 ++++++++++-------- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/dot/digest/helpers_test.go b/dot/digest/helpers_test.go index f603664a62..8637ccbaf2 100644 --- a/dot/digest/helpers_test.go +++ b/dot/digest/helpers_test.go @@ -28,7 +28,10 @@ func newWestendDevGenesisWithTrieAndHeader(t *testing.T) ( require.NoError(t, err) parentHash := common.NewHash([]byte{0}) + + // We are using state trie V0 since we are using the genesis trie where v0 is used stateRoot := trie.V0.MustHash(genesisTrie) + extrinsicRoot := trie.EmptyHash const number = 0 digest := types.NewDigest() diff --git a/dot/helpers_test.go b/dot/helpers_test.go index 1da49be908..a4539a9299 100644 --- a/dot/helpers_test.go +++ b/dot/helpers_test.go @@ -42,7 +42,10 @@ func newWestendDevGenesisWithTrieAndHeader(t *testing.T) ( require.NoError(t, err) parentHash := common.NewHash([]byte{0}) + + // We are using state trie V0 since we are using the genesis trie where v0 is used stateRoot := trie.V0.MustHash(genesisTrie) + extrinsicRoot := trie.EmptyHash const number = 0 digest := types.NewDigest() diff --git a/dot/import_integration_test.go b/dot/import_integration_test.go index b26ae60ec5..7a5816198e 100644 --- a/dot/import_integration_test.go +++ b/dot/import_integration_test.go @@ -23,10 +23,11 @@ func Test_newTrieFromPairs(t *testing.T) { t.Parallel() tests := []struct { - name string - filename string - want common.Hash - err error + name string + filename string + want common.Hash + stateVersion trie.Version + err error }{ { name: "no_arguments", @@ -34,9 +35,10 @@ func Test_newTrieFromPairs(t *testing.T) { want: common.Hash{}, }, { - name: "working example", - filename: setupStateFile(t), - want: common.MustHexToHash("0x09f9ca28df0560c2291aa16b56e15e07d1e1927088f51356d522722aa90ca7cb"), + name: "working example", + filename: setupStateFile(t), + want: common.MustHexToHash("0x09f9ca28df0560c2291aa16b56e15e07d1e1927088f51356d522722aa90ca7cb"), + stateVersion: trie.V0, }, } for _, tt := range tests { @@ -53,7 +55,7 @@ func Test_newTrieFromPairs(t *testing.T) { if tt.want.IsEmpty() { assert.Nil(t, got) } else { - assert.Equal(t, tt.want, trie.V0.MustHash(*got)) + assert.Equal(t, tt.want, tt.stateVersion.MustHash(*got)) } }) } From bc8531052bc623aa42d7de6f2c9297d68ce4a4e0 Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 13 Nov 2023 19:58:02 -0300 Subject: [PATCH 115/128] Get state trie version from runtime --- dot/core/service.go | 22 +++++++++++++++++++++- dot/core/service_test.go | 29 ++++++++++++++++++++++++++++- dot/import_integration_test.go | 6 ++++++ lib/trie/version.go | 2 ++ 4 files changed, 57 insertions(+), 2 deletions(-) diff --git a/dot/core/service.go b/dot/core/service.go index 217dff9e36..4774fa7f31 100644 --- a/dot/core/service.go +++ b/dot/core/service.go @@ -117,6 +117,21 @@ func (s *Service) Stop() error { return nil } +func (s *Service) GetCurrentStateTrieVersion() (trie.Version, error) { + bestBlockHash := s.blockState.BestBlockHash() + rt, err := s.blockState.GetRuntime(bestBlockHash) + if err != nil { + return trie.NoVersion, err + } + + runtimeVersion, err := rt.Version() + if err != nil { + return trie.NoVersion, err + } + + return trie.ParseVersion(runtimeVersion.StateVersion) +} + // StorageRoot returns the hash of the storage root func (s *Service) StorageRoot() (common.Hash, error) { ts, err := s.storageState.TrieState(nil) @@ -124,7 +139,12 @@ func (s *Service) StorageRoot() (common.Hash, error) { return common.Hash{}, err } - return ts.Root(trie.NoMaxInlineValueSize) + stateTrieVersion, err := s.GetCurrentStateTrieVersion() + if err != nil { + return common.Hash{}, err + } + + return stateTrieVersion.Hash(ts.Trie()) } // HandleBlockImport handles a block that was imported via the network diff --git a/dot/core/service_test.go b/dot/core/service_test.go index 629ee0a089..a21b516779 100644 --- a/dot/core/service_test.go +++ b/dot/core/service_test.go @@ -137,6 +137,7 @@ func Test_Service_StorageRoot(t *testing.T) { retErr error expErr error expErrMsg string + stateVersion uint32 }{ { name: "storage trie state error", @@ -147,12 +148,22 @@ func Test_Service_StorageRoot(t *testing.T) { trieStateCall: true, }, { - name: "storage trie state ok", + name: "storage trie state ok v0", service: &Service{}, exp: common.Hash{0x3, 0x17, 0xa, 0x2e, 0x75, 0x97, 0xb7, 0xb7, 0xe3, 0xd8, 0x4c, 0x5, 0x39, 0x1d, 0x13, 0x9a, 0x62, 0xb1, 0x57, 0xe7, 0x87, 0x86, 0xd8, 0xc0, 0x82, 0xf2, 0x9d, 0xcf, 0x4c, 0x11, 0x13, 0x14}, retTrieState: ts, trieStateCall: true, + stateVersion: 0, + }, + { + name: "storage trie state ok v1", + service: &Service{}, + exp: common.Hash{0x3, 0x17, 0xa, 0x2e, 0x75, 0x97, 0xb7, 0xb7, 0xe3, 0xd8, 0x4c, 0x5, 0x39, 0x1d, 0x13, 0x9a, + 0x62, 0xb1, 0x57, 0xe7, 0x87, 0x86, 0xd8, 0xc0, 0x82, 0xf2, 0x9d, 0xcf, 0x4c, 0x11, 0x13, 0x14}, + retTrieState: ts, + trieStateCall: true, + stateVersion: 1, }, } for _, tt := range tests { @@ -164,7 +175,23 @@ func Test_Service_StorageRoot(t *testing.T) { ctrl := gomock.NewController(t) mockStorageState := NewMockStorageState(ctrl) mockStorageState.EXPECT().TrieState(nil).Return(tt.retTrieState, tt.retErr) + service.storageState = mockStorageState + + if tt.retErr == nil { + mockRuntimeVersion := runtime.Version{ + StateVersion: tt.stateVersion, + } + + mockRuntime := NewMockInstance(ctrl) + mockRuntime.EXPECT().Version().Return(mockRuntimeVersion, nil) + + mockBlockState := NewMockBlockState(ctrl) + mockBlockState.EXPECT().BestBlockHash().Return(common.Hash{}) + mockBlockState.EXPECT().GetRuntime(gomock.Any()).Return(mockRuntime, nil) + + service.blockState = mockBlockState + } } res, err := service.StorageRoot() diff --git a/dot/import_integration_test.go b/dot/import_integration_test.go index 7a5816198e..66bef5d85b 100644 --- a/dot/import_integration_test.go +++ b/dot/import_integration_test.go @@ -40,6 +40,12 @@ func Test_newTrieFromPairs(t *testing.T) { want: common.MustHexToHash("0x09f9ca28df0560c2291aa16b56e15e07d1e1927088f51356d522722aa90ca7cb"), stateVersion: trie.V0, }, + { + name: "working example", + filename: setupStateFile(t), + want: common.MustHexToHash("0x09f9ca28df0560c2291aa16b56e15e07d1e1927088f51356d522722aa90ca7cb"), + stateVersion: trie.V1, + }, } for _, tt := range tests { tt := tt diff --git a/lib/trie/version.go b/lib/trie/version.go index a2f5e0a9cf..e2aa54a2d8 100644 --- a/lib/trie/version.go +++ b/lib/trie/version.go @@ -32,6 +32,8 @@ const ( V1 ) +var NoVersion = Version(math.MaxUint8) + // ErrParseVersion is returned when parsing a state trie version fails. var ErrParseVersion = errors.New("parsing version failed") From 1d1bd258f59a75605b07eeb1e58b8ebca8a8ffa5 Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 13 Nov 2023 20:08:16 -0300 Subject: [PATCH 116/128] Move comment --- dot/node_integration_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dot/node_integration_test.go b/dot/node_integration_test.go index 736e155d46..aad81b1331 100644 --- a/dot/node_integration_test.go +++ b/dot/node_integration_test.go @@ -388,8 +388,7 @@ func TestInitNode_LoadStorageRoot(t *testing.T) { expected, err := trie.LoadFromMap(gen.GenesisFields().Raw["top"]) require.NoError(t, err) - // TODO: get trie state version from runtime - expectedRoot, err := trie.V0.Hash(&expected) + expectedRoot, err := trie.V0.Hash(&expected) // Since we are using a runtime with state trie V0 require.NoError(t, err) coreServiceInterface := node.ServiceRegistry.Get(&core.Service{}) From 2648240c8406f8e7876255867218024971c43111 Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 13 Nov 2023 20:47:39 -0300 Subject: [PATCH 117/128] Add import state version --- cmd/gossamer/commands/import_state.go | 16 ++++++++++++++-- dot/import.go | 4 ++-- dot/import_integration_test.go | 24 +++++++++++++----------- dot/state/service.go | 4 ++-- dot/state/service_integration_test.go | 2 +- 5 files changed, 32 insertions(+), 18 deletions(-) diff --git a/cmd/gossamer/commands/import_state.go b/cmd/gossamer/commands/import_state.go index a12f137ba9..be924909e2 100644 --- a/cmd/gossamer/commands/import_state.go +++ b/cmd/gossamer/commands/import_state.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/ChainSafe/gossamer/dot" + "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" "github.com/spf13/cobra" ) @@ -14,6 +15,7 @@ import ( func init() { ImportStateCmd.Flags().String("chain", "", "Chain id used to load default configuration for specified chain") ImportStateCmd.Flags().String("state-file", "", "Path to JSON file consisting of key-value pairs") + ImportStateCmd.Flags().Uint32("state-version", uint32(trie.DefaultStateVersion), "State version to use when importing state") //nolint:lll ImportStateCmd.Flags().String("header-file", "", "Path to JSON file of block header corresponding to the given state") ImportStateCmd.Flags().Uint64("first-slot", 0, "The first BABE slot of the network") } @@ -26,7 +28,8 @@ var ImportStateCmd = &cobra.Command{ in the form of key-value pairs to be imported. Input can be generated by using the RPC function state_getPairs. Example: - gossamer import-state --state-file state.json --header-file header.json --first-slot `, + gossamer import-state --state-file state.json state-version 1 --header-file header.json + --first-slot `, RunE: func(cmd *cobra.Command, args []string) error { return execImportState(cmd) }, @@ -54,6 +57,15 @@ func execImportState(cmd *cobra.Command) error { return fmt.Errorf("state-file must be specified") } + stateVersion, err := cmd.Flags().GetUint32("state-version") + if err != nil { + return fmt.Errorf("failed to get state-file: %s", err) + } + stateTrieVersion, err := trie.ParseVersion(stateVersion) + if err != nil { + return fmt.Errorf("Invalid state version") + } + headerFile, err := cmd.Flags().GetString("header-file") if err != nil { return fmt.Errorf("failed to get header-file: %s", err) @@ -64,5 +76,5 @@ func execImportState(cmd *cobra.Command) error { basePath = utils.ExpandDir(basePath) - return dot.ImportState(basePath, stateFile, headerFile, firstSlot) + return dot.ImportState(basePath, stateFile, headerFile, stateTrieVersion, firstSlot) } diff --git a/dot/import.go b/dot/import.go index c5d9c85f4e..e9fe509518 100644 --- a/dot/import.go +++ b/dot/import.go @@ -20,7 +20,7 @@ import ( ) // ImportState imports the state in the given files to the database with the given path. -func ImportState(basepath, stateFP, headerFP string, firstSlot uint64) error { +func ImportState(basepath, stateFP, headerFP string, stateTrieVersion trie.Version, firstSlot uint64) error { tr, err := newTrieFromPairs(stateFP) if err != nil { return err @@ -38,7 +38,7 @@ func ImportState(basepath, stateFP, headerFP string, firstSlot uint64) error { LogLevel: log.Info, } srv := state.NewService(config) - return srv.Import(header, tr, firstSlot) + return srv.Import(header, tr, stateTrieVersion, firstSlot) } func newTrieFromPairs(filename string) (*trie.Trie, error) { diff --git a/dot/import_integration_test.go b/dot/import_integration_test.go index 66bef5d85b..e4a5639c56 100644 --- a/dot/import_integration_test.go +++ b/dot/import_integration_test.go @@ -43,7 +43,7 @@ func Test_newTrieFromPairs(t *testing.T) { { name: "working example", filename: setupStateFile(t), - want: common.MustHexToHash("0x09f9ca28df0560c2291aa16b56e15e07d1e1927088f51356d522722aa90ca7cb"), + want: common.MustHexToHash("0xcc25fe024a58297658e576e2e4c33691fe3a9fe5a7cdd2e55534164a0fcc0782"), stateVersion: trie.V1, }, } @@ -102,7 +102,7 @@ func TestImportState_Integration(t *testing.T) { headerFP := setupHeaderFile(t) const firstSlot = uint64(262493679) - err = ImportState(config.BasePath, stateFP, headerFP, firstSlot) + err = ImportState(config.BasePath, stateFP, headerFP, trie.V0, firstSlot) require.NoError(t, err) // confirm data is imported into db stateConfig := state.Config{ @@ -133,10 +133,11 @@ func TestImportState(t *testing.T) { headerFP := setupHeaderFile(t) type args struct { - basepath string - stateFP string - headerFP string - firstSlot uint64 + basepath string + stateFP string + headerFP string + stateVersion trie.Version + firstSlot uint64 } tests := []struct { name string @@ -150,10 +151,11 @@ func TestImportState(t *testing.T) { { name: "working_example", args: args{ - basepath: config.BasePath, - stateFP: stateFP, - headerFP: headerFP, - firstSlot: 262493679, + basepath: config.BasePath, + stateFP: stateFP, + headerFP: headerFP, + stateVersion: trie.V0, + firstSlot: 262493679, }, }, } @@ -162,7 +164,7 @@ func TestImportState(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - err := ImportState(tt.args.basepath, tt.args.stateFP, tt.args.headerFP, tt.args.firstSlot) + err := ImportState(tt.args.basepath, tt.args.stateFP, tt.args.headerFP, tt.args.stateVersion, tt.args.firstSlot) if tt.err != nil { assert.EqualError(t, err, tt.err.Error()) } else { diff --git a/dot/state/service.go b/dot/state/service.go index a5b405ee19..d839fda623 100644 --- a/dot/state/service.go +++ b/dot/state/service.go @@ -256,7 +256,7 @@ func (s *Service) Stop() error { // Import imports the given state corresponding to the given header and sets the head of the chain // to it. Additionally, it uses the first slot to correctly set the epoch number of the block. -func (s *Service) Import(header *types.Header, t *trie.Trie, firstSlot uint64) error { +func (s *Service) Import(header *types.Header, t *trie.Trie, stateTrieVersion trie.Version, firstSlot uint64) error { var err error // initialise database using data directory if !s.isMemDB { @@ -301,7 +301,7 @@ func (s *Service) Import(header *types.Header, t *trie.Trie, firstSlot uint64) e return err } - root := t.MustHash(trie.NoMaxInlineValueSize) + root := stateTrieVersion.MustHash(*t) if root != header.StateRoot { return fmt.Errorf("trie state root does not equal header state root") } diff --git a/dot/state/service_integration_test.go b/dot/state/service_integration_test.go index cf041c64ee..8fb9bc5de5 100644 --- a/dot/state/service_integration_test.go +++ b/dot/state/service_integration_test.go @@ -406,7 +406,7 @@ func TestService_Import(t *testing.T) { firstSlot := uint64(100) - err = serv.Import(header, tr, firstSlot) + err = serv.Import(header, tr, trie.V0, firstSlot) require.NoError(t, err) err = serv.Start() From 2de3013d1beaf13575af6cca0019966294c4eb6e Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 13 Nov 2023 21:14:51 -0300 Subject: [PATCH 118/128] Change error message --- cmd/gossamer/commands/import_state.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/gossamer/commands/import_state.go b/cmd/gossamer/commands/import_state.go index be924909e2..f316aeb44c 100644 --- a/cmd/gossamer/commands/import_state.go +++ b/cmd/gossamer/commands/import_state.go @@ -63,7 +63,7 @@ func execImportState(cmd *cobra.Command) error { } stateTrieVersion, err := trie.ParseVersion(stateVersion) if err != nil { - return fmt.Errorf("Invalid state version") + return fmt.Errorf("invalid state version") } headerFile, err := cmd.Flags().GetString("header-file") From c56bd71c199ceb6597db39707129c21d82f49be2 Mon Sep 17 00:00:00 2001 From: Diego Date: Fri, 17 Nov 2023 10:08:06 -0300 Subject: [PATCH 119/128] Removes Database interface --- lib/trie/db/db.go | 5 ++--- lib/trie/proof/generate.go | 8 +------- lib/trie/proof/generate_test.go | 15 ++++++++------- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/lib/trie/db/db.go b/lib/trie/db/db.go index dd38b113fb..2be63db075 100644 --- a/lib/trie/db/db.go +++ b/lib/trie/db/db.go @@ -69,13 +69,12 @@ func (mdb *MemoryDB) Get(key []byte) ([]byte, error) { if len(key) != common.HashLength { return nil, fmt.Errorf("expected %d bytes length key, given %d (%x)", common.HashLength, len(key), key) } - var hash common.Hash - copy(hash[:], key) + hashedKey := common.Hash(key) mdb.mutex.RLock() defer mdb.mutex.RUnlock() - if value, found := mdb.data[hash]; found { + if value, found := mdb.data[hashedKey]; found { return value, nil } diff --git a/lib/trie/proof/generate.go b/lib/trie/proof/generate.go index 3548ea1142..0daca83e52 100644 --- a/lib/trie/proof/generate.go +++ b/lib/trie/proof/generate.go @@ -20,17 +20,11 @@ var ( ErrKeyNotFound = errors.New("key not found") ) -// Database defines a key value Get method used -// for proof generation. -type Database interface { - db.DBGetter -} - // Generate generates and deduplicates the encoded proof nodes // for the trie corresponding to the root hash given, and for // the slice of (Little Endian) full keys given. The database given // is used to load the trie using the root hash given. -func Generate(rootHash []byte, fullKeys [][]byte, database Database) ( +func Generate(rootHash []byte, fullKeys [][]byte, database db.DBGetter) ( encodedProofNodes [][]byte, err error) { trie := trie.NewEmptyTrie() if err := trie.Load(database, common.BytesToHash(rootHash)); err != nil { diff --git a/lib/trie/proof/generate_test.go b/lib/trie/proof/generate_test.go index 49f519b3c0..184fd0106c 100644 --- a/lib/trie/proof/generate_test.go +++ b/lib/trie/proof/generate_test.go @@ -10,6 +10,7 @@ import ( "github.com/ChainSafe/gossamer/internal/trie/codec" "github.com/ChainSafe/gossamer/internal/trie/node" "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/lib/trie/db" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -31,14 +32,14 @@ func Test_Generate(t *testing.T) { testCases := map[string]struct { rootHash []byte fullKeysNibbles [][]byte - databaseBuilder func(ctrl *gomock.Controller) Database + databaseBuilder func(ctrl *gomock.Controller) db.DBGetter encodedProofNodes [][]byte errWrapped error errMessage string }{ "failed_loading_trie": { rootHash: someHash, - databaseBuilder: func(ctrl *gomock.Controller) Database { + databaseBuilder: func(ctrl *gomock.Controller) db.DBGetter { mockDatabase := NewMockDatabase(ctrl) mockDatabase.EXPECT().Get(someHash). Return(nil, errTest) @@ -53,7 +54,7 @@ func Test_Generate(t *testing.T) { "walk_error": { rootHash: someHash, fullKeysNibbles: [][]byte{{1}}, - databaseBuilder: func(ctrl *gomock.Controller) Database { + databaseBuilder: func(ctrl *gomock.Controller) db.DBGetter { mockDatabase := NewMockDatabase(ctrl) encodedRoot := encodeNode(t, node.Node{ PartialKey: []byte{1}, @@ -69,7 +70,7 @@ func Test_Generate(t *testing.T) { "leaf_root": { rootHash: someHash, fullKeysNibbles: [][]byte{{}}, - databaseBuilder: func(ctrl *gomock.Controller) Database { + databaseBuilder: func(ctrl *gomock.Controller) db.DBGetter { mockDatabase := NewMockDatabase(ctrl) encodedRoot := encodeNode(t, node.Node{ PartialKey: []byte{1}, @@ -89,7 +90,7 @@ func Test_Generate(t *testing.T) { "branch_root": { rootHash: someHash, fullKeysNibbles: [][]byte{{}}, - databaseBuilder: func(ctrl *gomock.Controller) Database { + databaseBuilder: func(ctrl *gomock.Controller) db.DBGetter { mockDatabase := NewMockDatabase(ctrl) encodedRoot := encodeNode(t, node.Node{ PartialKey: []byte{1}, @@ -125,7 +126,7 @@ func Test_Generate(t *testing.T) { fullKeysNibbles: [][]byte{ {1, 2, 3, 4}, }, - databaseBuilder: func(ctrl *gomock.Controller) Database { + databaseBuilder: func(ctrl *gomock.Controller) db.DBGetter { mockDatabase := NewMockDatabase(ctrl) rootNode := node.Node{ @@ -174,7 +175,7 @@ func Test_Generate(t *testing.T) { {1, 2, 4, 4}, {1, 2, 5, 5}, }, - databaseBuilder: func(ctrl *gomock.Controller) Database { + databaseBuilder: func(ctrl *gomock.Controller) db.DBGetter { mockDatabase := NewMockDatabase(ctrl) rootNode := node.Node{ From 2ff1a4e171f5b2974b7a16b02cb225f8542cc8fd Mon Sep 17 00:00:00 2001 From: Diego Date: Fri, 17 Nov 2023 10:30:03 -0300 Subject: [PATCH 120/128] Make GetCurrentStateTrieVersion private --- dot/core/service.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dot/core/service.go b/dot/core/service.go index 4774fa7f31..a7dc445a3e 100644 --- a/dot/core/service.go +++ b/dot/core/service.go @@ -117,7 +117,7 @@ func (s *Service) Stop() error { return nil } -func (s *Service) GetCurrentStateTrieVersion() (trie.Version, error) { +func (s *Service) getCurrentStateTrieVersion() (trie.TrieLayout, error) { bestBlockHash := s.blockState.BestBlockHash() rt, err := s.blockState.GetRuntime(bestBlockHash) if err != nil { @@ -139,7 +139,7 @@ func (s *Service) StorageRoot() (common.Hash, error) { return common.Hash{}, err } - stateTrieVersion, err := s.GetCurrentStateTrieVersion() + stateTrieVersion, err := s.getCurrentStateTrieVersion() if err != nil { return common.Hash{}, err } From 38b4b37f895b03742ffbf64f41b362ad06c34c8c Mon Sep 17 00:00:00 2001 From: Diego Date: Fri, 17 Nov 2023 10:31:57 -0300 Subject: [PATCH 121/128] Rename trie version to layout --- dot/import.go | 2 +- dot/import_integration_test.go | 4 ++-- dot/state/service.go | 2 +- lib/trie/{version.go => layout.go} | 20 ++++++++++---------- lib/trie/{version_test.go => layout_test.go} | 14 +++++++------- 5 files changed, 21 insertions(+), 21 deletions(-) rename lib/trie/{version.go => layout.go} (84%) rename lib/trie/{version_test.go => layout_test.go} (95%) diff --git a/dot/import.go b/dot/import.go index e9fe509518..208d648765 100644 --- a/dot/import.go +++ b/dot/import.go @@ -20,7 +20,7 @@ import ( ) // ImportState imports the state in the given files to the database with the given path. -func ImportState(basepath, stateFP, headerFP string, stateTrieVersion trie.Version, firstSlot uint64) error { +func ImportState(basepath, stateFP, headerFP string, stateTrieVersion trie.TrieLayout, firstSlot uint64) error { tr, err := newTrieFromPairs(stateFP) if err != nil { return err diff --git a/dot/import_integration_test.go b/dot/import_integration_test.go index e4a5639c56..436e4c2204 100644 --- a/dot/import_integration_test.go +++ b/dot/import_integration_test.go @@ -26,7 +26,7 @@ func Test_newTrieFromPairs(t *testing.T) { name string filename string want common.Hash - stateVersion trie.Version + stateVersion trie.TrieLayout err error }{ { @@ -136,7 +136,7 @@ func TestImportState(t *testing.T) { basepath string stateFP string headerFP string - stateVersion trie.Version + stateVersion trie.TrieLayout firstSlot uint64 } tests := []struct { diff --git a/dot/state/service.go b/dot/state/service.go index d839fda623..7066664ee4 100644 --- a/dot/state/service.go +++ b/dot/state/service.go @@ -256,7 +256,7 @@ func (s *Service) Stop() error { // Import imports the given state corresponding to the given header and sets the head of the chain // to it. Additionally, it uses the first slot to correctly set the epoch number of the block. -func (s *Service) Import(header *types.Header, t *trie.Trie, stateTrieVersion trie.Version, firstSlot uint64) error { +func (s *Service) Import(header *types.Header, t *trie.Trie, stateTrieVersion trie.TrieLayout, firstSlot uint64) error { var err error // initialise database using data directory if !s.isMemDB { diff --git a/lib/trie/version.go b/lib/trie/layout.go similarity index 84% rename from lib/trie/version.go rename to lib/trie/layout.go index e2aa54a2d8..5758a7b23a 100644 --- a/lib/trie/version.go +++ b/lib/trie/layout.go @@ -19,20 +19,20 @@ const ( V1MaxInlineValueSize = 32 ) -// Version is the state trie version which dictates how a +// TrieLayout is the state trie version which dictates how a // Merkle root should be constructed. It is defined in // https://spec.polkadot.network/#defn-state-version -type Version uint8 +type TrieLayout uint8 const ( // V0 is the state trie version 0 where the values of the keys are // inserted into the trie directly. // TODO set to iota once CI passes - V0 Version = iota + V0 TrieLayout = iota V1 ) -var NoVersion = Version(math.MaxUint8) +var NoVersion = TrieLayout(math.MaxUint8) // ErrParseVersion is returned when parsing a state trie version fails. var ErrParseVersion = errors.New("parsing version failed") @@ -48,7 +48,7 @@ type Entry struct{ Key, Value []byte } type Entries []Entry // String returns a string representation of trie version -func (v Version) String() string { +func (v TrieLayout) String() string { switch v { case V0: return "v0" @@ -60,7 +60,7 @@ func (v Version) String() string { } // MaxInlineValue returns the maximum size of a value to be inlined in the trie node -func (v Version) MaxInlineValue() int { +func (v TrieLayout) MaxInlineValue() int { switch v { case V0: return NoMaxInlineValueSize @@ -72,7 +72,7 @@ func (v Version) MaxInlineValue() int { } // Root returns the root hash of the trie built using the given entries -func (v Version) Root(entries Entries) (common.Hash, error) { +func (v TrieLayout) Root(entries Entries) (common.Hash, error) { t := NewEmptyTrie() for _, kv := range entries { @@ -86,17 +86,17 @@ func (v Version) Root(entries Entries) (common.Hash, error) { } // Hash returns the root hash of the trie built using the given entries -func (v Version) Hash(t *Trie) (common.Hash, error) { +func (v TrieLayout) Hash(t *Trie) (common.Hash, error) { return t.Hash(v.MaxInlineValue()) } // MustHash returns the root hash of the trie built using the given entries or panics if it fails -func (v Version) MustHash(t Trie) common.Hash { +func (v TrieLayout) MustHash(t Trie) common.Hash { return t.MustHash(v.MaxInlineValue()) } // ParseVersion parses a state trie version string. -func ParseVersion[T string | uint32](v T) (version Version, err error) { +func ParseVersion[T string | uint32](v T) (version TrieLayout, err error) { var s string switch value := any(v).(type) { case string: diff --git a/lib/trie/version_test.go b/lib/trie/layout_test.go similarity index 95% rename from lib/trie/version_test.go rename to lib/trie/layout_test.go index c092a4cb6c..b2562ad7ab 100644 --- a/lib/trie/version_test.go +++ b/lib/trie/layout_test.go @@ -14,7 +14,7 @@ func Test_Version_String(t *testing.T) { t.Parallel() testCases := map[string]struct { - version Version + version TrieLayout versionString string panicMessage string }{ @@ -23,7 +23,7 @@ func Test_Version_String(t *testing.T) { versionString: "v0", }, "invalid": { - version: Version(99), + version: TrieLayout(99), panicMessage: "unknown version 99", }, } @@ -51,7 +51,7 @@ func Test_ParseVersion(t *testing.T) { testCases := map[string]struct { v any - version Version + version TrieLayout errWrapped error errMessage string }{ @@ -96,7 +96,7 @@ func Test_ParseVersion(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - var version Version + var version TrieLayout var err error switch typed := testCase.v.(type) { @@ -121,7 +121,7 @@ func Test_Version_MaxInlineValue(t *testing.T) { t.Parallel() testCases := map[string]struct { - version Version + version TrieLayout max int panicMessage string }{ @@ -134,7 +134,7 @@ func Test_Version_MaxInlineValue(t *testing.T) { max: V1MaxInlineValueSize, }, "invalid": { - version: Version(99), + version: TrieLayout(99), max: 0, panicMessage: "unknown version 99", }, @@ -162,7 +162,7 @@ func Test_Version_Root(t *testing.T) { t.Parallel() testCases := map[string]struct { - version Version + version TrieLayout input Entries expected common.Hash }{ From 14ad372d8160d2e38f07d2cc159010b5383f2b6b Mon Sep 17 00:00:00 2001 From: Diego Date: Fri, 17 Nov 2023 10:32:18 -0300 Subject: [PATCH 122/128] use NoMaxInlineValueSize --- internal/trie/node/header_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/trie/node/header_test.go b/internal/trie/node/header_test.go index 672fd2b45f..5b57174628 100644 --- a/internal/trie/node/header_test.go +++ b/internal/trie/node/header_test.go @@ -293,7 +293,7 @@ func Test_encodeHeader_At_Maximum(t *testing.T) { PartialKey: make([]byte, keyLength), } - err := encodeHeader(node, math.MaxInt, buffer) + err := encodeHeader(node, NoMaxInlineValueSize, buffer) require.NoError(t, err) assert.Equal(t, expectedBytes, buffer.Bytes()) From 2185edb0fe647bb98cabcf01855d5693041fd4a2 Mon Sep 17 00:00:00 2001 From: Diego Date: Fri, 17 Nov 2023 10:50:46 -0300 Subject: [PATCH 123/128] Fix mocks --- lib/trie/proof/database_mocks_test.go | 30 +++++++++++++-------------- lib/trie/proof/generate_test.go | 12 +++++------ lib/trie/proof/mocks_generate_test.go | 2 +- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/lib/trie/proof/database_mocks_test.go b/lib/trie/proof/database_mocks_test.go index 69262dc315..7a4a6d31a8 100644 --- a/lib/trie/proof/database_mocks_test.go +++ b/lib/trie/proof/database_mocks_test.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/ChainSafe/gossamer/lib/trie/proof (interfaces: Database) +// Source: github.com/ChainSafe/gossamer/lib/trie/db (interfaces: DBGetter) // Package proof is a generated GoMock package. package proof @@ -10,31 +10,31 @@ import ( gomock "github.com/golang/mock/gomock" ) -// MockDatabase is a mock of Database interface. -type MockDatabase struct { +// MockDBGetter is a mock of DBGetter interface. +type MockDBGetter struct { ctrl *gomock.Controller - recorder *MockDatabaseMockRecorder + recorder *MockDBGetterMockRecorder } -// MockDatabaseMockRecorder is the mock recorder for MockDatabase. -type MockDatabaseMockRecorder struct { - mock *MockDatabase +// MockDBGetterMockRecorder is the mock recorder for MockDBGetter. +type MockDBGetterMockRecorder struct { + mock *MockDBGetter } -// NewMockDatabase creates a new mock instance. -func NewMockDatabase(ctrl *gomock.Controller) *MockDatabase { - mock := &MockDatabase{ctrl: ctrl} - mock.recorder = &MockDatabaseMockRecorder{mock} +// NewMockDBGetter creates a new mock instance. +func NewMockDBGetter(ctrl *gomock.Controller) *MockDBGetter { + mock := &MockDBGetter{ctrl: ctrl} + mock.recorder = &MockDBGetterMockRecorder{mock} return mock } // EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockDatabase) EXPECT() *MockDatabaseMockRecorder { +func (m *MockDBGetter) EXPECT() *MockDBGetterMockRecorder { return m.recorder } // Get mocks base method. -func (m *MockDatabase) Get(arg0 []byte) ([]byte, error) { +func (m *MockDBGetter) Get(arg0 []byte) ([]byte, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Get", arg0) ret0, _ := ret[0].([]byte) @@ -43,7 +43,7 @@ func (m *MockDatabase) Get(arg0 []byte) ([]byte, error) { } // Get indicates an expected call of Get. -func (mr *MockDatabaseMockRecorder) Get(arg0 interface{}) *gomock.Call { +func (mr *MockDBGetterMockRecorder) Get(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockDatabase)(nil).Get), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockDBGetter)(nil).Get), arg0) } diff --git a/lib/trie/proof/generate_test.go b/lib/trie/proof/generate_test.go index 184fd0106c..1390b1d08a 100644 --- a/lib/trie/proof/generate_test.go +++ b/lib/trie/proof/generate_test.go @@ -40,7 +40,7 @@ func Test_Generate(t *testing.T) { "failed_loading_trie": { rootHash: someHash, databaseBuilder: func(ctrl *gomock.Controller) db.DBGetter { - mockDatabase := NewMockDatabase(ctrl) + mockDatabase := NewMockDBGetter(ctrl) mockDatabase.EXPECT().Get(someHash). Return(nil, errTest) return mockDatabase @@ -55,7 +55,7 @@ func Test_Generate(t *testing.T) { rootHash: someHash, fullKeysNibbles: [][]byte{{1}}, databaseBuilder: func(ctrl *gomock.Controller) db.DBGetter { - mockDatabase := NewMockDatabase(ctrl) + mockDatabase := NewMockDBGetter(ctrl) encodedRoot := encodeNode(t, node.Node{ PartialKey: []byte{1}, StorageValue: []byte{2}, @@ -71,7 +71,7 @@ func Test_Generate(t *testing.T) { rootHash: someHash, fullKeysNibbles: [][]byte{{}}, databaseBuilder: func(ctrl *gomock.Controller) db.DBGetter { - mockDatabase := NewMockDatabase(ctrl) + mockDatabase := NewMockDBGetter(ctrl) encodedRoot := encodeNode(t, node.Node{ PartialKey: []byte{1}, StorageValue: []byte{2}, @@ -91,7 +91,7 @@ func Test_Generate(t *testing.T) { rootHash: someHash, fullKeysNibbles: [][]byte{{}}, databaseBuilder: func(ctrl *gomock.Controller) db.DBGetter { - mockDatabase := NewMockDatabase(ctrl) + mockDatabase := NewMockDBGetter(ctrl) encodedRoot := encodeNode(t, node.Node{ PartialKey: []byte{1}, StorageValue: []byte{2}, @@ -127,7 +127,7 @@ func Test_Generate(t *testing.T) { {1, 2, 3, 4}, }, databaseBuilder: func(ctrl *gomock.Controller) db.DBGetter { - mockDatabase := NewMockDatabase(ctrl) + mockDatabase := NewMockDBGetter(ctrl) rootNode := node.Node{ PartialKey: []byte{1, 2}, @@ -176,7 +176,7 @@ func Test_Generate(t *testing.T) { {1, 2, 5, 5}, }, databaseBuilder: func(ctrl *gomock.Controller) db.DBGetter { - mockDatabase := NewMockDatabase(ctrl) + mockDatabase := NewMockDBGetter(ctrl) rootNode := node.Node{ PartialKey: []byte{1, 2}, diff --git a/lib/trie/proof/mocks_generate_test.go b/lib/trie/proof/mocks_generate_test.go index 314b3490aa..adde5e4e89 100644 --- a/lib/trie/proof/mocks_generate_test.go +++ b/lib/trie/proof/mocks_generate_test.go @@ -3,4 +3,4 @@ package proof -//go:generate mockgen -destination=database_mocks_test.go -package=$GOPACKAGE . Database +//go:generate mockgen -destination=database_mocks_test.go -package=$GOPACKAGE github.com/ChainSafe/gossamer/lib/trie/db DBGetter From f4661c9b87996a3d20465cc93a65f86e3ab32a69 Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 20 Nov 2023 13:59:00 -0300 Subject: [PATCH 124/128] Fix tests using new runtime version parameter --- lib/runtime/constants.go | 3 +-- lib/runtime/wazero/imports.go | 12 ++++-------- lib/runtime/wazero/imports_test.go | 28 ++++++++++++++-------------- 3 files changed, 19 insertions(+), 24 deletions(-) diff --git a/lib/runtime/constants.go b/lib/runtime/constants.go index 132f6ce7e7..812263e3db 100644 --- a/lib/runtime/constants.go +++ b/lib/runtime/constants.go @@ -8,8 +8,7 @@ const ( // This wasm is generated using https://github.com/ChainSafe/polkadot-spec. HOST_API_TEST_RUNTIME = "hostapi_runtime" HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm" - HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/raw/master/test/" + - "runtimes/hostapi/hostapi_runtime.compact.wasm" + HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/raw/e1b3a95704373f2e51edda9c6cb174eed11e4f19/test/runtimes/hostapi/hostapi_runtime.compact.wasm" // v0.9.29 polkadot POLKADOT_RUNTIME_v0929 = "polkadot_runtime-v929" diff --git a/lib/runtime/wazero/imports.go b/lib/runtime/wazero/imports.go index 9822c5366f..d91d5ec630 100644 --- a/lib/runtime/wazero/imports.go +++ b/lib/runtime/wazero/imports.go @@ -791,8 +791,7 @@ func ext_trie_blake2_256_root_version_2(ctx context.Context, m api.Module, dataS panic("nil runtime context") } - stateVersionBytes, _ := m.Memory().Read(version, 4) - stateVersion, err := trie.ParseVersion(binary.LittleEndian.Uint32(stateVersionBytes)) + stateVersion, err := trie.ParseVersion(version) if err != nil { logger.Errorf("failed parsing state version: %s", err) return 0 @@ -838,8 +837,7 @@ func ext_trie_blake2_256_ordered_root_version_2( data := read(m, dataSpan) - stateVersionBytes, _ := m.Memory().Read(version, 4) - stateVersion, err := trie.ParseVersion(binary.LittleEndian.Uint32(stateVersionBytes)) + stateVersion, err := trie.ParseVersion(version) if err != nil { logger.Errorf("failed parsing state version: %s", err) return 0 @@ -921,8 +919,7 @@ func ext_trie_blake2_256_verify_proof_version_2( panic("nil runtime context") } - stateVersionBytes, _ := m.Memory().Read(version, 4) - _, err := trie.ParseVersion(binary.LittleEndian.Uint32(stateVersionBytes)) + _, err := trie.ParseVersion(version) if err != nil { logger.Errorf("failed parsing state version: %s", err) return 0 @@ -2272,8 +2269,7 @@ func ext_storage_root_version_2(ctx context.Context, m api.Module, version uint3 } storage := rtCtx.Storage - stateVersionBytes, _ := m.Memory().Read(version, 4) - stateVersion, err := trie.ParseVersion(binary.LittleEndian.Uint32(stateVersionBytes)) + stateVersion, err := trie.ParseVersion(version) if err != nil { logger.Errorf("failed parsing state version: %s", err) return mustWrite(m, rtCtx.Allocator, emptyByteVectorEncoded) diff --git a/lib/runtime/wazero/imports_test.go b/lib/runtime/wazero/imports_test.go index 3d8de44634..0b5ee9dc50 100644 --- a/lib/runtime/wazero/imports_test.go +++ b/lib/runtime/wazero/imports_test.go @@ -582,11 +582,11 @@ func Test_ext_trie_blake2_256_root_version_2(t *testing.T) { stateVersion := trie.V1 stateVersionInt := uint32(stateVersion) - stateVersionBytes := make([]byte, 4) - binary.LittleEndian.PutUint32(stateVersionBytes, stateVersionInt) + encVersion, err := scale.Marshal(stateVersionInt) + require.NoError(t, err) data := append([]byte{}, encInput...) - data = append(data, stateVersionBytes...) + data = append(data, encVersion...) res, err := inst.Exec("rtm_ext_trie_blake2_256_root_version_2", data) require.NoError(t, err) @@ -629,11 +629,11 @@ func Test_ext_trie_blake2_256_ordered_root_version_2(t *testing.T) { require.NoError(t, err) stateVersion := uint32(trie.V1) - stateVersionBytes := make([]byte, 4) - binary.LittleEndian.PutUint32(stateVersionBytes, stateVersion) + encVersion, err := scale.Marshal(stateVersion) + require.NoError(t, err) data := append([]byte{}, encValues...) - data = append(data, stateVersionBytes...) + data = append(data, encVersion...) res, err := inst.Exec("rtm_ext_trie_blake2_256_ordered_root_version_2", data) require.NoError(t, err) @@ -747,9 +747,10 @@ func Test_ext_trie_blake2_256_verify_proof_version_2(t *testing.T) { require.NoError(t, err) stateVersion := trie.V1 - stateVersionInt := uint32(trie.V1) - stateVersionBytes := make([]byte, 4) - binary.LittleEndian.PutUint32(stateVersionBytes, stateVersionInt) + + stateVersionInt := uint32(stateVersion) + encVersion, err := scale.Marshal(stateVersionInt) + require.NoError(t, err) otherTrie := trie.NewEmptyTrie() otherTrie.Put([]byte("simple"), []byte("cat")) @@ -827,7 +828,7 @@ func Test_ext_trie_blake2_256_verify_proof_version_2(t *testing.T) { require.NoError(t, err) args = append(args, valueEnc...) - args = append(args, stateVersionBytes...) + args = append(args, encVersion...) res, err := inst.Exec("rtm_ext_trie_blake2_256_verify_proof_version_2", args) require.NoError(t, err) @@ -1334,7 +1335,7 @@ func Test_ext_default_child_storage_root_version_2(t *testing.T) { require.NoError(t, err) stateVersionInt := uint32(stateVersion) - encVersion, err := scale.Marshal(&stateVersionInt) + encVersion, err := scale.Marshal(stateVersionInt) require.NoError(t, err) data := append([]byte{}, encChildKey...) @@ -2198,10 +2199,9 @@ func Test_ext_storage_root_version_2(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) stateVersion := uint32(trie.V1) - stateVersionBytes := make([]byte, 4) - binary.LittleEndian.PutUint32(stateVersionBytes, stateVersion) + encVersion, err := scale.Marshal(stateVersion) - ret, err := inst.Exec("rtm_ext_storage_root_version_2", stateVersionBytes) + ret, err := inst.Exec("rtm_ext_storage_root_version_2", encVersion) require.NoError(t, err) var hash []byte From 2cfbdca0ea119423e353f28adeac922ec1aa3862 Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 20 Nov 2023 14:07:25 -0300 Subject: [PATCH 125/128] Fix Test_ext_default_child_storage_root_version_2 --- lib/runtime/wazero/imports.go | 11 ++++------- lib/runtime/wazero/imports_test.go | 3 --- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/lib/runtime/wazero/imports.go b/lib/runtime/wazero/imports.go index d91d5ec630..fa4a18fa78 100644 --- a/lib/runtime/wazero/imports.go +++ b/lib/runtime/wazero/imports.go @@ -1264,21 +1264,18 @@ func ext_default_child_storage_root_version_2(ctx context.Context, m api.Module, panic("nil runtime context") } storage := rtCtx.Storage - child, err := storage.GetChild(read(m, childStorageKey)) + key := read(m, childStorageKey) + child, err := storage.GetChild(key) if err != nil { logger.Errorf("failed to retrieve child: %s", err) return mustWrite(m, rtCtx.Allocator, emptyByteVectorEncoded) } - // TODO: fix this to get the right version - /*stateVersionBytes, _ := m.Memory().Read(version, 4) - stateVersion, err := trie.ParseVersion(binary.LittleEndian.Uint32(stateVersionBytes)) + stateVersion, err := trie.ParseVersion(version) if err != nil { logger.Errorf("failed parsing state version: %s", err) return 0 - }*/ - - stateVersion := trie.V1 + } childRoot, err := stateVersion.Hash(child) if err != nil { diff --git a/lib/runtime/wazero/imports_test.go b/lib/runtime/wazero/imports_test.go index 0b5ee9dc50..8f9450cbb2 100644 --- a/lib/runtime/wazero/imports_test.go +++ b/lib/runtime/wazero/imports_test.go @@ -1331,15 +1331,12 @@ func Test_ext_default_child_storage_root_version_2(t *testing.T) { encChildKey, err := scale.Marshal(testChildKey) require.NoError(t, err) - encKey, err := scale.Marshal(testKey) - require.NoError(t, err) stateVersionInt := uint32(stateVersion) encVersion, err := scale.Marshal(stateVersionInt) require.NoError(t, err) data := append([]byte{}, encChildKey...) - data = append(data, encKey...) data = append(data, encVersion...) ret, err := inst.Exec("rtm_ext_default_child_storage_root_version_2", data) From 8f56a8d362130ca831852d24ce8a806f824bd1ce Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 20 Nov 2023 14:10:19 -0300 Subject: [PATCH 126/128] Use right test runtime URL --- lib/runtime/constants.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/runtime/constants.go b/lib/runtime/constants.go index 812263e3db..132f6ce7e7 100644 --- a/lib/runtime/constants.go +++ b/lib/runtime/constants.go @@ -8,7 +8,8 @@ const ( // This wasm is generated using https://github.com/ChainSafe/polkadot-spec. HOST_API_TEST_RUNTIME = "hostapi_runtime" HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm" - HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/raw/e1b3a95704373f2e51edda9c6cb174eed11e4f19/test/runtimes/hostapi/hostapi_runtime.compact.wasm" + HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/raw/master/test/" + + "runtimes/hostapi/hostapi_runtime.compact.wasm" // v0.9.29 polkadot POLKADOT_RUNTIME_v0929 = "polkadot_runtime-v929" From a747c37ea7704b6e740523afff6074895c0c1e34 Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 20 Nov 2023 14:14:23 -0300 Subject: [PATCH 127/128] Add missing err check --- lib/runtime/wazero/imports_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/runtime/wazero/imports_test.go b/lib/runtime/wazero/imports_test.go index 8f9450cbb2..5ecd82b41d 100644 --- a/lib/runtime/wazero/imports_test.go +++ b/lib/runtime/wazero/imports_test.go @@ -2197,6 +2197,7 @@ func Test_ext_storage_root_version_2(t *testing.T) { stateVersion := uint32(trie.V1) encVersion, err := scale.Marshal(stateVersion) + require.NoError(t, err) ret, err := inst.Exec("rtm_ext_storage_root_version_2", encVersion) require.NoError(t, err) From 8dcca66dbc9222008f06e0ec0128db5c3a3966ce Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 21 Nov 2023 12:48:01 -0300 Subject: [PATCH 128/128] Fix commands checks and doc --- cmd/gossamer/commands/import_state.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cmd/gossamer/commands/import_state.go b/cmd/gossamer/commands/import_state.go index f316aeb44c..43d24a4588 100644 --- a/cmd/gossamer/commands/import_state.go +++ b/cmd/gossamer/commands/import_state.go @@ -15,7 +15,10 @@ import ( func init() { ImportStateCmd.Flags().String("chain", "", "Chain id used to load default configuration for specified chain") ImportStateCmd.Flags().String("state-file", "", "Path to JSON file consisting of key-value pairs") - ImportStateCmd.Flags().Uint32("state-version", uint32(trie.DefaultStateVersion), "State version to use when importing state") //nolint:lll + ImportStateCmd.Flags().Uint32("state-version", + uint32(trie.DefaultStateVersion), + "State version to use when importing state", + ) ImportStateCmd.Flags().String("header-file", "", "Path to JSON file of block header corresponding to the given state") ImportStateCmd.Flags().Uint64("first-slot", 0, "The first BABE slot of the network") } @@ -28,7 +31,7 @@ var ImportStateCmd = &cobra.Command{ in the form of key-value pairs to be imported. Input can be generated by using the RPC function state_getPairs. Example: - gossamer import-state --state-file state.json state-version 1 --header-file header.json + gossamer import-state --state-file state.json --state-version 1 --header-file header.json --first-slot `, RunE: func(cmd *cobra.Command, args []string) error { return execImportState(cmd) @@ -59,7 +62,7 @@ func execImportState(cmd *cobra.Command) error { stateVersion, err := cmd.Flags().GetUint32("state-version") if err != nil { - return fmt.Errorf("failed to get state-file: %s", err) + return fmt.Errorf("failed to get state-version: %s", err) } stateTrieVersion, err := trie.ParseVersion(stateVersion) if err != nil {