feat(dot/rpc/modules) implement state_queryStorage rpc method (Chai…

…nSafe#1707)

* wip: state query storage rpc

* chore: query keys from block stateRoot

* chore: implemented rpc method and format response correctly

* chore: check starting block hash is nil

* chore: use GetStorage instead of TrieState

* chore: remove logs

* chore: resolve lint issues

* chore: preallocate slice, use varidic param

* chore: update variable name

* chore: made from not nulable

* chore: get best block hash if to is nil

* chore: remove pointer and compare with EmptyHash

* chore: fix tests, remove write to nil reference

* chore: fix deepsource rule

* chore: fix rpc test

* chore: uncomment conditional test

* chore: improve testing

* chore: change from null to a empty array

* chore: skip state_queryStorage RPC method test

* chore: remove unconsitent test check

.github/workflows/tests.yml

@@ -132,6 +132,7 @@ jobs:
         name: Run stable tests
         run: |
           docker run chainsafe/gossamer:test sh -c "make it-stable"
+  
   docker-rpc-tests:
     runs-on: ubuntu-latest
     steps:
@@ -155,6 +156,7 @@ jobs:
         name: Run rpc tests
         run: |
           docker run chainsafe/gossamer:test sh -c "make it-rpc"
+  
   docker-stress-tests:
     runs-on: ubuntu-latest
     steps:
@@ -178,6 +180,7 @@ jobs:
         name: Run stress
         run: |
           docker run chainsafe/gossamer:test sh -c "make it-stress"
+  
   docker-grandpa-tests:
     runs-on: ubuntu-latest
     steps:

dot/core/interface.go

@@ -60,6 +60,7 @@ type StorageState interface {
 	TrieState(root *common.Hash) (*rtstorage.TrieState, error)
 	StoreTrie(*rtstorage.TrieState, *types.Header) error
 	GetStateRootFromBlock(bhash *common.Hash) (*common.Hash, error)
+	GetStorage(root *common.Hash, key []byte) ([]byte, error)
 }
 
 // TransactionState is the interface for transaction state methods

dot/core/service.go

@@ -40,6 +40,9 @@ var (
 	logger log.Logger       = log.New("pkg", "core")
 )
 
+// QueryKeyValueChanges represents the key-value data inside a block storage
+type QueryKeyValueChanges map[string]string
+
 // Service is an overhead layer that allows communication between the runtime,
 // BABE session, and network service. It deals with the validation of transactions
 // and blocks by calling their respective validation functions in the runtime.
@@ -560,3 +563,58 @@ func (s *Service) GetMetadata(bhash *common.Hash) ([]byte, error) {
 	rt.SetContextStorage(ts)
 	return rt.Metadata()
 }
+
+// QueryStorage returns the key-value data by block based on `keys` params
+// on every block starting `from` until `to` block, if `to` is not nil
+func (s *Service) QueryStorage(from, to common.Hash, keys ...string) (map[common.Hash]QueryKeyValueChanges, error) {
+	if to == common.EmptyHash {
+		to = s.blockState.BestBlockHash()
+	}
+
+	blocksToQuery, err := s.blockState.SubChain(from, to)
+	if err != nil {
+		return nil, err
+	}
+
+	queries := make(map[common.Hash]QueryKeyValueChanges)
+
+	for _, hash := range blocksToQuery {
+		changes, err := s.tryQueryStorage(hash, keys...)
+		if err != nil {
+			return nil, err
+		}
+
+		queries[hash] = changes
+	}
+
+	return queries, nil
+}
+
+// tryQueryStorage will try to get all the `keys` inside the block's current state
+func (s *Service) tryQueryStorage(block common.Hash, keys ...string) (QueryKeyValueChanges, error) {
+	stateRootHash, err := s.storageState.GetStateRootFromBlock(&block)
+	if err != nil {
+		return nil, err
+	}
+
+	changes := make(QueryKeyValueChanges)
+	for _, k := range keys {
+		keyBytes, err := common.HexToBytes(k)
+		if err != nil {
+			return nil, err
+		}
+
+		storedData, err := s.storageState.GetStorage(stateRootHash, keyBytes)
+		if err != nil {
+			return nil, err
+		}
+
+		if storedData == nil {
+			continue
+		}
+
+		changes[k] = common.BytesToHex(storedData)
+	}
+
+	return changes, nil
+}

dot/core/service_test.go

@@ -33,8 +33,10 @@ import (
 	"github.com/ChainSafe/gossamer/lib/keystore"
 	"github.com/ChainSafe/gossamer/lib/runtime"
 	"github.com/ChainSafe/gossamer/lib/runtime/extrinsic"
+	"github.com/ChainSafe/gossamer/lib/runtime/storage"
 	"github.com/ChainSafe/gossamer/lib/runtime/wasmer"
 	"github.com/ChainSafe/gossamer/lib/transaction"
+	"github.com/ChainSafe/gossamer/lib/trie"
 	"github.com/ChainSafe/gossamer/lib/utils"
 	log "github.com/ChainSafe/log15"
 	"github.com/stretchr/testify/mock"
@@ -619,6 +621,173 @@ func TestService_HandleRuntimeChangesAfterCodeSubstitutes(t *testing.T) {
 	require.NotEqualf(t, codeHashBefore, rt.GetCodeHash(), "expected different code hash after runtime update") // codeHash should change after runtime change
 }
 
+func TestTryQueryStore_WhenThereIsDataToRetrieve(t *testing.T) {
+	s := NewTestService(t, nil)
+	storageStateTrie, err := storage.NewTrieState(trie.NewTrie(nil))
+
+	testKey, testValue := []byte("to"), []byte("0x1723712318238AB12312")
+	storageStateTrie.Set(testKey, testValue)
+	require.NoError(t, err)
+
+	header, err := types.NewHeader(s.blockState.GenesisHash(), storageStateTrie.MustRoot(),
+		common.Hash{}, big.NewInt(1), nil)
+	require.NoError(t, err)
+
+	err = s.storageState.StoreTrie(storageStateTrie, header)
+	require.NoError(t, err)
+
+	testBlock := &types.Block{
+		Header: header,
+		Body:   types.NewBody([]byte{}),
+	}
+
+	err = s.blockState.AddBlock(testBlock)
+	require.NoError(t, err)
+
+	blockhash := testBlock.Header.Hash()
+	hexKey := common.BytesToHex(testKey)
+	keys := []string{hexKey}
+
+	changes, err := s.tryQueryStorage(blockhash, keys...)
+	require.NoError(t, err)
+
+	require.Equal(t, changes[hexKey], common.BytesToHex(testValue))
+}
+
+func TestTryQueryStore_WhenDoesNotHaveDataToRetrieve(t *testing.T) {
+	s := NewTestService(t, nil)
+	storageStateTrie, err := storage.NewTrieState(trie.NewTrie(nil))
+	require.NoError(t, err)
+
+	header, err := types.NewHeader(s.blockState.GenesisHash(), storageStateTrie.MustRoot(),
+		common.Hash{}, big.NewInt(1), nil)
+	require.NoError(t, err)
+
+	err = s.storageState.StoreTrie(storageStateTrie, header)
+	require.NoError(t, err)
+
+	testBlock := &types.Block{
+		Header: header,
+		Body:   types.NewBody([]byte{}),
+	}
+
+	err = s.blockState.AddBlock(testBlock)
+	require.NoError(t, err)
+
+	testKey := []byte("to")
+	blockhash := testBlock.Header.Hash()
+	hexKey := common.BytesToHex(testKey)
+	keys := []string{hexKey}
+
+	changes, err := s.tryQueryStorage(blockhash, keys...)
+	require.NoError(t, err)
+
+	require.Empty(t, changes)
+}
+
+func TestTryQueryState_WhenDoesNotHaveStateRoot(t *testing.T) {
+	s := NewTestService(t, nil)
+
+	header, err := types.NewHeader(s.blockState.GenesisHash(), common.Hash{}, common.Hash{}, big.NewInt(1), nil)
+	require.NoError(t, err)
+
+	testBlock := &types.Block{
+		Header: header,
+		Body:   types.NewBody([]byte{}),
+	}
+
+	err = s.blockState.AddBlock(testBlock)
+	require.NoError(t, err)
+
+	testKey := []byte("to")
+	blockhash := testBlock.Header.Hash()
+	hexKey := common.BytesToHex(testKey)
+	keys := []string{hexKey}
+
+	changes, err := s.tryQueryStorage(blockhash, keys...)
+	require.Error(t, err)
+	require.Nil(t, changes)
+}
+
+func TestQueryStorate_WhenBlocksHasData(t *testing.T) {
+	keys := []string{
+		common.BytesToHex([]byte("transfer.to")),
+		common.BytesToHex([]byte("transfer.from")),
+		common.BytesToHex([]byte("transfer.value")),
+	}
+
+	s := NewTestService(t, nil)
+
+	firstKey, firstValue := []byte("transfer.to"), []byte("some-address-herer")
+	firstBlock := createNewBlockAndStoreDataAtBlock(
+		t, s, firstKey, firstValue, s.blockState.GenesisHash(), 1,
+	)
+
+	secondKey, secondValue := []byte("transfer.from"), []byte("another-address-here")
+	secondBlock := createNewBlockAndStoreDataAtBlock(
+		t, s, secondKey, secondValue, firstBlock.Header.Hash(), 2,
+	)
+
+	thirdKey, thirdValue := []byte("transfer.value"), []byte("value-gigamegablaster")
+	thirdBlock := createNewBlockAndStoreDataAtBlock(
+		t, s, thirdKey, thirdValue, secondBlock.Header.Hash(), 3,
+	)
+
+	from := firstBlock.Header.Hash()
+	data, err := s.QueryStorage(from, common.Hash{}, keys...)
+	require.NoError(t, err)
+	require.Len(t, data, 3)
+
+	require.Equal(t, data[firstBlock.Header.Hash()], QueryKeyValueChanges(
+		map[string]string{
+			common.BytesToHex(firstKey): common.BytesToHex(firstValue),
+		},
+	))
+
+	from = secondBlock.Header.Hash()
+	to := thirdBlock.Header.Hash()
+
+	data, err = s.QueryStorage(from, to, keys...)
+	require.NoError(t, err)
+	require.Len(t, data, 2)
+
+	require.Equal(t, data[secondBlock.Header.Hash()], QueryKeyValueChanges(
+		map[string]string{
+			common.BytesToHex(secondKey): common.BytesToHex(secondValue),
+		},
+	))
+	require.Equal(t, data[thirdBlock.Header.Hash()], QueryKeyValueChanges(
+		map[string]string{
+			common.BytesToHex(thirdKey): common.BytesToHex(thirdValue),
+		},
+	))
+}
+
+func createNewBlockAndStoreDataAtBlock(t *testing.T, s *Service, key, value []byte, parentHash common.Hash, number int64) *types.Block {
+	t.Helper()
+
+	storageStateTrie, err := storage.NewTrieState(trie.NewTrie(nil))
+	storageStateTrie.Set(key, value)
+	require.NoError(t, err)
+
+	header, err := types.NewHeader(parentHash, storageStateTrie.MustRoot(),
+		common.Hash{}, big.NewInt(number), nil)
+	require.NoError(t, err)
+
+	err = s.storageState.StoreTrie(storageStateTrie, header)
+	require.NoError(t, err)
+
+	testBlock := &types.Block{
+		Header: header,
+		Body:   types.NewBody([]byte{}),
+	}
+
+	err = s.blockState.AddBlock(testBlock)
+	require.NoError(t, err)
+
+	return testBlock
+}
+
 func TestDecodeSessionKeys(t *testing.T) {
 	mockInstance := new(runtimemocks.MockInstance)
 	mockInstance.On("DecodeSessionKeys", mock.AnythingOfType("[]uint8")).Return([]byte{}, nil).Once()

dot/network/service.go

@@ -304,9 +304,16 @@ func (s *Service) collectNetworkMetrics() {
 }
 
 func (s *Service) logPeerCount() {
+	ticker := time.NewTicker(time.Second * 30)
+	defer ticker.Stop()
+
 	for {
-		logger.Debug("peer count", "num", s.host.peerCount(), "min", s.cfg.MinPeers, "max", s.cfg.MaxPeers)
-		time.Sleep(time.Second * 30)
+		select {
+		case <-ticker.C:
+			logger.Debug("peer count", "num", s.host.peerCount(), "min", s.cfg.MinPeers, "max", s.cfg.MaxPeers)
+		case <-s.ctx.Done():
+			return
+		}
 	}
 }
 

dot/rpc/modules/api.go

@@ -3,6 +3,7 @@ package modules
 import (
 	"math/big"
 
+	"github.com/ChainSafe/gossamer/dot/core"
 	"github.com/ChainSafe/gossamer/dot/state"
 	"github.com/ChainSafe/gossamer/dot/types"
 	"github.com/ChainSafe/gossamer/lib/common"
@@ -77,6 +78,7 @@ type CoreAPI interface {
 	GetRuntimeVersion(bhash *common.Hash) (runtime.Version, error)
 	HandleSubmittedExtrinsic(types.Extrinsic) error
 	GetMetadata(bhash *common.Hash) ([]byte, error)
+	QueryStorage(from, to common.Hash, keys ...string) (map[common.Hash]core.QueryKeyValueChanges, error)
 	DecodeSessionKeys(enc []byte) ([]byte, error)
 }