[R4R] #2730 add tx search pagination related CLI/REST API parameter

PENDING.md

@@ -43,6 +43,7 @@ FEATURES
 
 * Gaia CLI  (`gaiacli`)
   * \#2399 Implement `params` command to query slashing parameters.
+  * [\#2730](https://github.com/cosmos/cosmos-sdk/issues/2730) Add tx search pagination parameter
   * [\#3027](https://github.com/cosmos/cosmos-sdk/issues/3027) Implement
   `query gov proposer [proposal-id]` to query for a proposal's proposer.
 

client/tx/search.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"strconv"
 	"strings"
 
 	"github.com/cosmos/cosmos-sdk/client"
@@ -17,11 +18,16 @@ import (
 	"github.com/spf13/viper"
 
 	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	"github.com/tendermint/tendermint/types"
 )
 
 const (
-	flagTags = "tags"
-	flagAny  = "any"
+	flagTags     = "tags"
+	flagAny      = "any"
+	flagPage     = "page"
+	flagLimit    = "limit"
+	defaultPage  = 1
+	defaultLimit = 30 // should be consistent with tendermint/tendermint/rpc/core/pipe.go:19
 )
 
 // default client command to search through tagged transactions
@@ -32,7 +38,7 @@ func SearchTxCmd(cdc *codec.Codec) *cobra.Command {
 		Long: strings.TrimSpace(`
 Search for transactions that match exactly the given tags. For example:
 
-$ gaiacli query txs --tags '<tag1>:<value1>&<tag2>:<value2>'
+$ gaiacli query txs --tags '<tag1>:<value1>&<tag2>:<value2>' --page 1 --limit 30
 `),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			tagsStr := viper.GetString(flagTags)
@@ -53,12 +59,18 @@ $ gaiacli query txs --tags '<tag1>:<value1>&<tag2>:<value2>'
 				}
 
 				keyValue := strings.Split(tag, ":")
-				tag = fmt.Sprintf("%s='%s'", keyValue[0], keyValue[1])
+				if keyValue[0] == types.TxHeightKey {
+					tag = fmt.Sprintf("%s=%s", keyValue[0], keyValue[1])
+				} else {
+					tag = fmt.Sprintf("%s='%s'", keyValue[0], keyValue[1])
+				}
 				tmTags = append(tmTags, tag)
 			}
+			page := viper.GetInt(flagPage)
+			limit := viper.GetInt(flagLimit)
 
 			cliCtx := context.NewCLIContext().WithCodec(cdc)
-			txs, err := SearchTxs(cliCtx, cdc, tmTags)
+			txs, err := SearchTxs(cliCtx, cdc, tmTags, page, limit)
 			if err != nil {
 				return err
 			}
@@ -86,17 +98,27 @@ $ gaiacli query txs --tags '<tag1>:<value1>&<tag2>:<value2>'
 	cmd.Flags().Bool(client.FlagTrustNode, false, "Trust connected full node (don't verify proofs for responses)")
 	viper.BindPFlag(client.FlagTrustNode, cmd.Flags().Lookup(client.FlagTrustNode))
 	cmd.Flags().String(flagTags, "", "tag:value list of tags that must match")
+	cmd.Flags().Int32(flagPage, defaultPage, "Query a specific page of paginated results")
+	cmd.Flags().Int32(flagLimit, defaultLimit, "Query number of transactions results per page returned")
 	return cmd
 }
 
 // SearchTxs performs a search for transactions for a given set of tags via
 // Tendermint RPC. It returns a slice of Info object containing txs and metadata.
 // An error is returned if the query fails.
-func SearchTxs(cliCtx context.CLIContext, cdc *codec.Codec, tags []string) ([]Info, error) {
+func SearchTxs(cliCtx context.CLIContext, cdc *codec.Codec, tags []string, page, limit int) ([]Info, error) {
 	if len(tags) == 0 {
 		return nil, errors.New("must declare at least one tag to search")
 	}
 
+	if page <= 0 {
+		return nil, errors.New("page must greater than 0")
+	}
+
+	if limit <= 0 {
+		return nil, errors.New("limit must greater than 0")
+	}
+
 	// XXX: implement ANY
 	query := strings.Join(tags, " AND ")
 
@@ -108,10 +130,7 @@ func SearchTxs(cliCtx context.CLIContext, cdc *codec.Codec, tags []string) ([]In
 
 	prove := !cliCtx.TrustNode
 
-	// TODO: take these as args
-	page := 0
-	perPage := 100
-	res, err := node.TxSearch(query, prove, page, perPage)
+	res, err := node.TxSearch(query, prove, page, limit)
 	if err != nil {
 		return nil, err
 	}
@@ -153,6 +172,7 @@ func FormatTxResults(cdc *codec.Codec, res []*ctypes.ResultTx) ([]Info, error) {
 func SearchTxRequestHandlerFn(cliCtx context.CLIContext, cdc *codec.Codec) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		var tags []string
+		var page, limit int
 		var txs []Info
 		err := r.ParseForm()
 		if err != nil {
@@ -164,18 +184,14 @@ func SearchTxRequestHandlerFn(cliCtx context.CLIContext, cdc *codec.Codec) http.
 			return
 		}
 
-		for key, values := range r.Form {
-			value, err := url.QueryUnescape(values[0])
-			if err != nil {
-				utils.WriteErrorResponse(w, http.StatusBadRequest, sdk.AppendMsgToErr("could not decode query value", err.Error()))
-				return
-			}
+		tags, page, limit, err = parseHTTPArgs(r)
 
-			tag := fmt.Sprintf("%s='%s'", key, value)
-			tags = append(tags, tag)
+		if err != nil {
+			utils.WriteErrorResponse(w, http.StatusBadRequest, err.Error())
+			return
 		}
 
-		txs, err = SearchTxs(cliCtx, cdc, tags)
+		txs, err = SearchTxs(cliCtx, cdc, tags, page, limit)
 		if err != nil {
 			utils.WriteErrorResponse(w, http.StatusInternalServerError, err.Error())
 			return
@@ -184,3 +200,51 @@ func SearchTxRequestHandlerFn(cliCtx context.CLIContext, cdc *codec.Codec) http.
 		utils.PostProcessResponse(w, cdc, txs, cliCtx.Indent)
 	}
 }
+
+func parseHTTPArgs(r *http.Request) (tags []string, page, limit int, err error) {
+	tags = make([]string, 0, len(r.Form))
+	for key, values := range r.Form {
+		if key == "page" || key == "limit" {
+			continue
+		}
+		var value string
+		value, err = url.QueryUnescape(values[0])
+		if err != nil {
+			return tags, page, limit, err
+		}
+
+		var tag string
+		if key == types.TxHeightKey {
+			tag = fmt.Sprintf("%s=%s", key, value)
+		} else {
+			tag = fmt.Sprintf("%s='%s'", key, value)
+		}
+		tags = append(tags, tag)
+	}
+
+	pageStr := r.FormValue("page")
+	if pageStr == "" {
+		page = defaultPage
+	} else {
+		page, err = strconv.Atoi(pageStr)
+		if err != nil {
+			return tags, page, limit, err
+		} else if page <= 0 {
+			return tags, page, limit, errors.New("page must greater than 0")
+		}
+	}
+
+	limitStr := r.FormValue("limit")
+	if limitStr == "" {
+		limit = defaultLimit
+	} else {
+		limit, err = strconv.Atoi(limitStr)
+		if err != nil {
+			return tags, page, limit, err
+		} else if limit <= 0 {
+			return tags, page, limit, errors.New("limit must greater than 0")
+		}
+	}
+
+	return tags, page, limit, nil
+}

cmd/gaia/app/app.go

@@ -44,8 +44,8 @@ type GaiaApp struct {
 	// keys to access the substores
 	keyMain          *sdk.KVStoreKey
 	keyAccount       *sdk.KVStoreKey
-	keyStaking         *sdk.KVStoreKey
-	tkeyStaking        *sdk.TransientStoreKey
+	keyStaking       *sdk.KVStoreKey
+	tkeyStaking      *sdk.TransientStoreKey
 	keySlashing      *sdk.KVStoreKey
 	keyMint          *sdk.KVStoreKey
 	keyDistr         *sdk.KVStoreKey
@@ -59,7 +59,7 @@ type GaiaApp struct {
 	accountKeeper       auth.AccountKeeper
 	feeCollectionKeeper auth.FeeCollectionKeeper
 	bankKeeper          bank.Keeper
-	stakingKeeper         staking.Keeper
+	stakingKeeper       staking.Keeper
 	slashingKeeper      slashing.Keeper
 	mintKeeper          mint.Keeper
 	distrKeeper         distr.Keeper
@@ -79,8 +79,8 @@ func NewGaiaApp(logger log.Logger, db dbm.DB, traceStore io.Writer, loadLatest b
 		cdc:              cdc,
 		keyMain:          sdk.NewKVStoreKey(bam.MainStoreKey),
 		keyAccount:       sdk.NewKVStoreKey(auth.StoreKey),
-		keyStaking:         sdk.NewKVStoreKey(staking.StoreKey),
-		tkeyStaking:        sdk.NewTransientStoreKey(staking.TStoreKey),
+		keyStaking:       sdk.NewKVStoreKey(staking.StoreKey),
+		tkeyStaking:      sdk.NewTransientStoreKey(staking.TStoreKey),
 		keyMint:          sdk.NewKVStoreKey(mint.StoreKey),
 		keyDistr:         sdk.NewKVStoreKey(distr.StoreKey),
 		tkeyDistr:        sdk.NewTransientStoreKey(distr.TStoreKey),

cmd/gaia/app/genesis.go

@@ -34,7 +34,7 @@ var (
 type GenesisState struct {
 	Accounts     []GenesisAccount      `json:"accounts"`
 	AuthData     auth.GenesisState     `json:"auth"`
-	StakingData    staking.GenesisState  `json:"staking"`
+	StakingData  staking.GenesisState  `json:"staking"`
 	MintData     mint.GenesisState     `json:"mint"`
 	DistrData    distr.GenesisState    `json:"distr"`
 	GovData      gov.GenesisState      `json:"gov"`
@@ -50,7 +50,7 @@ func NewGenesisState(accounts []GenesisAccount, authData auth.GenesisState,
 	return GenesisState{
 		Accounts:     accounts,
 		AuthData:     authData,
-		StakingData:    stakingData,
+		StakingData:  stakingData,
 		MintData:     mintData,
 		DistrData:    distrData,
 		GovData:      govData,
@@ -144,7 +144,7 @@ func NewDefaultGenesisState() GenesisState {
 	return GenesisState{
 		Accounts:     nil,
 		AuthData:     auth.DefaultGenesisState(),
-		StakingData:    staking.DefaultGenesisState(),
+		StakingData:  staking.DefaultGenesisState(),
 		MintData:     mint.DefaultGenesisState(),
 		DistrData:    distr.DefaultGenesisState(),
 		GovData:      gov.DefaultGenesisState(),

cmd/gaia/app/sim_test.go

@@ -163,7 +163,7 @@ func appStateFn(r *rand.Rand, accs []simulation.Account) json.RawMessage {
 	genesis := GenesisState{
 		Accounts:     genesisAccounts,
 		AuthData:     authGenesis,
-		StakingData:    stakingGenesis,
+		StakingData:  stakingGenesis,
 		MintData:     mintGenesis,
 		DistrData:    distr.DefaultGenesisWithValidators(valAddrs),
 		SlashingData: slashingGenesis,

cmd/gaia/cli_test/cli_test.go

@@ -1,6 +1,7 @@
 package clitest
 
 import (
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -354,7 +355,7 @@ func TestGaiaCLISubmitProposal(t *testing.T) {
 	tests.WaitForNextNBlocksTM(1, f.Port)
 
 	// Ensure transaction tags can be queried
-	txs := f.QueryTxs("action:submit_proposal", fmt.Sprintf("proposer:%s", fooAddr))
+	txs := f.QueryTxs(1, 50, "action:submit_proposal", fmt.Sprintf("proposer:%s", fooAddr))
 	require.Len(t, txs, 1)
 
 	// Ensure deposit was deducted
@@ -397,7 +398,7 @@ func TestGaiaCLISubmitProposal(t *testing.T) {
 	require.Equal(t, int64(15), deposit.Amount.AmountOf(denom).Int64())
 
 	// Ensure tags are set on the transaction
-	txs = f.QueryTxs("action:deposit", fmt.Sprintf("depositor:%s", fooAddr))
+	txs = f.QueryTxs(1, 50, "action:deposit", fmt.Sprintf("depositor:%s", fooAddr))
 	require.Len(t, txs, 1)
 
 	// Ensure account has expected amount of funds
@@ -434,7 +435,7 @@ func TestGaiaCLISubmitProposal(t *testing.T) {
 	require.Equal(t, gov.OptionYes, votes[0].Option)
 
 	// Ensure tags are applied to voting transaction properly
-	txs = f.QueryTxs("action:vote", fmt.Sprintf("voter:%s", fooAddr))
+	txs = f.QueryTxs(1, 50, "action:vote", fmt.Sprintf("voter:%s", fooAddr))
 	require.Len(t, txs, 1)
 
 	// Ensure no proposals in deposit period
@@ -456,6 +457,54 @@ func TestGaiaCLISubmitProposal(t *testing.T) {
 	f.Cleanup()
 }
 
+func TestGaiaCLIQueryTxPagination(t *testing.T) {
+	t.Parallel()
+	f := InitFixtures(t)
+
+	// start gaiad server
+	proc := f.GDStart()
+	defer proc.Stop(false)
+
+	fooAddr := f.KeyAddress(keyFoo)
+	barAddr := f.KeyAddress(keyBar)
+
+	for i := 1; i <= 30; i++ {
+		success := executeWrite(t, fmt.Sprintf(
+			"gaiacli tx send %s --amount=%dfootoken --to=%s --from=foo",
+			f.Flags(), i, barAddr), app.DefaultKeyPass)
+		require.True(t, success)
+		tests.WaitForNextNBlocksTM(1, f.Port)
+	}
+
+	// perPage = 15, 2 pages
+	txsPage1 := f.QueryTxs(1, 15, fmt.Sprintf("sender:%s", fooAddr))
+	require.Len(t, txsPage1, 15)
+	txsPage2 := f.QueryTxs(2, 15, fmt.Sprintf("sender:%s", fooAddr))
+	require.Len(t, txsPage2, 15)
+	require.NotEqual(t, txsPage1, txsPage2)
+	txsPage3 := f.QueryTxs(3, 15, fmt.Sprintf("sender:%s", fooAddr))
+	require.Len(t, txsPage3, 15)
+	require.Equal(t, txsPage2, txsPage3)
+
+	// perPage = 16, 2 pages
+	txsPage1 = f.QueryTxs(1, 16, fmt.Sprintf("sender:%s", fooAddr))
+	require.Len(t, txsPage1, 16)
+	txsPage2 = f.QueryTxs(2, 16, fmt.Sprintf("sender:%s", fooAddr))
+	require.Len(t, txsPage2, 14)
+	require.NotEqual(t, txsPage1, txsPage2)
+
+	// perPage = 50
+	txsPageFull := f.QueryTxs(1, 50, fmt.Sprintf("sender:%s", fooAddr))
+	require.Len(t, txsPageFull, 30)
+	require.Equal(t, txsPageFull, append(txsPage1, txsPage2...))
+
+	// perPage = 0
+	f.QueryTxsInvalid(errors.New("ERROR: page must greater than 0"), 0, 50, fmt.Sprintf("sender:%s", fooAddr))
+
+	// limit = 0
+	f.QueryTxsInvalid(errors.New("ERROR: limit must greater than 0"), 1, 0, fmt.Sprintf("sender:%s", fooAddr))
+}
+
 func TestGaiaCLIValidateSignatures(t *testing.T) {
 	t.Parallel()
 	f := InitFixtures(t)

cmd/gaia/cli_test/test_helpers.go

@@ -295,8 +295,8 @@ func (f *Fixtures) QueryAccount(address sdk.AccAddress, flags ...string) auth.Ba
 // gaiacli query txs
 
 // QueryTxs is gaiacli query txs
-func (f *Fixtures) QueryTxs(tags ...string) []tx.Info {
-	cmd := fmt.Sprintf("gaiacli query txs --tags='%s' %v", queryTags(tags), f.Flags())
+func (f *Fixtures) QueryTxs(page, limit int, tags ...string) []tx.Info {
+	cmd := fmt.Sprintf("gaiacli query txs --page=%d --limit=%d --tags='%s' %v", page, limit, queryTags(tags), f.Flags())
 	out, _ := tests.ExecuteT(f.T, cmd, "")
 	var txs []tx.Info
 	cdc := app.MakeCodec()
@@ -305,6 +305,13 @@ func (f *Fixtures) QueryTxs(tags ...string) []tx.Info {
 	return txs
 }
 
+// QueryTxsInvalid query txs with wrong parameters and compare expected error
+func (f *Fixtures) QueryTxsInvalid(expectedErr error, page, limit int, tags ...string) {
+	cmd := fmt.Sprintf("gaiacli query txs --page=%d --limit=%d --tags='%s' %v", page, limit, queryTags(tags), f.Flags())
+	_, err := tests.ExecuteT(f.T, cmd, "")
+	require.EqualError(f.T, expectedErr, err)
+}
+
 //___________________________________________________________________________________
 // gaiacli query staking
 

cmd/gaia/cmd/gaiadebug/hack.go

@@ -133,8 +133,8 @@ type GaiaApp struct {
 	// keys to access the substores
 	keyMain     *sdk.KVStoreKey
 	keyAccount  *sdk.KVStoreKey
-	keyStaking    *sdk.KVStoreKey
-	tkeyStaking   *sdk.TransientStoreKey
+	keyStaking  *sdk.KVStoreKey
+	tkeyStaking *sdk.TransientStoreKey
 	keySlashing *sdk.KVStoreKey
 	keyParams   *sdk.KVStoreKey
 	tkeyParams  *sdk.TransientStoreKey
@@ -160,8 +160,8 @@ func NewGaiaApp(logger log.Logger, db dbm.DB, baseAppOptions ...func(*bam.BaseAp
 		cdc:         cdc,
 		keyMain:     sdk.NewKVStoreKey(bam.MainStoreKey),
 		keyAccount:  sdk.NewKVStoreKey(auth.StoreKey),
-		keyStaking:    sdk.NewKVStoreKey(staking.StoreKey),
-		tkeyStaking:   sdk.NewTransientStoreKey(staking.TStoreKey),
+		keyStaking:  sdk.NewKVStoreKey(staking.StoreKey),
+		tkeyStaking: sdk.NewTransientStoreKey(staking.TStoreKey),
 		keySlashing: sdk.NewKVStoreKey(slashing.StoreKey),
 		keyParams:   sdk.NewKVStoreKey(params.StoreKey),
 		tkeyParams:  sdk.NewTransientStoreKey(params.TStoreKey),