From 59fd1951c3fab00fb3972b5c02f796baa271b960 Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Wed, 7 Aug 2019 10:52:23 -0400 Subject: [PATCH 1/5] Cleanup --- x/simulation/simulate.go | 9 +++------ x/simulation/util.go | 16 +++++++++------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/x/simulation/simulate.go b/x/simulation/simulate.go index ca2b507c109c..8d835e5cf842 100644 --- a/x/simulation/simulate.go +++ b/x/simulation/simulate.go @@ -7,7 +7,6 @@ import ( "math/rand" "os" "os/signal" - "runtime/debug" "syscall" "testing" "time" @@ -126,14 +125,12 @@ func SimulateFromSeed( if !testingMode { b.ResetTimer() } else { - // Recover logs in case of panic + // recover logs in case of panic defer func() { if r := recover(); r != nil { - fmt.Fprintf(w, "panic with err: %v\n", r) - stackTrace := string(debug.Stack()) - fmt.Println(stackTrace) + _, _ = fmt.Fprintf(w, "simulation halted due to panic on block %d; %v\n", header.Height, r) logWriter.PrintLogs() - err = fmt.Errorf("Simulation halted due to panic on block %d", header.Height) + panic(r) } }() } diff --git a/x/simulation/util.go b/x/simulation/util.go index 5ed572b94da0..28c0f7ab091e 100644 --- a/x/simulation/util.go +++ b/x/simulation/util.go @@ -53,21 +53,23 @@ func getTestingMode(tb testing.TB) (testingMode bool, t *testing.T, b *testing.B // - "over stuffed" blocks with average size of 2 * avgblocksize, // - normal sized blocks, hitting avgBlocksize on average, // - and empty blocks, with no txs / only txs scheduled from the past. -func getBlockSize(r *rand.Rand, params Params, - lastBlockSizeState, avgBlockSize int) (state, blocksize int) { - +func getBlockSize(r *rand.Rand, params Params, lastBlockSizeState, avgBlockSize int) (state, blockSize int) { // TODO: Make default blocksize transition matrix actually make the average // blocksize equal to avgBlockSize. state = params.BlockSizeTransitionMatrix.NextState(r, lastBlockSizeState) + switch state { case 0: - blocksize = r.Intn(avgBlockSize * 4) + blockSize = r.Intn(avgBlockSize * 4) + case 1: - blocksize = r.Intn(avgBlockSize * 2) + blockSize = r.Intn(avgBlockSize * 2) + default: - blocksize = 0 + blockSize = 0 } - return state, blocksize + + return state, blockSize } // PeriodicInvariants returns an array of wrapped Invariants. Where each From 8d0b6b5ec77b5bbc8e9c55edd96f6ca3b2c85bb7 Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Wed, 7 Aug 2019 11:28:29 -0400 Subject: [PATCH 2/5] Cleanup and fix Makefile --- Makefile | 6 +++--- simapp/sim_test.go | 17 ++++++++++------- x/simulation/simulate.go | 17 +++++++++++------ 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 6ca698276529..eea23bde2539 100644 --- a/Makefile +++ b/Makefile @@ -86,8 +86,9 @@ test_race: @VERSION=$(VERSION) go test -mod=readonly -race $(PACKAGES_NOSIMULATION) test_sim_nondeterminism: - @echo "Running nondeterminism test..." - @go test -mod=readonly $(SIMAPP) -run TestAppStateDeterminism -Enabled=true -v -timeout 10m + @echo "Running non-determinism test..." + @go test -mod=readonly $(SIMAPP) -run TestAppStateDeterminism -Enabled=true \ + -NumBlocks=100 -BlockSize=200 -Commit=true -v -timeout 24h test_sim_custom_genesis_fast: @echo "Running custom genesis simulation..." @@ -125,7 +126,6 @@ test_sim_benchmark_invariants: .PHONY: test \ test_sim_nondeterminism \ test_sim_custom_genesis_fast \ -test_sim_fast \ test_sim_import_export \ test_sim_after_import \ test_sim_custom_genesis_multi_seed \ diff --git a/simapp/sim_test.go b/simapp/sim_test.go index ee828f3bae75..3c05c2b1daa6 100644 --- a/simapp/sim_test.go +++ b/simapp/sim_test.go @@ -681,7 +681,7 @@ func TestAppSimulationAfterImport(t *testing.T) { defer func() { newDB.Close() - os.RemoveAll(newDir) + _ = os.RemoveAll(newDir) }() newApp := NewSimApp(log.NewNopLogger(), newDB, nil, true, 0, fauxMerkleModeOpt) @@ -708,21 +708,24 @@ func TestAppStateDeterminism(t *testing.T) { for i := 0; i < numSeeds; i++ { seed := rand.Int63() + for j := 0; j < numTimesToRunPerSeed; j++ { logger := log.NewNopLogger() db := dbm.NewMemDB() app := NewSimApp(logger, db, nil, true, 0) - // Run randomized simulation - simulation.SimulateFromSeed( - t, os.Stdout, app.BaseApp, appStateFn, seed, - testAndRunTxs(app), []sdk.Invariant{}, - 1, 50, 100, 0, "", - false, true, false, false, false, app.ModuleAccountAddrs(), + _, _, err := simulation.SimulateFromSeed( + t, os.Stdout, app.BaseApp, appStateFn, seed, testAndRunTxs(app), + []sdk.Invariant{}, 1, numBlocks, exportParamsHeight, + blockSize, "", false, commit, lean, + false, false, app.ModuleAccountAddrs(), ) + require.NoError(t, err) + appHash := app.LastCommitID().Hash appHashList[j] = appHash } + for k := 1; k < numTimesToRunPerSeed; k++ { require.Equal(t, appHashList[0], appHashList[k], "appHash list: %v", appHashList) } diff --git a/x/simulation/simulate.go b/x/simulation/simulate.go index 8d835e5cf842..2b5857811db8 100644 --- a/x/simulation/simulate.go +++ b/x/simulation/simulate.go @@ -255,22 +255,27 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, w io.Wr operationQueue OperationQueue, timeOperationQueue []FutureOperation, totalNumBlocks, avgBlockSize int, logWriter LogWriter, lean, onOperation, allInvariants bool) blockSimFn { - lastBlocksizeState := 0 // state for [4 * uniform distribution] + lastBlockSizeState := 0 // state for [4 * uniform distribution] blocksize := 0 selectOp := ops.getSelectOpFn() - return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, - accounts []Account, header abci.Header) (opCount int) { + return func( + r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accounts []Account, header abci.Header, + ) (opCount int) { - fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ", - header.Height, totalNumBlocks, opCount, blocksize) - lastBlocksizeState, blocksize = getBlockSize(r, params, lastBlocksizeState, avgBlockSize) + _, _ = fmt.Fprintf( + w, "\rSimulating... block %d/%d, operation %d/%d.", + header.Height, totalNumBlocks, opCount, blocksize, + ) + lastBlockSizeState, blocksize = getBlockSize(r, params, lastBlockSizeState, avgBlockSize) type opAndR struct { op Operation rand *rand.Rand } + opAndRz := make([]opAndR, 0, blocksize) + // Predetermine the blocksize slice so that we can do things like block // out certain operations without changing the ops that follow. for i := 0; i < blocksize; i++ { From 95903a41ad2e4f7cf6f295dc14cfbcf86c00051c Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Wed, 7 Aug 2019 11:41:59 -0400 Subject: [PATCH 3/5] Add print statement for exec --- simapp/sim_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/simapp/sim_test.go b/simapp/sim_test.go index 3c05c2b1daa6..843496e9d47b 100644 --- a/simapp/sim_test.go +++ b/simapp/sim_test.go @@ -714,6 +714,11 @@ func TestAppStateDeterminism(t *testing.T) { db := dbm.NewMemDB() app := NewSimApp(logger, db, nil, true, 0) + fmt.Printf( + "Running non-determinism simulation; seed: %d/%d (%d), attempt: %d/%d\n", + i+1, numSeeds, seed, j+1, numTimesToRunPerSeed, + ) + _, _, err := simulation.SimulateFromSeed( t, os.Stdout, app.BaseApp, appStateFn, seed, testAndRunTxs(app), []sdk.Invariant{}, 1, numBlocks, exportParamsHeight, From 822057193ff358b0020253159c3fbb232bd240a8 Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Wed, 7 Aug 2019 11:43:35 -0400 Subject: [PATCH 4/5] Add pending log entry --- .pending/bugfixes/simulation/_Fix-non-determinism | 1 + 1 file changed, 1 insertion(+) create mode 100644 .pending/bugfixes/simulation/_Fix-non-determinism diff --git a/.pending/bugfixes/simulation/_Fix-non-determinism b/.pending/bugfixes/simulation/_Fix-non-determinism new file mode 100644 index 000000000000..fa1a0cb6fd0c --- /dev/null +++ b/.pending/bugfixes/simulation/_Fix-non-determinism @@ -0,0 +1 @@ +Fix non-determinism simulation by using CLI flags as input and updating Makefile target \ No newline at end of file From 995940f0c76a8d7234ccc0b4112ddb4beb10db8e Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Wed, 7 Aug 2019 12:22:04 -0400 Subject: [PATCH 5/5] Update pending log --- .pending/bugfixes/simulation/_Fix-non-determinism | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pending/bugfixes/simulation/_Fix-non-determinism b/.pending/bugfixes/simulation/_Fix-non-determinism index fa1a0cb6fd0c..7a1b606ffae1 100644 --- a/.pending/bugfixes/simulation/_Fix-non-determinism +++ b/.pending/bugfixes/simulation/_Fix-non-determinism @@ -1 +1,2 @@ -Fix non-determinism simulation by using CLI flags as input and updating Makefile target \ No newline at end of file +[\#4861](https://github.com/cosmos/cosmos-sdk/pull/4861) Fix non-determinism simulation +by using CLI flags as input and updating Makefile target. \ No newline at end of file