diff --git a/libraries/chain/CMakeLists.txt b/libraries/chain/CMakeLists.txt index 2a47d89cfa..64462ab654 100644 --- a/libraries/chain/CMakeLists.txt +++ b/libraries/chain/CMakeLists.txt @@ -93,6 +93,7 @@ add_library( eosio_chain name.cpp transaction.cpp block.cpp + block_handle.cpp block_header.cpp block_header_state.cpp block_state.cpp diff --git a/libraries/chain/block_handle.cpp b/libraries/chain/block_handle.cpp new file mode 100644 index 0000000000..3855a6f4f6 --- /dev/null +++ b/libraries/chain/block_handle.cpp @@ -0,0 +1,37 @@ +#include +#include +#include + +namespace eosio::chain { + +void block_handle::write(const std::filesystem::path& state_file) { + if (!is_valid()) + return; + + ilog("Writing chain_head block ${bn} ${id}", ("bn", block_num())("id", id())); + + fc::datastream f; + f.set_file_path(state_file); + f.open("wb"); + + fc::raw::pack(f, *this); +} + +bool block_handle::read(const std::filesystem::path& state_file) { + if (!std::filesystem::exists(state_file)) + return false; + + fc::datastream f; + f.set_file_path(state_file); + f.open("rb"); + + fc::raw::unpack(f, *this); + + ilog("Loading chain_head block ${bn} ${id}", ("bn", block_num())("id", id())); + + std::filesystem::remove(state_file); + + return true; +} + +} /// namespace eosio::chain diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index cc9481f4e3..6752b4d97a 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1238,7 +1238,7 @@ struct controller_impl { chain_id( chain_id ), read_mode( cfg.read_mode ), thread_pool(), - my_finalizers(cfg.finalizers_dir / "safety.dat"), + my_finalizers(cfg.finalizers_dir / config::safety_filename), wasmif( conf.wasm_runtime, conf.eosvmoc_tierup, db, conf.state_dir, conf.eosvmoc_config, !conf.profile_accounts.empty() ) { assert(cfg.chain_thread_pool_size > 0); @@ -1610,7 +1610,10 @@ struct controller_impl { // note if is_proper_svnn_block is not reached then transistion will happen live } }); - if( check_shutdown() ) break; // needed on every loop for terminate-at-block + if( check_shutdown() ) { // needed on every loop for terminate-at-block + ilog( "quitting from replay_block_log because of shutdown" ); + break; + } if( next->block_num() % 500 == 0 ) { ilog( "${n} of ${head}", ("n", next->block_num())("head", blog_head->block_num()) ); } @@ -1648,13 +1651,54 @@ struct controller_impl { ilog( "no block log found" ); } + if( check_shutdown() ) { + ilog( "quitting from replay because of shutdown" ); + return; + } + try { - if (startup != startup_t::existing_state) - open_fork_db(); + open_fork_db(); } catch (const fc::exception& e) { elog( "Unable to open fork database, continuing without reversible blocks: ${e}", ("e", e)); } + if (startup == startup_t::existing_state) { + EOS_ASSERT(fork_db_has_head(), fork_database_exception, + "No existing fork database despite existing chain state. Replay required." ); + uint32_t lib_num = fork_db_root_block_num(); + auto first_block_num = blog.first_block_num(); + if(blog_head) { + EOS_ASSERT( first_block_num <= lib_num && lib_num <= blog_head->block_num(), + block_log_exception, + "block log (ranging from ${block_log_first_num} to ${block_log_last_num}) does not contain the last irreversible block (${fork_db_lib})", + ("block_log_first_num", first_block_num) + ("block_log_last_num", blog_head->block_num()) + ("fork_db_lib", lib_num) + ); + lib_num = blog_head->block_num(); + } else { + if( first_block_num != (lib_num + 1) ) { + blog.reset( chain_id, lib_num + 1 ); + } + } + + auto do_startup = [&](auto& forkdb) { + if( read_mode == db_read_mode::IRREVERSIBLE) { + auto head = forkdb.head(); + auto root = forkdb.root(); + if (head && root && head->id() != root->id()) { + forkdb.rollback_head_to_root(); + chain_head = block_handle{forkdb.head()}; + // rollback db to LIB + while( db.revision() > chain_head.block_num() ) { + db.undo(); + } + } + } + }; + fork_db.apply(do_startup); + } + auto fork_db_reset_root_to_chain_head = [&]() { fork_db.apply([&](auto& forkdb) { block_handle_accessor::apply(chain_head, [&](const auto& head) { @@ -1818,40 +1862,17 @@ struct controller_impl { EOS_ASSERT( db.revision() >= 1, database_exception, "This version of controller::startup does not work with a fresh state database." ); - open_fork_db(); - - EOS_ASSERT( fork_db_has_head(), fork_database_exception, - "No existing fork database despite existing chain state. Replay required." ); - this->shutdown = std::move(shutdown); assert(this->shutdown); this->check_shutdown = std::move(check_shutdown); assert(this->check_shutdown); - uint32_t lib_num = fork_db_root_block_num(); - auto first_block_num = blog.first_block_num(); - if( auto blog_head = blog.head() ) { - EOS_ASSERT( first_block_num <= lib_num && lib_num <= blog_head->block_num(), - block_log_exception, - "block log (ranging from ${block_log_first_num} to ${block_log_last_num}) does not contain the last irreversible block (${fork_db_lib})", - ("block_log_first_num", first_block_num) - ("block_log_last_num", blog_head->block_num()) - ("fork_db_lib", lib_num) - ); - lib_num = blog_head->block_num(); - } else { - if( first_block_num != (lib_num + 1) ) { - blog.reset( chain_id, lib_num + 1 ); - } - } - auto do_startup = [&](auto& forkdb) { - if( read_mode == db_read_mode::IRREVERSIBLE && forkdb.head()->id() != forkdb.root()->id() ) { - forkdb.rollback_head_to_root(); - } - chain_head = block_handle{forkdb.head()}; - }; + bool valid = chain_head.read(conf.state_dir / config::chain_head_filename); + EOS_ASSERT( valid, database_exception, "No existing chain_head.dat file"); - fork_db.apply(do_startup); + EOS_ASSERT(db.revision() == chain_head.block_num(), database_exception, + "chain_head block num ${bn} does not match chainbase revision ${r}", + ("bn", chain_head.block_num())("r", db.revision())); init(startup_t::existing_state); } @@ -1888,9 +1909,9 @@ struct controller_impl { }); } - // At this point head != nullptr + // At this point chain_head != nullptr EOS_ASSERT( db.revision() >= chain_head.block_num(), fork_database_exception, - "fork database head (${head}) is inconsistent with state (${db})", + "chain head (${head}) is inconsistent with state (${db})", ("db", db.revision())("head", chain_head.block_num()) ); if( db.revision() > chain_head.block_num() ) { @@ -1976,6 +1997,7 @@ struct controller_impl { } ~controller_impl() { + chain_head.write(conf.state_dir / config::chain_head_filename); pending.reset(); //only log this not just if configured to, but also if initialization made it to the point we'd log the startup too if(okay_to_print_integrity_hash_on_stop && conf.integrity_hash_on_stop) diff --git a/libraries/chain/fork_database.cpp b/libraries/chain/fork_database.cpp index a898731a6e..cc23473973 100644 --- a/libraries/chain/fork_database.cpp +++ b/libraries/chain/fork_database.cpp @@ -207,8 +207,8 @@ namespace eosio::chain { void fork_database_impl::close_impl(std::ofstream& out) { assert(!!head && !!root); // if head or root are null, we don't save and shouldn't get here - ilog("Writing fork_database with root ${rn}:${r} and head ${hn}:${h}", - ("rn", root->block_num())("r", root->id())("hn", head->block_num())("h", head->id())); + ilog("Writing fork_database ${b} blocks with root ${rn}:${r} and head ${hn}:${h}", + ("b", head->block_num() - root->block_num())("rn", root->block_num())("r", root->id())("hn", head->block_num())("h", head->id())); fc::raw::pack( out, *root ); diff --git a/libraries/chain/include/eosio/chain/block_handle.hpp b/libraries/chain/include/eosio/chain/block_handle.hpp index 107a312c67..81f8b6e2b4 100644 --- a/libraries/chain/include/eosio/chain/block_handle.hpp +++ b/libraries/chain/include/eosio/chain/block_handle.hpp @@ -2,6 +2,7 @@ #include #include +#include namespace eosio::chain { @@ -11,6 +12,7 @@ struct block_handle { private: std::variant _bsp; + friend struct fc::reflector; friend struct controller_impl; // for `internal()` access below from controller friend struct block_handle_accessor; // for `internal()` access below from controller @@ -22,6 +24,8 @@ struct block_handle { explicit block_handle(block_state_legacy_ptr bsp) : _bsp(std::move(bsp)) {} explicit block_handle(block_state_ptr bsp) : _bsp(std::move(bsp)) {} + bool is_valid() const { return _bsp.index() != std::variant_npos && std::visit([](const auto& bsp) { return !!bsp; }, _bsp); } + uint32_t block_num() const { return std::visit([](const auto& bsp) { return bsp->block_num(); }, _bsp); } block_timestamp_type block_time() const { return std::visit([](const auto& bsp) { return bsp->timestamp(); }, _bsp); }; const block_id_type& id() const { return std::visit([](const auto& bsp) -> const block_id_type& { return bsp->id(); }, _bsp); } @@ -30,7 +34,10 @@ struct block_handle { const block_header& header() const { return std::visit([](const auto& bsp) -> const block_header& { return bsp->header; }, _bsp); }; account_name producer() const { return std::visit([](const auto& bsp) { return bsp->producer(); }, _bsp); } - + void write(const std::filesystem::path& state_file); + bool read(const std::filesystem::path& state_file); }; } // namespace eosio::chain + +FC_REFLECT(eosio::chain::block_handle, (_bsp)) diff --git a/libraries/chain/include/eosio/chain/config.hpp b/libraries/chain/include/eosio/chain/config.hpp index 41d663f2eb..f2822e9715 100644 --- a/libraries/chain/include/eosio/chain/config.hpp +++ b/libraries/chain/include/eosio/chain/config.hpp @@ -13,7 +13,8 @@ const static auto reversible_blocks_dir_name = "reversible"; const static auto default_state_dir_name = "state"; const static auto forkdb_filename = "fork_db.dat"; -const static auto safetydb_filename = "safety_db.dat"; +const static auto safety_filename = "safety.dat"; +const static auto chain_head_filename = "chain_head.dat"; const static auto default_state_size = 1*1024*1024*1024ll; const static auto default_state_guard_size = 128*1024*1024ll; diff --git a/libraries/testing/include/eosio/testing/tester.hpp b/libraries/testing/include/eosio/testing/tester.hpp index b48fe37c37..ac85283d90 100644 --- a/libraries/testing/include/eosio/testing/tester.hpp +++ b/libraries/testing/include/eosio/testing/tester.hpp @@ -71,6 +71,12 @@ namespace eosio::testing { full }; + enum class call_startup_t { + no, // tester does not call startup() during initialization. The user must call + // `startup()` explicitly. See unittests/blocks_log_replay_tests.cpp for example. + yes // tester calls startup() during initialization. + }; + std::ostream& operator<<(std::ostream& os, setup_policy p); std::vector read_wasm( const char* fn ); @@ -171,7 +177,7 @@ namespace eosio::testing { void init(const setup_policy policy = setup_policy::full, db_read_mode read_mode = db_read_mode::HEAD, std::optional genesis_max_inline_action_size = std::optional{}); void init(controller::config config, const snapshot_reader_ptr& snapshot); - void init(controller::config config, const genesis_state& genesis); + void init(controller::config config, const genesis_state& genesis, call_startup_t call_startup); void init(controller::config config); void init(controller::config config, protocol_feature_set&& pfs, const snapshot_reader_ptr& snapshot); void init(controller::config config, protocol_feature_set&& pfs, const genesis_state& genesis); @@ -181,10 +187,10 @@ namespace eosio::testing { void close(); void open( protocol_feature_set&& pfs, std::optional expected_chain_id, const std::function& lambda ); void open( protocol_feature_set&& pfs, const snapshot_reader_ptr& snapshot ); - void open( protocol_feature_set&& pfs, const genesis_state& genesis ); + void open( protocol_feature_set&& pfs, const genesis_state& genesis, call_startup_t call_startup ); void open( protocol_feature_set&& pfs, std::optional expected_chain_id = {} ); void open( const snapshot_reader_ptr& snapshot ); - void open( const genesis_state& genesis ); + void open( const genesis_state& genesis, call_startup_t call_startup ); void open( std::optional expected_chain_id = {} ); bool is_same_chain( base_tester& other ); @@ -566,8 +572,15 @@ namespace eosio::testing { init(policy, read_mode, genesis_max_inline_action_size); } - tester(controller::config config, const genesis_state& genesis) { - init(std::move(config), genesis); + // If `call_startup` is `yes`, tester starts the chain during initialization. + // + // If `call_startup` is `no`, tester does NOT start the chain during initialization; + // the user must call `startup()` explicitly. + // Before calling `startup()`, the user can do additional setups like connecting + // to a particular signal, and customizing shutdown conditions. + // See blocks_log_replay_tests.cpp in unit_test for an example. + tester(controller::config config, const genesis_state& genesis, call_startup_t call_startup = call_startup_t::yes) { + init(std::move(config), genesis, call_startup); } tester(controller::config config) { @@ -583,7 +596,7 @@ namespace eosio::testing { cfg = def_conf.first; if (use_genesis) { - init(cfg, def_conf.second); + init(cfg, def_conf.second, call_startup_t::yes); } else { init(cfg); @@ -597,7 +610,7 @@ namespace eosio::testing { conf_edit(cfg); if (use_genesis) { - init(cfg, def_conf.second); + init(cfg, def_conf.second, call_startup_t::yes); } else { init(cfg); @@ -697,7 +710,7 @@ namespace eosio::testing { validating_node = create_validating_node(vcfg, def_conf.second, true, dmlog); - init(def_conf.first, def_conf.second); + init(def_conf.first, def_conf.second, call_startup_t::yes); execute_setup_policy(p); } @@ -721,7 +734,7 @@ namespace eosio::testing { validating_node = create_validating_node(vcfg, def_conf.second, use_genesis); if (use_genesis) { - init(def_conf.first, def_conf.second); + init(def_conf.first, def_conf.second, call_startup_t::yes); } else { init(def_conf.first); } @@ -736,7 +749,7 @@ namespace eosio::testing { validating_node = create_validating_node(vcfg, def_conf.second, use_genesis); if (use_genesis) { - init(def_conf.first, def_conf.second); + init(def_conf.first, def_conf.second, call_startup_t::yes); } else { init(def_conf.first); } diff --git a/libraries/testing/tester.cpp b/libraries/testing/tester.cpp index b946880ae9..de5e766945 100644 --- a/libraries/testing/tester.cpp +++ b/libraries/testing/tester.cpp @@ -190,7 +190,7 @@ namespace eosio::testing { def_conf.first.read_mode = read_mode; cfg = def_conf.first; - open(def_conf.second); + open(def_conf.second, call_startup_t::yes); execute_setup_policy(policy); } @@ -199,9 +199,9 @@ namespace eosio::testing { open(snapshot); } - void base_tester::init(controller::config config, const genesis_state& genesis) { + void base_tester::init(controller::config config, const genesis_state& genesis, call_startup_t call_startup) { cfg = std::move(config); - open(genesis); + open(genesis, call_startup); } void base_tester::init(controller::config config) { @@ -216,7 +216,7 @@ namespace eosio::testing { void base_tester::init(controller::config config, protocol_feature_set&& pfs, const genesis_state& genesis) { cfg = std::move(config); - open(std::move(pfs), genesis); + open(std::move(pfs), genesis, call_startup_t::yes); } void base_tester::init(controller::config config, protocol_feature_set&& pfs) { @@ -312,8 +312,8 @@ namespace eosio::testing { open( make_protocol_feature_set(), snapshot ); } - void base_tester::open( const genesis_state& genesis ) { - open( make_protocol_feature_set(), genesis ); + void base_tester::open( const genesis_state& genesis, call_startup_t call_startup ) { + open( make_protocol_feature_set(), genesis, call_startup ); } void base_tester::open( std::optional expected_chain_id ) { @@ -371,10 +371,14 @@ namespace eosio::testing { }); } - void base_tester::open( protocol_feature_set&& pfs, const genesis_state& genesis ) { - open(std::move(pfs), genesis.compute_chain_id(), [&genesis,&control=this->control]() { - control->startup( [](){}, []() { return false; }, genesis ); - }); + void base_tester::open( protocol_feature_set&& pfs, const genesis_state& genesis, call_startup_t call_startup ) { + if (call_startup == call_startup_t::yes) { + open(std::move(pfs), genesis.compute_chain_id(), [&genesis,&control=this->control]() { + control->startup( [](){}, []() { return false; }, genesis ); + }); + } else { + open(std::move(pfs), genesis.compute_chain_id(), nullptr); + } } void base_tester::open( protocol_feature_set&& pfs, std::optional expected_chain_id ) { diff --git a/tests/nodeos_read_terminate_at_block_test.py b/tests/nodeos_read_terminate_at_block_test.py index 6963a52f4e..4b2e7f830b 100755 --- a/tests/nodeos_read_terminate_at_block_test.py +++ b/tests/nodeos_read_terminate_at_block_test.py @@ -74,6 +74,9 @@ def executeTest(cluster, testNodeId, testNodeArgs, resultMsgs): # also checking it stops at the correct block. checkReplay(testNode, testNodeArgs) + # verify node can be restarted after a replay + checkRestart(testNode, "--replay-blockchain") + resultDesc = "!!!TEST CASE #{} ({}) IS SUCCESSFUL".format( testNodeId, testNodeArgs @@ -141,6 +144,21 @@ def checkReplay(testNode, testNodeArgs): head, lib = getBlockNumInfo(testNode) assert head == termAtBlock, f"head {head} termAtBlock {termAtBlock}" +def checkRestart(testNode, rmChainArgs): + """Test restart of node continues""" + if testNode and not testNode.killed: + assert testNode.kill(signal.SIGTERM) + + if not testNode.relaunch(rmArgs=rmChainArgs): + Utils.errorExit(f"Unable to relaunch after {rmChainArgs}") + + assert testNode.verifyAlive(), f"relaunch failed after {rmChainArgs}" + + # getBlockNumInfo asserts relaunch was successful + head, lib = getBlockNumInfo(testNode) + + assert head >= lib, f"Sanity check of head {head} >= lib {lib} failed" + def getBlockNumInfo(testNode): head = None @@ -212,6 +230,27 @@ def executeSnapshotBlocklogTest(cluster, testNodeId, resultMsgs, nodeArgs, termA if testNode and not testNode.killed: assert testNode.kill(signal.SIGTERM) + if not testNode.relaunch(rmArgs=chainArg): + Utils.errorExit(f"Unable to relaunch after terminate-at-block {termAtBlock}") + + if testNode and not testNode.killed: + assert testNode.kill(signal.SIGTERM) + + if testResult: + testResult = False + # Check the node continued past the terminate block + errFileName=f"{cluster.nodeosLogPath}/node_{str(testNodeId).zfill(2)}/stderr.txt" + with open(errFileName) as errFile: + for line in errFile: + m=re.search(r"Writing chain_head block ([\d]+)", line) + if m: + assert int(m.group(1)) > termAtBlock, f"End block number {m.group(1)} not greater than termAtBlock {termAtBlock}" + resultDesc = f"!!!TEST CASE #{testNodeId}a (replay block log after terminate, mode {nodeArgs} --terminate-at-block {termAtBlock}) IS SUCCESSFUL" + testResult = True + + Print(resultDesc) + resultMsgs.append(resultDesc) + return testResult # Setup cluster and it's wallet manager diff --git a/unittests/blocks_log_replay_tests.cpp b/unittests/blocks_log_replay_tests.cpp new file mode 100644 index 0000000000..f1acd6dea2 --- /dev/null +++ b/unittests/blocks_log_replay_tests.cpp @@ -0,0 +1,153 @@ +#include +#include + +// Test scenarios +// * replay through blocks log and reversible blocks +// * replay stopping in the middle of blocks log and resuming +// * replay stopping in the middle of reversible blocks and resuming +// + +BOOST_AUTO_TEST_SUITE(blocks_log_replay_tests) + +using namespace eosio::testing; +using namespace eosio::chain; + +struct blog_replay_fixture { + eosio::testing::tester chain; + uint32_t last_head_block_num {0}; // head_block_num at stopping + uint32_t last_irreversible_block_num {0}; // LIB at stopping + + // Create blocks log + blog_replay_fixture() { + // Create a few accounts and produce a few blocks to fill in blocks log + chain.create_account("replay1"_n); + chain.produce_blocks(1); + chain.create_account("replay2"_n); + chain.produce_blocks(1); + chain.create_account("replay3"_n); + + chain.produce_blocks(10); + + // Make sure the accounts were created + BOOST_REQUIRE_NO_THROW(chain.control->get_account("replay1"_n)); + BOOST_REQUIRE_NO_THROW(chain.control->get_account("replay2"_n)); + BOOST_REQUIRE_NO_THROW(chain.control->get_account("replay3"_n)); + + // Store head_block_num and irreversible_block_num when the node is stopped + last_head_block_num = chain.control->head_block_num(); + last_irreversible_block_num = chain.control->last_irreversible_block_num(); + + // Stop the node and save blocks_log + chain.close(); + } + + // Stop replay at block number `stop_at` via simulated ctrl-c and resume the replay afterward + void stop_and_resume_replay(uint32_t stop_at) try { + controller::config copied_config = chain.get_config(); + + auto genesis = block_log::extract_genesis_state(copied_config.blocks_dir); + BOOST_REQUIRE(genesis); + + // remove the state files to make sure starting from blocks log + remove_existing_states(copied_config.state_dir); + + // Create a replay chain without starting it + eosio::testing::tester replay_chain(copied_config, *genesis, call_startup_t::no); + + // Simulate shutdown by CTRL-C + bool is_quiting = false; + auto check_shutdown = [&is_quiting](){ return is_quiting; }; + + // Set up shutdown at a particular block number + replay_chain.control->irreversible_block().connect([&](const block_signal_params& t) { + const auto& [ block, id ] = t; + // Stop replay at block `stop_at` + if (block->block_num() == stop_at) { + is_quiting = true; + } + }); + + // Make sure reversible fork_db exists + BOOST_CHECK(std::filesystem::exists(copied_config.blocks_dir / config::reversible_blocks_dir_name / "fork_db.dat")); + + // Start replay and stop at block `stop_at` + replay_chain.control->startup( [](){}, check_shutdown, *genesis ); + replay_chain.close(); + + // Make sure reversible fork_db still exists + BOOST_CHECK(std::filesystem::exists(copied_config.blocks_dir / config::reversible_blocks_dir_name / "fork_db.dat")); + + // Prepare resuming replay + controller::config copied_config_1 = replay_chain.get_config(); + + // Resume replay + eosio::testing::tester replay_chain_1(copied_config_1, *genesis, call_startup_t::no); + replay_chain_1.control->startup( [](){}, []()->bool{ return false; } ); + + replay_chain_1.control->accepted_block().connect([&](const block_signal_params& t) { + const auto& [ block, id ] = t; + BOOST_TEST(block->block_num() > stop_at); + static uint32_t first = block->block_num(); + BOOST_TEST(first == stop_at); + }); + + // Make sure new chain contain the account created by original chain + BOOST_REQUIRE_NO_THROW(replay_chain_1.control->get_account("replay1"_n)); + BOOST_REQUIRE_NO_THROW(replay_chain_1.control->get_account("replay2"_n)); + BOOST_REQUIRE_NO_THROW(replay_chain_1.control->get_account("replay3"_n)); + + // Make sure replayed irreversible_block_num and head_block_num match + // with last_irreversible_block_num and last_head_block_num + BOOST_CHECK(replay_chain_1.control->last_irreversible_block_num() == last_irreversible_block_num); + BOOST_CHECK(replay_chain_1.control->head_block_num() == last_head_block_num); + } FC_LOG_AND_RETHROW() + + void remove_existing_states(std::filesystem::path& state_path) { + std::filesystem::remove_all(state_path); + std::filesystem::create_directories(state_path); + } +}; + +// Test replay through blocks log and reversible blocks +BOOST_FIXTURE_TEST_CASE(replay_through, blog_replay_fixture) try { + eosio::chain::controller::config copied_config = chain.get_config(); + + auto genesis = eosio::chain::block_log::extract_genesis_state(copied_config.blocks_dir); + BOOST_REQUIRE(genesis); + + // remove the state files to make sure we are starting from block log + remove_existing_states(copied_config.state_dir); + eosio::testing::tester replay_chain(copied_config, *genesis); + + // Make sure new chain contain the account created by original chain + BOOST_REQUIRE_NO_THROW(replay_chain.control->get_account("replay1"_n)); + BOOST_REQUIRE_NO_THROW(replay_chain.control->get_account("replay2"_n)); + BOOST_REQUIRE_NO_THROW(replay_chain.control->get_account("replay3"_n)); + + // Make sure replayed irreversible_block_num and head_block_num match + // with last_irreversible_block_num and last_head_block_num + BOOST_CHECK(replay_chain.control->last_irreversible_block_num() == last_irreversible_block_num); + BOOST_CHECK(replay_chain.control->head_block_num() == last_head_block_num); +} FC_LOG_AND_RETHROW() + +// Test replay stopping in the middle of blocks log and resuming +BOOST_FIXTURE_TEST_CASE(replay_stop_in_middle, blog_replay_fixture) try { + // block `last_irreversible_block_num - 1` is within blocks log + stop_and_resume_replay(last_irreversible_block_num - 1); +} FC_LOG_AND_RETHROW() + +// Test replay stopping in the middle of reversible blocks and resuming +BOOST_FIXTURE_TEST_CASE(replay_stop_in_reversible_blocks, blog_replay_fixture) try { + // block `last_head_block_num - 1` is within reversible_blocks, since in Savanna + // we have at least 2 reversible blocks + stop_and_resume_replay(last_head_block_num - 1); +} FC_LOG_AND_RETHROW() + +// Test replay stopping in the middle of reversible blocks and resuming +BOOST_FIXTURE_TEST_CASE(replay_stop_multiple, blog_replay_fixture) try { + stop_and_resume_replay(last_irreversible_block_num - 5); + stop_and_resume_replay(last_irreversible_block_num); + stop_and_resume_replay(last_head_block_num - 3); +} FC_LOG_AND_RETHROW() + +BOOST_AUTO_TEST_SUITE_END()