Blocks log replay fix, enhance libtester to support customizing startup options, and add blocks log replay tests

libraries/chain/CMakeLists.txt

@@ -93,6 +93,7 @@ add_library( eosio_chain
              name.cpp
              transaction.cpp
              block.cpp
+             block_handle.cpp
              block_header.cpp
              block_header_state.cpp
              block_state.cpp

libraries/chain/block_handle.cpp

@@ -0,0 +1,37 @@
+#include <eosio/chain/block_handle.hpp>
+#include <fc/io/cfile.hpp>
+#include <filesystem>
+
+namespace eosio::chain {
+
+void block_handle::write(const std::filesystem::path& state_file) {
+   if (!is_valid())
+      return;
+
+   ilog("Writing chain_head block ${bn} ${id}", ("bn", block_num())("id", id()));
+
+   fc::datastream<fc::cfile> f;
+   f.set_file_path(state_file);
+   f.open("wb");
+
+   fc::raw::pack(f, *this);
+}
+
+bool block_handle::read(const std::filesystem::path& state_file) {
+   if (!std::filesystem::exists(state_file))
+      return false;
+
+   fc::datastream<fc::cfile> f;
+   f.set_file_path(state_file);
+   f.open("rb");
+
+   fc::raw::unpack(f, *this);
+
+   ilog("Loading chain_head block ${bn} ${id}", ("bn", block_num())("id", id()));
+
+   std::filesystem::remove(state_file);
+
+   return true;
+}
+
+} /// namespace eosio::chain

libraries/chain/controller.cpp

@@ -1238,7 +1238,7 @@ struct controller_impl {
     chain_id( chain_id ),
     read_mode( cfg.read_mode ),
     thread_pool(),
-    my_finalizers(cfg.finalizers_dir / "safety.dat"),
+    my_finalizers(cfg.finalizers_dir / config::safety_filename),
     wasmif( conf.wasm_runtime, conf.eosvmoc_tierup, db, conf.state_dir, conf.eosvmoc_config, !conf.profile_accounts.empty() )
    {
       assert(cfg.chain_thread_pool_size > 0);
@@ -1610,7 +1610,10 @@ struct controller_impl {
                      // note if is_proper_svnn_block is not reached then transistion will happen live
                   }
                });
-               if( check_shutdown() ) break; // needed on every loop for terminate-at-block
+               if( check_shutdown() ) {  // needed on every loop for terminate-at-block
+                  ilog( "quitting from replay_block_log because of shutdown" );
+                  break;
+               }
                if( next->block_num() % 500 == 0 ) {
                   ilog( "${n} of ${head}", ("n", next->block_num())("head", blog_head->block_num()) );
                }
@@ -1648,13 +1651,54 @@ struct controller_impl {
          ilog( "no block log found" );
       }
 
+      if( check_shutdown() ) {
+         ilog( "quitting from replay because of shutdown" );
+         return;
+      }
+
       try {
-         if (startup != startup_t::existing_state)
-           open_fork_db();
+         open_fork_db();
       } catch (const fc::exception& e) {
          elog( "Unable to open fork database, continuing without reversible blocks: ${e}", ("e", e));
       }
 
+      if (startup == startup_t::existing_state) {
+         EOS_ASSERT(fork_db_has_head(), fork_database_exception,
+                    "No existing fork database despite existing chain state. Replay required." );
+         uint32_t lib_num = fork_db_root_block_num();
+         auto first_block_num = blog.first_block_num();
+         if(blog_head) {
+            EOS_ASSERT( first_block_num <= lib_num && lib_num <= blog_head->block_num(),
+                        block_log_exception,
+                        "block log (ranging from ${block_log_first_num} to ${block_log_last_num}) does not contain the last irreversible block (${fork_db_lib})",
+                        ("block_log_first_num", first_block_num)
+                        ("block_log_last_num", blog_head->block_num())
+                        ("fork_db_lib", lib_num)
+            );
+            lib_num = blog_head->block_num();
+         } else {
+            if( first_block_num != (lib_num + 1) ) {
+               blog.reset( chain_id, lib_num + 1 );
+            }
+         }
+
+         auto do_startup = [&](auto& forkdb) {
+            if( read_mode == db_read_mode::IRREVERSIBLE) {
+               auto head = forkdb.head();
+               auto root = forkdb.root();
+               if (head && root && head->id() != root->id()) {
+                  forkdb.rollback_head_to_root();
+                  chain_head = block_handle{forkdb.head()};
+                  // rollback db to LIB
+                  while( db.revision() > chain_head.block_num() ) {
+                     db.undo();
+                  }
+               }
+            }
+         };
+         fork_db.apply<void>(do_startup);
+      }
+
       auto fork_db_reset_root_to_chain_head = [&]() {
          fork_db.apply<void>([&](auto& forkdb) {
             block_handle_accessor::apply<void>(chain_head, [&](const auto& head) {
@@ -1818,40 +1862,17 @@ struct controller_impl {
       EOS_ASSERT( db.revision() >= 1, database_exception,
                   "This version of controller::startup does not work with a fresh state database." );
 
-      open_fork_db();
-
-      EOS_ASSERT( fork_db_has_head(), fork_database_exception,
-                  "No existing fork database despite existing chain state. Replay required." );
-
       this->shutdown = std::move(shutdown);
       assert(this->shutdown);
       this->check_shutdown = std::move(check_shutdown);
       assert(this->check_shutdown);
-      uint32_t lib_num = fork_db_root_block_num();
-      auto first_block_num = blog.first_block_num();
-      if( auto blog_head = blog.head() ) {
-         EOS_ASSERT( first_block_num <= lib_num && lib_num <= blog_head->block_num(),
-                     block_log_exception,
-                     "block log (ranging from ${block_log_first_num} to ${block_log_last_num}) does not contain the last irreversible block (${fork_db_lib})",
-                     ("block_log_first_num", first_block_num)
-                     ("block_log_last_num", blog_head->block_num())
-                     ("fork_db_lib", lib_num)
-         );
-         lib_num = blog_head->block_num();
-      } else {
-         if( first_block_num != (lib_num + 1) ) {
-            blog.reset( chain_id, lib_num + 1 );
-         }
-      }
 
-      auto do_startup = [&](auto& forkdb) {
-         if( read_mode == db_read_mode::IRREVERSIBLE && forkdb.head()->id() != forkdb.root()->id() ) {
-            forkdb.rollback_head_to_root();
-         }
-         chain_head = block_handle{forkdb.head()};
-      };
+      bool valid = chain_head.read(conf.state_dir / config::chain_head_filename);
+      EOS_ASSERT( valid, database_exception, "No existing chain_head.dat file");
 
-      fork_db.apply<void>(do_startup);
+      EOS_ASSERT(db.revision() == chain_head.block_num(), database_exception,
+                 "chain_head block num ${bn} does not match chainbase revision ${r}",
+                 ("bn", chain_head.block_num())("r", db.revision()));
 
       init(startup_t::existing_state);
    }
@@ -1888,9 +1909,9 @@ struct controller_impl {
          });
       }
 
-      // At this point head != nullptr
+      // At this point chain_head != nullptr
       EOS_ASSERT( db.revision() >= chain_head.block_num(), fork_database_exception,
-                  "fork database head (${head}) is inconsistent with state (${db})",
+                  "chain head (${head}) is inconsistent with state (${db})",
                   ("db", db.revision())("head", chain_head.block_num()) );
 
       if( db.revision() > chain_head.block_num() ) {
@@ -1976,6 +1997,7 @@ struct controller_impl {
    }
 
    ~controller_impl() {
+      chain_head.write(conf.state_dir / config::chain_head_filename);
       pending.reset();
       //only log this not just if configured to, but also if initialization made it to the point we'd log the startup too
       if(okay_to_print_integrity_hash_on_stop && conf.integrity_hash_on_stop)

libraries/chain/fork_database.cpp

@@ -207,8 +207,8 @@ namespace eosio::chain {
    void fork_database_impl<BSP>::close_impl(std::ofstream& out) {
       assert(!!head && !!root); // if head or root are null, we don't save and shouldn't get here
 
-      ilog("Writing fork_database with root ${rn}:${r} and head ${hn}:${h}",
-           ("rn", root->block_num())("r", root->id())("hn", head->block_num())("h", head->id()));
+      ilog("Writing fork_database ${b} blocks with root ${rn}:${r} and head ${hn}:${h}",
+           ("b", head->block_num() - root->block_num())("rn", root->block_num())("r", root->id())("hn", head->block_num())("h", head->id()));
 
       fc::raw::pack( out, *root );
 

libraries/chain/include/eosio/chain/block_handle.hpp

@@ -2,6 +2,7 @@
 
 #include <eosio/chain/block_state_legacy.hpp>
 #include <eosio/chain/block_state.hpp>
+#include <fc/reflect/reflect.hpp>
 
 namespace eosio::chain {
 
@@ -11,6 +12,7 @@ struct block_handle {
 private:
    std::variant<block_state_legacy_ptr, block_state_ptr> _bsp;
 
+   friend struct fc::reflector<block_handle>;
    friend struct controller_impl;       // for `internal()` access below from controller
    friend struct block_handle_accessor; // for `internal()` access below from controller
 
@@ -22,6 +24,8 @@ struct block_handle {
    explicit block_handle(block_state_legacy_ptr bsp) : _bsp(std::move(bsp)) {}
    explicit block_handle(block_state_ptr bsp) : _bsp(std::move(bsp)) {}
 
+   bool is_valid() const { return _bsp.index() != std::variant_npos && std::visit([](const auto& bsp) { return !!bsp; }, _bsp); }
+
    uint32_t                block_num() const { return std::visit([](const auto& bsp) { return bsp->block_num(); }, _bsp); }
    block_timestamp_type    block_time() const { return std::visit([](const auto& bsp) { return bsp->timestamp(); }, _bsp); };
    const block_id_type&    id() const { return std::visit<const block_id_type&>([](const auto& bsp) -> const block_id_type& { return bsp->id(); }, _bsp); }
@@ -30,7 +34,10 @@ struct block_handle {
    const block_header&     header() const { return std::visit<const block_header&>([](const auto& bsp) -> const block_header& { return bsp->header; }, _bsp); };
    account_name            producer() const { return std::visit([](const auto& bsp) { return bsp->producer(); }, _bsp); }
 
-
+   void write(const std::filesystem::path& state_file);
+   bool read(const std::filesystem::path& state_file);
 };
 
 } // namespace eosio::chain
+
+FC_REFLECT(eosio::chain::block_handle, (_bsp))

libraries/chain/include/eosio/chain/config.hpp

@@ -13,7 +13,8 @@ const static auto reversible_blocks_dir_name  = "reversible";
 
 const static auto default_state_dir_name      = "state";
 const static auto forkdb_filename             = "fork_db.dat";
-const static auto safetydb_filename           = "safety_db.dat";
+const static auto safety_filename             = "safety.dat";
+const static auto chain_head_filename         = "chain_head.dat";
 const static auto default_state_size          = 1*1024*1024*1024ll;
 const static auto default_state_guard_size    =    128*1024*1024ll;
 

libraries/testing/include/eosio/testing/tester.hpp

@@ -71,6 +71,12 @@ namespace eosio::testing {
       full
    };
 
+   enum class call_startup_t {
+      no, // tester does not call startup() during initialization. The user must call
+          // `startup()` explicitly. See unittests/blocks_log_replay_tests.cpp for example.
+      yes // tester calls startup() during initialization.
+   };
+
    std::ostream& operator<<(std::ostream& os, setup_policy p);
 
    std::vector<uint8_t> read_wasm( const char* fn );
@@ -171,7 +177,7 @@ namespace eosio::testing {
 
          void              init(const setup_policy policy = setup_policy::full, db_read_mode read_mode = db_read_mode::HEAD, std::optional<uint32_t> genesis_max_inline_action_size = std::optional<uint32_t>{});
          void              init(controller::config config, const snapshot_reader_ptr& snapshot);
-         void              init(controller::config config, const genesis_state& genesis);
+         void              init(controller::config config, const genesis_state& genesis, call_startup_t call_startup);
          void              init(controller::config config);
          void              init(controller::config config, protocol_feature_set&& pfs, const snapshot_reader_ptr& snapshot);
          void              init(controller::config config, protocol_feature_set&& pfs, const genesis_state& genesis);
@@ -181,10 +187,10 @@ namespace eosio::testing {
          void              close();
          void              open( protocol_feature_set&& pfs, std::optional<chain_id_type> expected_chain_id, const std::function<void()>& lambda );
          void              open( protocol_feature_set&& pfs, const snapshot_reader_ptr& snapshot );
-         void              open( protocol_feature_set&& pfs, const genesis_state& genesis );
+         void              open( protocol_feature_set&& pfs, const genesis_state& genesis, call_startup_t call_startup );
          void              open( protocol_feature_set&& pfs, std::optional<chain_id_type> expected_chain_id = {} );
          void              open( const snapshot_reader_ptr& snapshot );
-         void              open( const genesis_state& genesis );
+         void              open( const genesis_state& genesis, call_startup_t call_startup );
          void              open( std::optional<chain_id_type> expected_chain_id = {} );
          bool              is_same_chain( base_tester& other );
 
@@ -566,8 +572,15 @@ namespace eosio::testing {
          init(policy, read_mode, genesis_max_inline_action_size);
       }
 
-      tester(controller::config config, const genesis_state& genesis) {
-         init(std::move(config), genesis);
+      // If `call_startup` is `yes`, tester starts the chain during initialization.
+      //
+      // If `call_startup` is `no`, tester does NOT start the chain during initialization;
+      // the user must call `startup()` explicitly.
+      // Before calling `startup()`, the user can do additional setups like connecting
+      // to a particular signal, and customizing shutdown conditions.
+      // See blocks_log_replay_tests.cpp in unit_test for an example.
+      tester(controller::config config, const genesis_state& genesis, call_startup_t call_startup = call_startup_t::yes) {
+         init(std::move(config), genesis, call_startup);
       }
 
       tester(controller::config config) {
@@ -583,7 +596,7 @@ namespace eosio::testing {
          cfg = def_conf.first;
 
          if (use_genesis) {
-            init(cfg, def_conf.second);
+            init(cfg, def_conf.second, call_startup_t::yes);
          }
          else {
             init(cfg);
@@ -597,7 +610,7 @@ namespace eosio::testing {
          conf_edit(cfg);
 
          if (use_genesis) {
-            init(cfg, def_conf.second);
+            init(cfg, def_conf.second, call_startup_t::yes);
          }
          else {
             init(cfg);
@@ -697,7 +710,7 @@ namespace eosio::testing {
 
          validating_node = create_validating_node(vcfg, def_conf.second, true, dmlog);
 
-         init(def_conf.first, def_conf.second);
+         init(def_conf.first, def_conf.second, call_startup_t::yes);
          execute_setup_policy(p);
       }
 
@@ -721,7 +734,7 @@ namespace eosio::testing {
          validating_node = create_validating_node(vcfg, def_conf.second, use_genesis);
 
          if (use_genesis) {
-            init(def_conf.first, def_conf.second);
+            init(def_conf.first, def_conf.second, call_startup_t::yes);
          } else {
             init(def_conf.first);
          }
@@ -736,7 +749,7 @@ namespace eosio::testing {
          validating_node = create_validating_node(vcfg, def_conf.second, use_genesis);
 
          if (use_genesis) {
-            init(def_conf.first, def_conf.second);
+            init(def_conf.first, def_conf.second, call_startup_t::yes);
          }  else {
             init(def_conf.first);
          }