Merge pull request #331 from AntelopeIO/clear_finality_log

[1.0-beta3 -> main] clear state history's finality log if savanna transition is forked out

libraries/state_history/include/eosio/state_history/log_catalog.hpp

@@ -156,6 +156,16 @@ class log_catalog {
       return first == second;
    }
 
+   void clear() {
+      if(empty())
+         return;
+
+      while(!retained_log_files.empty())
+         delete_bundle(retained_log_files.extract(retained_log_files.begin()).value().path_and_basename);
+      delete_head_log();
+      open_head_log();
+   }
+
 private:
    template<typename F>
    typename std::invoke_result_t<F,state_history_log&&> call_for_log(const uint32_t block_num, F&& f) {
@@ -222,11 +232,7 @@ class log_catalog {
    void unrotate_log() {
       catalog_t::node_type last_catalogued_file = retained_log_files.extract(std::prev(retained_log_files.end()));
 
-      for(const char* ext : {"log", "index"}) {
-         std::filesystem::path fp = std::filesystem::path(head_log_path_and_basename).replace_extension(ext);
-         if(std::filesystem::exists(fp))
-            std::filesystem::remove(fp);
-      }
+      delete_head_log();
 
       rename_bundle(last_catalogued_file.value().path_and_basename, head_log_path_and_basename);
       head_log = std::move(last_catalogued_file.value().log); //don't reopen the log, if we can avoid it
@@ -249,11 +255,7 @@ class log_catalog {
       } catch(std::exception& e) {
          wlog("Failed to rotate log ${pbn}", ("pbn", head_log_path_and_basename.string()));
          //remove any potentially created new head log files
-         for(const char* ext : {"log", "index"}) {
-            std::filesystem::path fp = std::filesystem::path(head_log_path_and_basename).replace_extension(ext);
-            if(std::filesystem::exists(fp))
-               std::filesystem::remove(fp);
-         }
+         delete_bundle(head_log_path_and_basename);
          //rename old logs back, restore head_log instance that was never closed, and don't continue with rotation
          rename_bundle(new_log_basenamepath, head_log_path_and_basename);
          head_log = std::move(old_head_log);
@@ -268,12 +270,10 @@ class log_catalog {
       while(retained_log_files.size() > max_retained_files) {
          const catalog_t::iterator it = retained_log_files.begin();
          std::filesystem::path oldest_log_path_and_basename = it->path_and_basename;
-         if(archive_dir.empty()) {
-            std::filesystem::remove(oldest_log_path_and_basename.replace_extension("log"));
-            std::filesystem::remove(oldest_log_path_and_basename.replace_extension("index"));
-         } else {
+         if(archive_dir.empty())
+            delete_bundle(oldest_log_path_and_basename);
+         else
             rename_bundle(oldest_log_path_and_basename, archive_dir / oldest_log_path_and_basename.filename());
-         }
          retained_log_files.erase(it);
       }
    }
@@ -282,6 +282,18 @@ class log_catalog {
       head_log.emplace(head_log_path_and_basename, non_local_get_block_id, prune_config);
    }
 
+   void delete_head_log() {
+      delete_bundle(head_log_path_and_basename);
+   }
+
+   void delete_bundle(std::filesystem::path path_and_basename) {
+      for(const char* ext : {"log", "index"}) {
+         std::filesystem::path fp = path_and_basename.replace_extension(ext);
+         if(std::filesystem::exists(fp))
+            std::filesystem::remove(fp);
+      }
+   }
+
    static std::filesystem::path make_absolute_dir(const std::filesystem::path& base_dir, std::filesystem::path new_dir) {
       if(new_dir.is_relative())
          new_dir = base_dir / new_dir;

plugins/state_history_plugin/state_history_plugin.cpp

@@ -191,8 +191,10 @@ struct state_history_plugin_impl {
          return;
 
       std::optional<finality_data_t> finality_data = chain_plug->chain().head_finality_data();
-      if(!finality_data.has_value())
+      if(!finality_data.has_value()) {
+         finality_data_log->clear();
          return;
+      }
 
       finality_data_log->pack_and_write_entry(id, previous_id, [finality_data](bio::filtering_ostreambuf& buf) {
          fc::datastream<boost::iostreams::filtering_ostreambuf&> ds{buf};

tests/ship_log.cpp

@@ -1449,4 +1449,72 @@ BOOST_AUTO_TEST_CASE(rewrite_too_old_pruned_block) try {
 
 } FC_LOG_AND_RETHROW();
 
+//verificaiton of clear()
+const state_history::state_history_log_config log_configs_for_clear[] = {
+   {std::monostate()},
+   {state_history::partition_config{
+      .retained_dir = {},
+      .archive_dir = {},
+      .stride = 5
+   }},
+   {state_history::partition_config{
+      .retained_dir = {},
+      .archive_dir = {},
+      .stride = 5,
+      .max_retained_files = 2
+   }},
+   {state_history::prune_config{
+      .prune_blocks = 5,
+      .prune_threshold = 2
+   }}
+};
+BOOST_DATA_TEST_CASE(clear, bdata::make(log_configs_for_clear) * bdata::make({9u, 10u, 11u}), config, after_clear_begin_block) try {
+   const fc::temp_directory tmpdir;
+
+   const unsigned before_clear_begin_block = 10;
+   const unsigned before_clear_end_block = 42;
+
+   const unsigned after_clear_end_block = after_clear_begin_block+4;
+
+   {
+      eosio::state_history::log_catalog lc(tmpdir.path(), config, "clearme");
+      for(unsigned i = before_clear_begin_block; i < before_clear_end_block; ++i)
+         lc.pack_and_write_entry(fake_blockid_for_num(i), fake_blockid_for_num(i-1), [&](bio::filtering_ostreambuf& obuf) {});
+
+      auto [begin_block, end_block] = lc.block_range();
+      //not checking begin_block because logs could have been rotated or pruned depending on test case
+      BOOST_REQUIRE_EQUAL(end_block, before_clear_end_block);
+
+      lc.clear();
+      BOOST_REQUIRE(lc.empty());
+      //head log should be empty
+      BOOST_REQUIRE_EQUAL(0u, std::filesystem::file_size(tmpdir.path() / "clearme.log"));
+      BOOST_REQUIRE_EQUAL(0u, std::filesystem::file_size(tmpdir.path() / "clearme.index"));
+      //make sure no retained logs exist
+      for(const std::string& suffix : {"log"s, "index"s}) {
+         const std::regex retained_logfile_regex(R"(^clearme-\d+-\d+\.)" + suffix + "$");
+
+         unsigned found = 0;
+         for(const std::filesystem::directory_entry& dir_entry : std::filesystem::directory_iterator(tmpdir.path()))
+            found += std::regex_search(dir_entry.path().filename().string(), retained_logfile_regex);
+         BOOST_REQUIRE_EQUAL(found, 0u);
+      }
+
+      for(unsigned i = after_clear_begin_block; i < after_clear_end_block; ++i)
+         lc.pack_and_write_entry(fake_blockid_for_num(i), fake_blockid_for_num(i-1), [&](bio::filtering_ostreambuf& obuf) {});
+
+      std::tie(begin_block, end_block) = lc.block_range();
+      BOOST_REQUIRE_EQUAL(begin_block, after_clear_begin_block);
+      BOOST_REQUIRE_EQUAL(end_block, after_clear_end_block);
+   }
+
+   //reopen for sanity check
+   {
+      eosio::state_history::log_catalog lc(tmpdir.path(), config, "clearme");
+      const auto [begin_block, end_block] = lc.block_range();
+      BOOST_REQUIRE_EQUAL(begin_block, after_clear_begin_block);
+      BOOST_REQUIRE_EQUAL(end_block, after_clear_end_block);
+   }
+} FC_LOG_AND_RETHROW();
+
 BOOST_AUTO_TEST_SUITE_END()