[3.2] Enhancement: Add capability of not genereating block log by setting option block-log-retain-blocks to 0

libraries/chain/block_log.cpp

@@ -48,14 +48,21 @@ namespace eosio { namespace chain {
             uint32_t                 first_block_num = 0;       //the first number available to read
             uint32_t                 index_first_block_num = 0; //the first number in index & the log had it not been pruned
             std::optional<block_log_prune_config> prune_config;
+            bool                     not_generate_block_log = false;
 
             explicit block_log_impl(std::optional<block_log_prune_config> prune_conf) :
               prune_config(prune_conf) {
                if(prune_config) {
-                  EOS_ASSERT(prune_config->prune_blocks, block_log_exception, "block log prune configuration requires at least one block");
-                  EOS_ASSERT(__builtin_popcount(prune_config->prune_threshold) == 1, block_log_exception, "block log prune threshold must be power of 2");
-                  //switch this over to the mask that will be used
-                  prune_config->prune_threshold = ~(prune_config->prune_threshold-1);
+                  if (prune_config->prune_blocks == 0 ) {
+                     // not to generate blocks.log
+                     // disable prune log handling by resetting prune_config
+                     prune_config.reset();
+                     not_generate_block_log = true;
+                  } else {
+                     EOS_ASSERT(__builtin_popcount(prune_config->prune_threshold) == 1, block_log_exception, "block log prune threshold must be power of 2");
+                     //switch this over to the mask that will be used
+                     prune_config->prune_threshold = ~(prune_config->prune_threshold-1);
+                  }
                }
             }
 
@@ -99,6 +106,8 @@ namespace eosio { namespace chain {
             template<typename T>
             void reset( const T& t, const signed_block_ptr& genesis_block, uint32_t first_block_num );
 
+            void remove();
+
             void write( const genesis_state& gs );
 
             void write( const chain_id_type& chain_id );
@@ -107,6 +116,8 @@ namespace eosio { namespace chain {
 
             void append(const signed_block_ptr& b, const block_id_type& id, const std::vector<char>& packed_block);
 
+            void update_head(const signed_block_ptr& b, const std::optional<block_id_type>& id={});
+
             void prune();
 
             void vacuum();
@@ -291,12 +302,7 @@ namespace eosio { namespace chain {
          }
          my->index_first_block_num = my->first_block_num;
 
-         my->head = read_head();
-         if( my->head ) {
-            my->head_id = my->head->calculate_id();
-         } else {
-            my->head_id = {};
-         }
+         my->update_head(read_head());
 
          my->block_file.seek_end(0);
          if(is_currently_pruned && my->head) {
@@ -367,6 +373,11 @@ namespace eosio { namespace chain {
       try {
          EOS_ASSERT( genesis_written_to_block_log, block_log_append_fail, "Cannot append to block log until the genesis is first written" );
 
+         if (not_generate_block_log) {
+            update_head(b, id);
+            return;
+         }
+
          check_open_files();
 
          block_file.seek_end(0);
@@ -385,8 +396,8 @@ namespace eosio { namespace chain {
          block_file.write((char*)&pos, sizeof(pos));
          const uint64_t end = block_file.tellp();
          index_file.write((char*)&pos, sizeof(pos));
-         head = b;
-         head_id = id;
+
+         update_head(b, id);
 
          if(prune_config) {
             if((pos&prune_config->prune_threshold) != (end&prune_config->prune_threshold))
@@ -401,6 +412,19 @@ namespace eosio { namespace chain {
       FC_LOG_AND_RETHROW()
    }
 
+   void detail::block_log_impl::update_head(const signed_block_ptr& b, const std::optional<block_id_type>& id) {
+      head = b;
+      if (id) {
+         head_id = *id;
+      } else {
+         if (head) {
+            head_id = b->calculate_id();
+         } else {
+            head_id = {};
+         }
+      }
+   }
+
    void detail::block_log_impl::prune() {
       if(!head)
          return;
@@ -425,6 +449,9 @@ namespace eosio { namespace chain {
    }
 
    void block_log::flush() {
+      if (my->not_generate_block_log) {
+         return;
+      }
       my->flush();
    }
 
@@ -592,15 +619,30 @@ namespace eosio { namespace chain {
    }
 
    void block_log::reset( const genesis_state& gs, const signed_block_ptr& first_block ) {
+      // At startup, OK to be called in no blocks.log mode from controller.cpp
       my->reset(gs, first_block, 1);
    }
 
    void block_log::reset( const chain_id_type& chain_id, uint32_t first_block_num ) {
+      // At startup, OK to be called in no blocks.log mode from controller.cpp
       EOS_ASSERT( first_block_num > 1, block_log_exception,
                   "Block log version ${ver} needs to be created with a genesis state if starting from block number 1." );
       my->reset(chain_id, signed_block_ptr(), first_block_num);
    }
 
+   void detail::block_log_impl::remove() {
+      close();
+
+      fc::remove( block_file.get_file_path() );
+      fc::remove( index_file.get_file_path() );
+
+      ilog("block log ${l}, block index ${i} removed", ("l", block_file.get_file_path()) ("i", index_file.get_file_path()));
+   }
+
+   void block_log::remove() {
+      my->remove();
+   }
+
    void detail::block_log_impl::write( const genesis_state& gs ) {
       auto data = fc::raw::pack(gs);
       block_file.write(data.data(), data.size());
@@ -611,6 +653,10 @@ namespace eosio { namespace chain {
    }
 
    signed_block_ptr block_log::read_block(uint64_t pos)const {
+      if (my->not_generate_block_log) {
+         return nullptr;
+      }
+
       my->check_open_files();
 
       my->block_file.seek(pos);
@@ -621,6 +667,10 @@ namespace eosio { namespace chain {
    }
 
    void block_log::read_block_header(block_header& bh, uint64_t pos)const {
+      if (my->not_generate_block_log) {
+         return;
+      }
+
       my->check_open_files();
 
       my->block_file.seek(pos);
@@ -631,6 +681,12 @@ namespace eosio { namespace chain {
    signed_block_ptr block_log::read_block_by_num(uint32_t block_num)const {
       try {
          signed_block_ptr b;
+
+         if (my->not_generate_block_log) {
+            // No blocks exist. Avoid cascading failures if going further.
+            return b;
+         }
+
          uint64_t pos = get_block_pos(block_num);
          if (pos != npos) {
             b = read_block(pos);
@@ -643,6 +699,9 @@ namespace eosio { namespace chain {
 
    block_id_type block_log::read_block_id_by_num(uint32_t block_num)const {
       try {
+         if (my->not_generate_block_log) {
+            return {};
+         }
          uint64_t pos = get_block_pos(block_num);
          if (pos != npos) {
             block_header bh;
@@ -666,10 +725,17 @@ namespace eosio { namespace chain {
    }
 
    uint64_t block_log::get_block_pos(uint32_t block_num) const {
+      if (my->not_generate_block_log) {
+         return block_log::npos;
+      }
       return my->get_block_pos(block_num);
    }
 
    signed_block_ptr block_log::read_head()const {
+      if (my->not_generate_block_log) {
+         return {};
+      }
+
       my->check_open_files();
 
       uint64_t pos;
@@ -709,6 +775,11 @@ namespace eosio { namespace chain {
    }
 
    void block_log::construct_index() {
+      if (my->not_generate_block_log) {
+         ilog("Not need to construct index in no blocks.log mode (block-log-retain-blocks=0)");
+         return;
+      }
+
       ilog("Reconstructing Block Log Index...");
       my->close();
 

libraries/chain/controller.cpp

@@ -582,6 +582,9 @@ struct controller_impl {
          shutdown();
       }
 
+      if (conf.prune_config && conf.prune_config->prune_blocks == 0) {
+         blog.remove();
+      }
    }
 
    void startup(std::function<void()> shutdown, std::function<bool()> check_shutdown, const genesis_state& genesis) {
@@ -615,6 +618,10 @@ struct controller_impl {
          blog.reset( genesis, head->block );
       }
       init(check_shutdown);
+
+      if (conf.prune_config && conf.prune_config->prune_blocks == 0) {
+         blog.remove();
+      }
    }
 
    void startup(std::function<void()> shutdown, std::function<bool()> check_shutdown) {
@@ -645,6 +652,10 @@ struct controller_impl {
       head = fork_db.head();
 
       init(check_shutdown);
+
+      if (conf.prune_config && conf.prune_config->prune_blocks == 0) {
+         blog.remove();
+      }
    }
 
 

libraries/chain/include/eosio/chain/block_log.hpp

@@ -54,6 +54,7 @@ namespace eosio { namespace chain {
          void flush();
          void reset( const genesis_state& gs, const signed_block_ptr& genesis_block );
          void reset( const chain_id_type& chain_id, uint32_t first_block_num );
+         void remove(); // remove blocks.log and blocks.index
 
          signed_block_ptr read_block(uint64_t file_pos)const;
          void             read_block_header(block_header& bh, uint64_t file_pos)const;

plugins/chain_plugin/chain_plugin.cpp

@@ -335,8 +335,8 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip
          ("integrity-hash-on-start", bpo::bool_switch(), "Log the state integrity hash on startup")
          ("integrity-hash-on-stop", bpo::bool_switch(), "Log the state integrity hash on shutdown");
 
-   if(cfile::supports_hole_punching())
-      cfg.add_options()("block-log-retain-blocks", bpo::value<uint32_t>(), "if set, periodically prune the block log to store only configured number of most recent blocks");
+    cfg.add_options()("block-log-retain-blocks", bpo::value<uint32_t>(), "If set to greater than 0, periodically prune the block log to store only configured number of most recent blocks.\n"
+        "If set to 0, no blocks are be written to the block log; block log file is removed after startup.");
 
 
 // TODO: rate limiting
@@ -868,7 +868,17 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
       if(options.count( "block-log-retain-blocks" )) {
          my->chain_config->prune_config.emplace();
          my->chain_config->prune_config->prune_blocks = options.at( "block-log-retain-blocks" ).as<uint32_t>();
-         EOS_ASSERT(my->chain_config->prune_config->prune_blocks, plugin_config_exception, "block-log-retain-blocks cannot be 0");
+
+         if ( my->chain_config->prune_config->prune_blocks == 0 ) {
+            // clear out empty blocks.log. otherwise block_log::extract_genesis_state
+            // will return version 0 which asserts.
+            if( fc::exists( my->blocks_dir / "blocks.log" ) && fc::file_size( my->blocks_dir / "blocks.log" ) == 0 ) {
+               fc::remove( my->blocks_dir / "blocks.log" );
+               fc::remove( my->blocks_dir / "blocks.index" );
+            }
+         } else {
+            EOS_ASSERT(cfile::supports_hole_punching(), plugin_config_exception, "block-log-retain-blocks cannot be greater than 0 because the file system does not support hole punching");
+         }
       }
 
       if( options.at( "delete-all-blocks" ).as<bool>()) {

tests/CMakeLists.txt

@@ -22,6 +22,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/TestHelper.py ${CMAKE_CURRENT_BINARY_
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/p2p_tests/dawn_515/test.sh ${CMAKE_CURRENT_BINARY_DIR}/p2p_tests/dawn_515/test.sh COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/block_log_util_test.py ${CMAKE_CURRENT_BINARY_DIR}/block_log_util_test.py COPYONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/block_log_retain_blocks_test.py ${CMAKE_CURRENT_BINARY_DIR}/block_log_retain_blocks_test.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/distributed-transactions-test.py ${CMAKE_CURRENT_BINARY_DIR}/distributed-transactions-test.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/distributed-transactions-remote-test.py ${CMAKE_CURRENT_BINARY_DIR}/distributed-transactions-remote-test.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/sample-cluster-map.json ${CMAKE_CURRENT_BINARY_DIR}/sample-cluster-map.json COPYONLY)
@@ -77,6 +78,8 @@ add_test(NAME nodeos_run_test COMMAND tests/nodeos_run_test.py -v --clean-run --
 set_property(TEST nodeos_run_test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME block_log_util_test COMMAND tests/block_log_util_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST block_log_util_test PROPERTY LABELS nonparallelizable_tests)
+add_test(NAME block_log_retain_blocks_test COMMAND tests/block_log_retain_blocks_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST block_log_retain_blocks_test PROPERTY LABELS nonparallelizable_tests)
 
 option(ABIEOS_ONLY_LIBRARY "define and build the ABIEOS library" ON)
 set(ABIEOS_INSTALL_COMPONENT "dev")

tests/block_log.cpp

@@ -531,4 +531,77 @@ BOOST_DATA_TEST_CASE(empty_prune_to_nonprune_transitions, bdata::xrange(2) * bda
       t.check_not_present(starting_block);
 }  FC_LOG_AND_RETHROW() }
 
-BOOST_AUTO_TEST_SUITE_END()
+// Test when prune_blocks is set to 0, no block log is generated
+BOOST_DATA_TEST_CASE(no_block_log_basic_genesis, bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2),
+                                               enable_read, reopen_on_mark, remove_index_on_reopen, vacuum_on_exit_if_small)  { try {
+   // set enable_read to false: when it is true, startup calls
+   // log->read_block_by_num which always returns null when block log does not exist.
+   // set reopen_on_mark to false: when it is ture, check_n_bounce resets block
+   // object but does not reinitialze.
+   block_log_fixture t(false, false, remove_index_on_reopen, vacuum_on_exit_if_small, 0);
+
+   t.startup(1);
+
+   t.add(2, payload_size(), 'A');
+   t.check_not_present(2);
+
+   t.add(3, payload_size(), 'B');
+   t.add(4, payload_size(), 'C');
+   t.check_not_present(3);
+   t.check_not_present(4);
+
+   t.add(5, payload_size(), 'D');
+   t.check_not_present(5);
+}  FC_LOG_AND_RETHROW() }
+
+// Test when prune_blocks is set to 0, no block log is generated
+BOOST_DATA_TEST_CASE(no_block_log_basic_nongenesis, bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2),
+                                               enable_read, reopen_on_mark, remove_index_on_reopen, vacuum_on_exit_if_small)  { try {
+   block_log_fixture t(enable_read, reopen_on_mark, remove_index_on_reopen, vacuum_on_exit_if_small, 0);
+
+   t.startup(10);
+
+   t.add(10, payload_size(), 'A');
+   t.check_not_present(10);
+
+   t.add(11, payload_size(), 'B');
+   t.add(12, payload_size(), 'C');
+   t.check_not_present(11);
+   t.check_not_present(12);
+
+   t.add(13, payload_size(), 'D');
+   t.check_not_present(13);
+}  FC_LOG_AND_RETHROW() }
+
+void no_block_log_public_functions_test( block_log_fixture& t) {
+   BOOST_REQUIRE_NO_THROW(t.log->flush());
+   BOOST_REQUIRE(t.log->read_block(1) == nullptr);
+   BOOST_REQUIRE_NO_THROW(
+      eosio::chain::block_header bh;
+      t.log->read_block_header(bh, 1);
+   );
+   BOOST_REQUIRE(t.log->read_block_by_num(1) == nullptr);
+   BOOST_REQUIRE(t.log->read_block_id_by_num(1) == eosio::chain::block_id_type{});
+   BOOST_REQUIRE(t.log->get_block_pos(1) == eosio::chain::block_log::npos);
+   BOOST_REQUIRE(t.log->read_head() == nullptr);
+}
+
+// Test when prune_blocks is set to 0, block_log's public methods work
+BOOST_DATA_TEST_CASE(no_block_log_public_functions_genesis, bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2),
+                                               enable_read, reopen_on_mark, remove_index_on_reopen, vacuum_on_exit_if_small)  { try {
+   block_log_fixture t(false, false, remove_index_on_reopen, vacuum_on_exit_if_small, 0);
+
+   t.startup(1);
+   no_block_log_public_functions_test(t);
+}  FC_LOG_AND_RETHROW() }
+
+// Test when prune_blocks is set to 0, block_log's public methods work
+BOOST_DATA_TEST_CASE(no_block_log_public_functions_nogenesis, bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2),
+                                               enable_read, reopen_on_mark, remove_index_on_reopen, vacuum_on_exit_if_small)  { try {
+   block_log_fixture t(enable_read, reopen_on_mark, remove_index_on_reopen, vacuum_on_exit_if_small, 0);
+
+   t.startup(10);
+   no_block_log_public_functions_test(t);
+}  FC_LOG_AND_RETHROW() }
+
+BOOST_AUTO_TEST_SUITE_END()