diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 79678fa764..d3afa77977 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -48,14 +48,21 @@ namespace eosio { namespace chain { uint32_t first_block_num = 0; //the first number available to read uint32_t index_first_block_num = 0; //the first number in index & the log had it not been pruned std::optional prune_config; + bool not_generate_block_log = false; explicit block_log_impl(std::optional prune_conf) : prune_config(prune_conf) { if(prune_config) { - EOS_ASSERT(prune_config->prune_blocks, block_log_exception, "block log prune configuration requires at least one block"); - EOS_ASSERT(__builtin_popcount(prune_config->prune_threshold) == 1, block_log_exception, "block log prune threshold must be power of 2"); - //switch this over to the mask that will be used - prune_config->prune_threshold = ~(prune_config->prune_threshold-1); + if (prune_config->prune_blocks == 0 ) { + // not to generate blocks.log + // disable prune log handling by resetting prune_config + prune_config.reset(); + not_generate_block_log = true; + } else { + EOS_ASSERT(__builtin_popcount(prune_config->prune_threshold) == 1, block_log_exception, "block log prune threshold must be power of 2"); + //switch this over to the mask that will be used + prune_config->prune_threshold = ~(prune_config->prune_threshold-1); + } } } @@ -99,6 +106,8 @@ namespace eosio { namespace chain { template void reset( const T& t, const signed_block_ptr& genesis_block, uint32_t first_block_num ); + void remove(); + void write( const genesis_state& gs ); void write( const chain_id_type& chain_id ); @@ -107,6 +116,8 @@ namespace eosio { namespace chain { void append(const signed_block_ptr& b, const block_id_type& id, const std::vector& packed_block); + void update_head(const signed_block_ptr& b, const std::optional& id={}); + void prune(); void vacuum(); @@ -291,12 +302,7 @@ namespace eosio { namespace chain { } my->index_first_block_num = my->first_block_num; - my->head = read_head(); - if( my->head ) { - my->head_id = my->head->calculate_id(); - } else { - my->head_id = {}; - } + my->update_head(read_head()); my->block_file.seek_end(0); if(is_currently_pruned && my->head) { @@ -367,6 +373,11 @@ namespace eosio { namespace chain { try { EOS_ASSERT( genesis_written_to_block_log, block_log_append_fail, "Cannot append to block log until the genesis is first written" ); + if (not_generate_block_log) { + update_head(b, id); + return; + } + check_open_files(); block_file.seek_end(0); @@ -385,8 +396,8 @@ namespace eosio { namespace chain { block_file.write((char*)&pos, sizeof(pos)); const uint64_t end = block_file.tellp(); index_file.write((char*)&pos, sizeof(pos)); - head = b; - head_id = id; + + update_head(b, id); if(prune_config) { if((pos&prune_config->prune_threshold) != (end&prune_config->prune_threshold)) @@ -401,6 +412,19 @@ namespace eosio { namespace chain { FC_LOG_AND_RETHROW() } + void detail::block_log_impl::update_head(const signed_block_ptr& b, const std::optional& id) { + head = b; + if (id) { + head_id = *id; + } else { + if (head) { + head_id = b->calculate_id(); + } else { + head_id = {}; + } + } + } + void detail::block_log_impl::prune() { if(!head) return; @@ -425,6 +449,9 @@ namespace eosio { namespace chain { } void block_log::flush() { + if (my->not_generate_block_log) { + return; + } my->flush(); } @@ -592,15 +619,30 @@ namespace eosio { namespace chain { } void block_log::reset( const genesis_state& gs, const signed_block_ptr& first_block ) { + // At startup, OK to be called in no blocks.log mode from controller.cpp my->reset(gs, first_block, 1); } void block_log::reset( const chain_id_type& chain_id, uint32_t first_block_num ) { + // At startup, OK to be called in no blocks.log mode from controller.cpp EOS_ASSERT( first_block_num > 1, block_log_exception, "Block log version ${ver} needs to be created with a genesis state if starting from block number 1." ); my->reset(chain_id, signed_block_ptr(), first_block_num); } + void detail::block_log_impl::remove() { + close(); + + fc::remove( block_file.get_file_path() ); + fc::remove( index_file.get_file_path() ); + + ilog("block log ${l}, block index ${i} removed", ("l", block_file.get_file_path()) ("i", index_file.get_file_path())); + } + + void block_log::remove() { + my->remove(); + } + void detail::block_log_impl::write( const genesis_state& gs ) { auto data = fc::raw::pack(gs); block_file.write(data.data(), data.size()); @@ -611,6 +653,10 @@ namespace eosio { namespace chain { } signed_block_ptr block_log::read_block(uint64_t pos)const { + if (my->not_generate_block_log) { + return nullptr; + } + my->check_open_files(); my->block_file.seek(pos); @@ -621,6 +667,10 @@ namespace eosio { namespace chain { } void block_log::read_block_header(block_header& bh, uint64_t pos)const { + if (my->not_generate_block_log) { + return; + } + my->check_open_files(); my->block_file.seek(pos); @@ -631,6 +681,12 @@ namespace eosio { namespace chain { signed_block_ptr block_log::read_block_by_num(uint32_t block_num)const { try { signed_block_ptr b; + + if (my->not_generate_block_log) { + // No blocks exist. Avoid cascading failures if going further. + return b; + } + uint64_t pos = get_block_pos(block_num); if (pos != npos) { b = read_block(pos); @@ -643,6 +699,9 @@ namespace eosio { namespace chain { block_id_type block_log::read_block_id_by_num(uint32_t block_num)const { try { + if (my->not_generate_block_log) { + return {}; + } uint64_t pos = get_block_pos(block_num); if (pos != npos) { block_header bh; @@ -666,10 +725,17 @@ namespace eosio { namespace chain { } uint64_t block_log::get_block_pos(uint32_t block_num) const { + if (my->not_generate_block_log) { + return block_log::npos; + } return my->get_block_pos(block_num); } signed_block_ptr block_log::read_head()const { + if (my->not_generate_block_log) { + return {}; + } + my->check_open_files(); uint64_t pos; @@ -709,6 +775,11 @@ namespace eosio { namespace chain { } void block_log::construct_index() { + if (my->not_generate_block_log) { + ilog("Not need to construct index in no blocks.log mode (block-log-retain-blocks=0)"); + return; + } + ilog("Reconstructing Block Log Index..."); my->close(); diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 2274d0b48c..46bb7183b1 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -582,6 +582,9 @@ struct controller_impl { shutdown(); } + if (conf.prune_config && conf.prune_config->prune_blocks == 0) { + blog.remove(); + } } void startup(std::function shutdown, std::function check_shutdown, const genesis_state& genesis) { @@ -615,6 +618,10 @@ struct controller_impl { blog.reset( genesis, head->block ); } init(check_shutdown); + + if (conf.prune_config && conf.prune_config->prune_blocks == 0) { + blog.remove(); + } } void startup(std::function shutdown, std::function check_shutdown) { @@ -645,6 +652,10 @@ struct controller_impl { head = fork_db.head(); init(check_shutdown); + + if (conf.prune_config && conf.prune_config->prune_blocks == 0) { + blog.remove(); + } } diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp index 089fe5eff5..57f5149921 100644 --- a/libraries/chain/include/eosio/chain/block_log.hpp +++ b/libraries/chain/include/eosio/chain/block_log.hpp @@ -54,6 +54,7 @@ namespace eosio { namespace chain { void flush(); void reset( const genesis_state& gs, const signed_block_ptr& genesis_block ); void reset( const chain_id_type& chain_id, uint32_t first_block_num ); + void remove(); // remove blocks.log and blocks.index signed_block_ptr read_block(uint64_t file_pos)const; void read_block_header(block_header& bh, uint64_t file_pos)const; diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index 9141072cde..966d0ced36 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -335,8 +335,8 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip ("integrity-hash-on-start", bpo::bool_switch(), "Log the state integrity hash on startup") ("integrity-hash-on-stop", bpo::bool_switch(), "Log the state integrity hash on shutdown"); - if(cfile::supports_hole_punching()) - cfg.add_options()("block-log-retain-blocks", bpo::value(), "if set, periodically prune the block log to store only configured number of most recent blocks"); + cfg.add_options()("block-log-retain-blocks", bpo::value(), "If set to greater than 0, periodically prune the block log to store only configured number of most recent blocks.\n" + "If set to 0, no blocks are be written to the block log; block log file is removed after startup."); // TODO: rate limiting @@ -868,7 +868,17 @@ void chain_plugin::plugin_initialize(const variables_map& options) { if(options.count( "block-log-retain-blocks" )) { my->chain_config->prune_config.emplace(); my->chain_config->prune_config->prune_blocks = options.at( "block-log-retain-blocks" ).as(); - EOS_ASSERT(my->chain_config->prune_config->prune_blocks, plugin_config_exception, "block-log-retain-blocks cannot be 0"); + + if ( my->chain_config->prune_config->prune_blocks == 0 ) { + // clear out empty blocks.log. otherwise block_log::extract_genesis_state + // will return version 0 which asserts. + if( fc::exists( my->blocks_dir / "blocks.log" ) && fc::file_size( my->blocks_dir / "blocks.log" ) == 0 ) { + fc::remove( my->blocks_dir / "blocks.log" ); + fc::remove( my->blocks_dir / "blocks.index" ); + } + } else { + EOS_ASSERT(cfile::supports_hole_punching(), plugin_config_exception, "block-log-retain-blocks cannot be greater than 0 because the file system does not support hole punching"); + } } if( options.at( "delete-all-blocks" ).as()) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3f088c0d3b..9dc8a542d1 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -22,6 +22,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/TestHelper.py ${CMAKE_CURRENT_BINARY_ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/p2p_tests/dawn_515/test.sh ${CMAKE_CURRENT_BINARY_DIR}/p2p_tests/dawn_515/test.sh COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/block_log_util_test.py ${CMAKE_CURRENT_BINARY_DIR}/block_log_util_test.py COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/block_log_retain_blocks_test.py ${CMAKE_CURRENT_BINARY_DIR}/block_log_retain_blocks_test.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/distributed-transactions-test.py ${CMAKE_CURRENT_BINARY_DIR}/distributed-transactions-test.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/distributed-transactions-remote-test.py ${CMAKE_CURRENT_BINARY_DIR}/distributed-transactions-remote-test.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/sample-cluster-map.json ${CMAKE_CURRENT_BINARY_DIR}/sample-cluster-map.json COPYONLY) @@ -77,6 +78,8 @@ add_test(NAME nodeos_run_test COMMAND tests/nodeos_run_test.py -v --clean-run -- set_property(TEST nodeos_run_test PROPERTY LABELS nonparallelizable_tests) add_test(NAME block_log_util_test COMMAND tests/block_log_util_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST block_log_util_test PROPERTY LABELS nonparallelizable_tests) +add_test(NAME block_log_retain_blocks_test COMMAND tests/block_log_retain_blocks_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +set_property(TEST block_log_retain_blocks_test PROPERTY LABELS nonparallelizable_tests) option(ABIEOS_ONLY_LIBRARY "define and build the ABIEOS library" ON) set(ABIEOS_INSTALL_COMPONENT "dev") diff --git a/tests/block_log.cpp b/tests/block_log.cpp index 599c562e87..95907b13f5 100644 --- a/tests/block_log.cpp +++ b/tests/block_log.cpp @@ -531,4 +531,77 @@ BOOST_DATA_TEST_CASE(empty_prune_to_nonprune_transitions, bdata::xrange(2) * bda t.check_not_present(starting_block); } FC_LOG_AND_RETHROW() } -BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file +// Test when prune_blocks is set to 0, no block log is generated +BOOST_DATA_TEST_CASE(no_block_log_basic_genesis, bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2), + enable_read, reopen_on_mark, remove_index_on_reopen, vacuum_on_exit_if_small) { try { + // set enable_read to false: when it is true, startup calls + // log->read_block_by_num which always returns null when block log does not exist. + // set reopen_on_mark to false: when it is ture, check_n_bounce resets block + // object but does not reinitialze. + block_log_fixture t(false, false, remove_index_on_reopen, vacuum_on_exit_if_small, 0); + + t.startup(1); + + t.add(2, payload_size(), 'A'); + t.check_not_present(2); + + t.add(3, payload_size(), 'B'); + t.add(4, payload_size(), 'C'); + t.check_not_present(3); + t.check_not_present(4); + + t.add(5, payload_size(), 'D'); + t.check_not_present(5); +} FC_LOG_AND_RETHROW() } + +// Test when prune_blocks is set to 0, no block log is generated +BOOST_DATA_TEST_CASE(no_block_log_basic_nongenesis, bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2), + enable_read, reopen_on_mark, remove_index_on_reopen, vacuum_on_exit_if_small) { try { + block_log_fixture t(enable_read, reopen_on_mark, remove_index_on_reopen, vacuum_on_exit_if_small, 0); + + t.startup(10); + + t.add(10, payload_size(), 'A'); + t.check_not_present(10); + + t.add(11, payload_size(), 'B'); + t.add(12, payload_size(), 'C'); + t.check_not_present(11); + t.check_not_present(12); + + t.add(13, payload_size(), 'D'); + t.check_not_present(13); +} FC_LOG_AND_RETHROW() } + +void no_block_log_public_functions_test( block_log_fixture& t) { + BOOST_REQUIRE_NO_THROW(t.log->flush()); + BOOST_REQUIRE(t.log->read_block(1) == nullptr); + BOOST_REQUIRE_NO_THROW( + eosio::chain::block_header bh; + t.log->read_block_header(bh, 1); + ); + BOOST_REQUIRE(t.log->read_block_by_num(1) == nullptr); + BOOST_REQUIRE(t.log->read_block_id_by_num(1) == eosio::chain::block_id_type{}); + BOOST_REQUIRE(t.log->get_block_pos(1) == eosio::chain::block_log::npos); + BOOST_REQUIRE(t.log->read_head() == nullptr); +} + +// Test when prune_blocks is set to 0, block_log's public methods work +BOOST_DATA_TEST_CASE(no_block_log_public_functions_genesis, bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2), + enable_read, reopen_on_mark, remove_index_on_reopen, vacuum_on_exit_if_small) { try { + block_log_fixture t(false, false, remove_index_on_reopen, vacuum_on_exit_if_small, 0); + + t.startup(1); + no_block_log_public_functions_test(t); +} FC_LOG_AND_RETHROW() } + +// Test when prune_blocks is set to 0, block_log's public methods work +BOOST_DATA_TEST_CASE(no_block_log_public_functions_nogenesis, bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2), + enable_read, reopen_on_mark, remove_index_on_reopen, vacuum_on_exit_if_small) { try { + block_log_fixture t(enable_read, reopen_on_mark, remove_index_on_reopen, vacuum_on_exit_if_small, 0); + + t.startup(10); + no_block_log_public_functions_test(t); +} FC_LOG_AND_RETHROW() } + +BOOST_AUTO_TEST_SUITE_END() diff --git a/tests/block_log_retain_blocks_test.py b/tests/block_log_retain_blocks_test.py new file mode 100755 index 0000000000..0dff258fcb --- /dev/null +++ b/tests/block_log_retain_blocks_test.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 + +from testUtils import Utils +from Cluster import Cluster +from WalletMgr import WalletMgr +from TestHelper import TestHelper + +import random +import os +import signal + +############################################################### +# block_log_retain_blocks_test +# +# A basic test for --block-log-retain-blocks option. It validates +# * no blocks.log is generated when the option is set to 0 +# * blocks.log is generated when the option is set to greater than 0 +# * blocks.log is generated when the option is not present. +# +############################################################### + +Print=Utils.Print +errorExit=Utils.errorExit + +args=TestHelper.parse_args({"--keep-logs" ,"--dump-error-details","-v","--leave-running","--clean-run" }) +debug=args.v +killEosInstances= not args.leave_running +dumpErrorDetails=args.dump_error_details +keepLogs=args.keep_logs +killAll=args.clean_run + +seed=1 +Utils.Debug=debug +testSuccessful=False + +random.seed(seed) # Use a fixed seed for repeatability. +cluster=Cluster(walletd=True) +walletMgr=WalletMgr(True) + +# the first node for --block-log-retain-blocks 0, +# the second for --block-log-retain-blocks 10, +# the third for -block-log-retain-blocks not configured +pnodes=1 +total_nodes=pnodes + 2 + +try: + TestHelper.printSystemInfo("BEGIN") + + cluster.setWalletMgr(walletMgr) + + cluster.killall(allInstances=killAll) + cluster.cleanup() + walletMgr.killall(allInstances=killAll) + walletMgr.cleanup() + + specificExtraNodeosArgs={} + specificExtraNodeosArgs[0]=f' --block-log-retain-blocks 0 ' + specificExtraNodeosArgs[1]=f' --block-log-retain-blocks 10 ' + extraNodeosArgs=" --plugin eosio::trace_api_plugin --trace-no-abis " + + Print("Stand up cluster") + if cluster.launch(pnodes=pnodes, totalNodes=total_nodes, extraNodeosArgs=extraNodeosArgs, specificExtraNodeosArgs=specificExtraNodeosArgs) is False: + errorExit("Failed to stand up eos cluster.") + + Print ("Wait for Cluster stabilization") + # wait for cluster to start producing blocks + if not cluster.waitOnClusterBlockNumSync(3): + errorExit("Cluster never stabilized") + Print ("Cluster stabilized") + + # node 0 started with --block-log-retain-blocks 0. no blocks.log should + # be generated + blocksLog0=os.path.join(Utils.getNodeDataDir(0), "blocks", "blocks.log") + if os.path.exists(blocksLog0): + errorExit(f'{blocksLog0} not expected to exist. Test failed') + Print ("Verified no blocks.log existed for --block-log-retain-blocks 0"); + + # node 1 started with --block-log-retain-blocks 10. blocks.log should + # be generated + blocksLog1=os.path.join(Utils.getNodeDataDir(1), "blocks", "blocks.log") + if not os.path.exists(blocksLog1): + errorExit(f'{blocksLog1} expected to exist. Test failed') + Print ("Verified blocks.log existed for --block-log-retain-blocks 10"); + + # node 2 started without --block-log-retain-blocks. blocks.log should + # be generated + blocksLog2=os.path.join(Utils.getNodeDataDir(2), "blocks", "blocks.log") + if not os.path.exists(blocksLog2): + errorExit(f'{blocksLog2} expected to exist. Test failed') + Print ("Verified blocks.log existed for no --block-log-retain-blocks configured"); + + testSuccessful=True +finally: + TestHelper.shutdown(cluster, walletMgr, testSuccessful=testSuccessful, killEosInstances=killEosInstances, killWallet=killEosInstances, keepLogs=keepLogs, cleanRun=killAll, dumpErrorDetails=dumpErrorDetails) + +exitCode = 0 if testSuccessful else 1 +exit(exitCode)