From bb3968dd511eb0b8a200340bd76cb68dbae5c1c4 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Mon, 11 Jul 2022 10:33:19 -0400 Subject: [PATCH 1/4] Add capability of not genereating block log by setting option return-blocks to 0 --- libraries/chain/block_log.cpp | 49 ++++++++- libraries/chain/controller.cpp | 11 ++ .../chain/include/eosio/chain/block_log.hpp | 1 + plugins/chain_plugin/chain_plugin.cpp | 16 ++- tests/CMakeLists.txt | 3 + tests/block_log.cpp | 44 +++++++- tests/block_log_retain_blocks_test.py | 101 ++++++++++++++++++ 7 files changed, 216 insertions(+), 9 deletions(-) create mode 100755 tests/block_log_retain_blocks_test.py diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index c3dc98d15e..043b57556f 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -49,14 +49,22 @@ namespace eosio { namespace chain { uint32_t first_block_num = 0; //the first number available to read uint32_t index_first_block_num = 0; //the first number in index & the log had it not been pruned std::optional prune_config; + bool not_generate_block_log = false; block_log_impl(std::optional prune_conf) : prune_config(prune_conf) { if(prune_config) { - EOS_ASSERT(prune_config->prune_blocks, block_log_exception, "block log prune configuration requires at least one block"); - EOS_ASSERT(__builtin_popcount(prune_config->prune_threshold) == 1, block_log_exception, "block log prune threshold must be power of 2"); - //switch this over to the mask that will be used - prune_config->prune_threshold = ~(prune_config->prune_threshold-1); + if (prune_config->prune_blocks == 0 ) { + // not to generate blocks.log + // disable prune log handling by resetting prune_config + prune_config.reset(); + not_generate_block_log = true; + } else { + EOS_ASSERT(prune_config->prune_blocks, block_log_exception, "block log prune configuration requires at least one block"); + EOS_ASSERT(__builtin_popcount(prune_config->prune_threshold) == 1, block_log_exception, "block log prune threshold must be power of 2"); + //switch this over to the mask that will be used + prune_config->prune_threshold = ~(prune_config->prune_threshold-1); + } } } @@ -99,6 +107,8 @@ namespace eosio { namespace chain { template void reset( const T& t, const signed_block_ptr& genesis_block, uint32_t first_block_num ); + void remove(); + void write( const genesis_state& gs ); void write( const chain_id_type& chain_id ); @@ -107,6 +117,8 @@ namespace eosio { namespace chain { void append(const signed_block_ptr& b); + void update_head(const signed_block_ptr& b); + void prune(); void vacuum(); @@ -363,6 +375,11 @@ namespace eosio { namespace chain { try { EOS_ASSERT( genesis_written_to_block_log, block_log_append_fail, "Cannot append to block log until the genesis is first written" ); + if (not_generate_block_log) { + update_head(b); + return; + } + check_open_files(); block_file.seek_end(0); @@ -398,6 +415,11 @@ namespace eosio { namespace chain { FC_LOG_AND_RETHROW() } + void detail::block_log_impl::update_head(const signed_block_ptr& b) { + head = b; + head_id = b->calculate_id(); + } + void detail::block_log_impl::prune() { if(!head) return; @@ -598,6 +620,17 @@ namespace eosio { namespace chain { my->reset(chain_id, signed_block_ptr(), first_block_num); } + void detail::block_log_impl::remove() { + close(); + + fc::remove( block_file.get_file_path() ); + fc::remove( index_file.get_file_path() ); + } + + void block_log::remove() { + my->remove(); + } + void detail::block_log_impl::write( const genesis_state& gs ) { auto data = fc::raw::pack(gs); block_file.write(data.data(), data.size()); @@ -628,6 +661,12 @@ namespace eosio { namespace chain { signed_block_ptr block_log::read_block_by_num(uint32_t block_num)const { try { signed_block_ptr b; + + if (my->not_generate_block_log) { + // No blocks exist. Avoid cascading failures if going further. + return b; + } + uint64_t pos = get_block_pos(block_num); if (pos != npos) { b = read_block(pos); @@ -1429,4 +1468,4 @@ namespace eosio { namespace chain { // used only for unit test to adjust the buffer length void block_log_set_buff_len(uint64_t len){ eosio::chain::detail::reverse_iterator::_buf_len = len; -} \ No newline at end of file +} diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index c787da828a..a8fae270ea 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -570,6 +570,9 @@ struct controller_impl { shutdown(); } + if (conf.prune_config && conf.prune_config->prune_blocks == 0) { + blog.remove(); + } } void startup(std::function shutdown, std::function check_shutdown, const genesis_state& genesis) { @@ -603,6 +606,10 @@ struct controller_impl { blog.reset( genesis, head->block ); } init(check_shutdown); + + if (conf.prune_config && conf.prune_config->prune_blocks == 0) { + blog.remove(); + } } void startup(std::function shutdown, std::function check_shutdown) { @@ -633,6 +640,10 @@ struct controller_impl { head = fork_db.head(); init(check_shutdown); + + if (conf.prune_config && conf.prune_config->prune_blocks == 0) { + blog.remove(); + } } diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp index 47376e7876..9537421eaf 100644 --- a/libraries/chain/include/eosio/chain/block_log.hpp +++ b/libraries/chain/include/eosio/chain/block_log.hpp @@ -52,6 +52,7 @@ namespace eosio { namespace chain { void flush(); void reset( const genesis_state& gs, const signed_block_ptr& genesis_block ); void reset( const chain_id_type& chain_id, uint32_t first_block_num ); + void remove(); // remove blocks.log and blocks.index signed_block_ptr read_block(uint64_t file_pos)const; void read_block_header(block_header& bh, uint64_t file_pos)const; diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index cfdc63fcc3..f91c53f4ea 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -335,8 +335,8 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip ("integrity-hash-on-start", bpo::bool_switch(), "Log the state integrity hash on startup") ("integrity-hash-on-stop", bpo::bool_switch(), "Log the state integrity hash on shutdown"); - if(cfile::supports_hole_punching()) - cfg.add_options()("block-log-retain-blocks", bpo::value(), "if set, periodically prune the block log to store only configured number of most recent blocks"); + cfg.add_options()("block-log-retain-blocks", bpo::value(), "If set to greater than 0, periodically prune the block log to store only configured number of most recent blocks.\n" + "If set to 0, no blocks are be written to the block log; block log file is removed after startup."); // TODO: rate limiting @@ -868,7 +868,17 @@ void chain_plugin::plugin_initialize(const variables_map& options) { if(options.count( "block-log-retain-blocks" )) { my->chain_config->prune_config.emplace(); my->chain_config->prune_config->prune_blocks = options.at( "block-log-retain-blocks" ).as(); - EOS_ASSERT(my->chain_config->prune_config->prune_blocks, plugin_config_exception, "block-log-retain-blocks cannot be 0"); + + if ( my->chain_config->prune_config->prune_blocks == 0 ) { + // clear out empty blocks.log. otherwise block_log::extract_genesis_state + // will return version 0 which asserts. + if( fc::exists( my->blocks_dir / "blocks.log" ) && fc::file_size( my->blocks_dir / "blocks.log" ) == 0 ) { + fc::remove( my->blocks_dir / "blocks.log" ); + fc::remove( my->blocks_dir / "blocks.index" ); + } + } else { + EOS_ASSERT(cfile::supports_hole_punching(), plugin_config_exception, "block-log-retain-blocks cannot be greater than 0 because the file system does not support hole punching"); + } } if( options.at( "delete-all-blocks" ).as()) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0e4aa70af7..816ce54c44 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -22,6 +22,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/TestHelper.py ${CMAKE_CURRENT_BINARY_ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/p2p_tests/dawn_515/test.sh ${CMAKE_CURRENT_BINARY_DIR}/p2p_tests/dawn_515/test.sh COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/block_log_util_test.py ${CMAKE_CURRENT_BINARY_DIR}/block_log_util_test.py COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/block_log_retain_blocks_test.py ${CMAKE_CURRENT_BINARY_DIR}/block_log_retain_blocks_test.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/distributed-transactions-test.py ${CMAKE_CURRENT_BINARY_DIR}/distributed-transactions-test.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/distributed-transactions-remote-test.py ${CMAKE_CURRENT_BINARY_DIR}/distributed-transactions-remote-test.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/sample-cluster-map.json ${CMAKE_CURRENT_BINARY_DIR}/sample-cluster-map.json COPYONLY) @@ -75,6 +76,8 @@ add_test(NAME nodeos_run_test COMMAND tests/nodeos_run_test.py -v --clean-run -- set_property(TEST nodeos_run_test PROPERTY LABELS nonparallelizable_tests) add_test(NAME block_log_util_test COMMAND tests/block_log_util_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST block_log_util_test PROPERTY LABELS nonparallelizable_tests) +add_test(NAME block_log_retain_blocks_test COMMAND tests/block_log_retain_blocks_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +set_property(TEST block_log_retain_blocks_test PROPERTY LABELS nonparallelizable_tests) option(ABIEOS_ONLY_LIBRARY "define and build the ABIEOS library" ON) set(ABIEOS_INSTALL_COMPONENT "dev") diff --git a/tests/block_log.cpp b/tests/block_log.cpp index 2e1d8536d0..b3e29bad2a 100644 --- a/tests/block_log.cpp +++ b/tests/block_log.cpp @@ -531,4 +531,46 @@ BOOST_DATA_TEST_CASE(empty_prune_to_nonprune_transitions, bdata::xrange(2) * bda t.check_not_present(starting_block); } FC_LOG_AND_RETHROW() } -BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file +// Test when prune_blocks is set to 0, no block log is generated +BOOST_DATA_TEST_CASE(not_generate_block_log_genesis, bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2), + enable_read, reopen_on_mark, remove_index_on_reopen, vacuum_on_exit_if_small) { try { + // set enable_read to false: when it is true, startup calls + // log->read_block_by_num which always returns null when block log does not exist. + // set reopen_on_mark to false: when it is ture, check_n_bounce resets block + // object but does not reinitialze. + block_log_fixture t(false, false, remove_index_on_reopen, vacuum_on_exit_if_small, 0); + + t.startup(1); + + t.add(2, payload_size(), 'A'); + t.check_not_present(2); + + t.add(3, payload_size(), 'B'); + t.add(4, payload_size(), 'C'); + t.check_not_present(3); + t.check_not_present(4); + + t.add(5, payload_size(), 'D'); + t.check_not_present(5); +} FC_LOG_AND_RETHROW() } + +// Test when prune_blocks is set to 0, no block log is generated +BOOST_DATA_TEST_CASE(not_generate_block_log_nongenesis, bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2), + enable_read, reopen_on_mark, remove_index_on_reopen, vacuum_on_exit_if_small) { try { + block_log_fixture t(enable_read, reopen_on_mark, remove_index_on_reopen, vacuum_on_exit_if_small, 0); + + t.startup(10); + + t.add(10, payload_size(), 'A'); + t.check_not_present(10); + + t.add(11, payload_size(), 'B'); + t.add(12, payload_size(), 'C'); + t.check_not_present(11); + t.check_not_present(12); + + t.add(13, payload_size(), 'D'); + t.check_not_present(13); +} FC_LOG_AND_RETHROW() } + +BOOST_AUTO_TEST_SUITE_END() diff --git a/tests/block_log_retain_blocks_test.py b/tests/block_log_retain_blocks_test.py new file mode 100755 index 0000000000..d415b6edf7 --- /dev/null +++ b/tests/block_log_retain_blocks_test.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 + +from testUtils import Utils +from TestHelper import TestHelper + +import subprocess +import signal +import time +import os +import shutil + +############################################################### +# block-log-retain-blocks test +# +# A basic test for --block-log-retain-blocks option. It validates no blocks.log +# is generated when the option is set to 0 and blocks.log is generated when the +# option is set to greater than 0. +# +############################################################### + +Print=Utils.Print +errorExit=Utils.errorExit + +args = TestHelper.parse_args({"--keep-logs", "--dump-error-details", "-v", "--leave-running", "--clean-run"}) +debug=args.v +keepLogs=args.keep_logs +dumpErrorDetails=args.dump_error_details +killAll=args.clean_run +killEosInstances= not args.leave_running + +Utils.Debug=debug + +def is_in_file(string, filename): + with open(filename) as myfile: + if string in myfile.read(): + return True + else: + return False + +def start_and_stop_nodeos(retain_blocks=0, produced_block=10): + cmd=f'programs/nodeos/nodeos -e -p eosio --plugin eosio::producer_api_plugin --plugin eosio::producer_plugin --plugin eosio::chain_api_plugin --plugin eosio::chain_plugin --data-dir {data_dir} --block-log-retain-blocks {retain_blocks}' + Print(f'cmd to launch nodeos: {cmd}') + + stdout_filename=data_dir + "stdout.out" + stderr_filename=data_dir + "stderr.out" + stdout_file=open(stdout_filename, "w") + stderr_file=open(stderr_filename, "w") + proc=subprocess.Popen(cmd.split(), stdout=stdout_file, stderr=stderr_file) + + for i in range(1,20): + if is_in_file(f'lib: {produced_block}', stderr_filename): + Print(f'nodeos launched and lib: {produced_block} produced') + break; + elif i == 19: + os.system('killall nodeos') + errorExit("nodeos failed to start after 20 seconds") + time.sleep(1) + + proc.send_signal(signal.SIGINT) + proc.wait() + stdout_file.close() + stderr_file.close() + Print(f'nodeos stopped') + +def expect_no_block_log_file(): + if os.path.exists(blocklog_file): + errorExit(f'{blocklog_file} not expected to exist. Test failed') + +def expect_block_log_file(): + if not os.path.exists(blocklog_file): + errorExit(f'{blocklog_file} expected to exist. Test failed') + +TestHelper.printSystemInfo("BEGIN") + +if killAll: + os.system('killall nodeos') +data_dir="var/lib/node/" # OK to be a global +blocklog_file=data_dir + "blocks/blocks.log" +shutil.rmtree(data_dir, ignore_errors=True) +os.makedirs(data_dir, exist_ok=True) + +Print("Start nodeos with retain_blocks as 0 the first time") +start_and_stop_nodeos(retain_blocks=0, produced_block=10) +expect_no_block_log_file() + +Print("Start nodeos with retain_blocks as 0 the second time") +start_and_stop_nodeos(retain_blocks=0, produced_block=20) +expect_no_block_log_file() + +shutil.rmtree(data_dir, ignore_errors=True) +os.makedirs(data_dir, exist_ok=True) + +Print("Start nodeos with retain_blocks as 10") +start_and_stop_nodeos(retain_blocks=5, produced_block=10) +expect_block_log_file() + +Print("Test succeeded") +if not keepLogs: + shutil.rmtree(data_dir, ignore_errors=True) + +exit(0) From 6548c151b38e68757a5fcaa6bbf19a59eaf1e582 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Mon, 18 Jul 2022 10:32:12 -0400 Subject: [PATCH 2/4] Rewrite block_log_retain_blocks_test.py using standard test Cluster, take appropriate actions in block_log public functions when no block log mode is enabled and add corresponding unit tests, use update_head for all occasions --- libraries/chain/block_log.cpp | 54 +++++++-- tests/block_log.cpp | 35 +++++- tests/block_log_retain_blocks_test.py | 159 +++++++++++++------------- 3 files changed, 155 insertions(+), 93 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 043b57556f..a9ece8a436 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -60,7 +60,6 @@ namespace eosio { namespace chain { prune_config.reset(); not_generate_block_log = true; } else { - EOS_ASSERT(prune_config->prune_blocks, block_log_exception, "block log prune configuration requires at least one block"); EOS_ASSERT(__builtin_popcount(prune_config->prune_threshold) == 1, block_log_exception, "block log prune threshold must be power of 2"); //switch this over to the mask that will be used prune_config->prune_threshold = ~(prune_config->prune_threshold-1); @@ -303,12 +302,7 @@ namespace eosio { namespace chain { } my->index_first_block_num = my->first_block_num; - my->head = read_head(); - if( my->head ) { - my->head_id = my->head->calculate_id(); - } else { - my->head_id = {}; - } + my->update_head(read_head()); my->block_file.seek_end(0); if(is_currently_pruned && my->head) { @@ -399,8 +393,7 @@ namespace eosio { namespace chain { block_file.write((char*)&pos, sizeof(pos)); const uint64_t end = block_file.tellp(); index_file.write((char*)&pos, sizeof(pos)); - head = b; - head_id = b->calculate_id(); + update_head(b); if(prune_config) { if((pos&prune_config->prune_threshold) != (end&prune_config->prune_threshold)) @@ -417,7 +410,11 @@ namespace eosio { namespace chain { void detail::block_log_impl::update_head(const signed_block_ptr& b) { head = b; - head_id = b->calculate_id(); + if (head) { + head_id = b->calculate_id(); + } else { + head_id = {}; + } } void detail::block_log_impl::prune() { @@ -444,6 +441,10 @@ namespace eosio { namespace chain { } void block_log::flush() { + if (my->not_generate_block_log) { + ilog("Not possible to flush in no blocks.log mode (block-log-retain-blocks=0)"); + return; + } my->flush(); } @@ -611,10 +612,12 @@ namespace eosio { namespace chain { } void block_log::reset( const genesis_state& gs, const signed_block_ptr& first_block ) { + // At startup, OK to be called in no blocks.log mode from controller.cpp my->reset(gs, first_block, 1); } void block_log::reset( const chain_id_type& chain_id, uint32_t first_block_num ) { + // At startup, OK to be called in no blocks.log mode from controller.cpp EOS_ASSERT( first_block_num > 1, block_log_exception, "Block log version ${ver} needs to be created with a genesis state if starting from block number 1." ); my->reset(chain_id, signed_block_ptr(), first_block_num); @@ -625,6 +628,8 @@ namespace eosio { namespace chain { fc::remove( block_file.get_file_path() ); fc::remove( index_file.get_file_path() ); + + ilog("block log ${l}, block index ${i} removed", ("l", block_file.get_file_path()) ("i", index_file.get_file_path())); } void block_log::remove() { @@ -641,6 +646,11 @@ namespace eosio { namespace chain { } signed_block_ptr block_log::read_block(uint64_t pos)const { + if (my->not_generate_block_log) { + ilog("Not possible to read_block in no blocks.log mode (block-log-retain-blocks=0)"); + return nullptr; + } + my->check_open_files(); my->block_file.seek(pos); @@ -651,6 +661,11 @@ namespace eosio { namespace chain { } void block_log::read_block_header(block_header& bh, uint64_t pos)const { + if (my->not_generate_block_log) { + ilog("Not possible to read_block_header in no blocks.log mode (block-log-retain-blocks=0)"); + return; + } + my->check_open_files(); my->block_file.seek(pos); @@ -664,6 +679,7 @@ namespace eosio { namespace chain { if (my->not_generate_block_log) { // No blocks exist. Avoid cascading failures if going further. + ilog("Not possible to read_block_by_num in no blocks.log mode (block-log-retain-blocks=0)"); return b; } @@ -679,6 +695,10 @@ namespace eosio { namespace chain { block_id_type block_log::read_block_id_by_num(uint32_t block_num)const { try { + if (my->not_generate_block_log) { + ilog("Not possible to read_block_id_by_num in no blocks.log mode (block-log-retain-blocks=0)"); + return {}; + } uint64_t pos = get_block_pos(block_num); if (pos != npos) { block_header bh; @@ -702,10 +722,19 @@ namespace eosio { namespace chain { } uint64_t block_log::get_block_pos(uint32_t block_num) const { + if (my->not_generate_block_log) { + ilog("Not possible to get_block_pos in no blocks.log mode (block-log-retain-blocks=0)"); + return block_log::npos; + } return my->get_block_pos(block_num); } signed_block_ptr block_log::read_head()const { + if (my->not_generate_block_log) { + ilog("Not possible to read_head in no blocks.log mode (block-log-retain-blocks=0)"); + return {}; + } + my->check_open_files(); uint64_t pos; @@ -745,6 +774,11 @@ namespace eosio { namespace chain { } void block_log::construct_index() { + if (my->not_generate_block_log) { + ilog("Not need to construct index in no blocks.log mode (block-log-retain-blocks=0)"); + return; + } + ilog("Reconstructing Block Log Index..."); my->close(); diff --git a/tests/block_log.cpp b/tests/block_log.cpp index b3e29bad2a..a3826704d9 100644 --- a/tests/block_log.cpp +++ b/tests/block_log.cpp @@ -532,7 +532,7 @@ BOOST_DATA_TEST_CASE(empty_prune_to_nonprune_transitions, bdata::xrange(2) * bda } FC_LOG_AND_RETHROW() } // Test when prune_blocks is set to 0, no block log is generated -BOOST_DATA_TEST_CASE(not_generate_block_log_genesis, bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2), +BOOST_DATA_TEST_CASE(no_block_log_basic_genesis, bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2), enable_read, reopen_on_mark, remove_index_on_reopen, vacuum_on_exit_if_small) { try { // set enable_read to false: when it is true, startup calls // log->read_block_by_num which always returns null when block log does not exist. @@ -555,7 +555,7 @@ BOOST_DATA_TEST_CASE(not_generate_block_log_genesis, bdata::xrange(2) * bdata::x } FC_LOG_AND_RETHROW() } // Test when prune_blocks is set to 0, no block log is generated -BOOST_DATA_TEST_CASE(not_generate_block_log_nongenesis, bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2), +BOOST_DATA_TEST_CASE(no_block_log_basic_nongenesis, bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2), enable_read, reopen_on_mark, remove_index_on_reopen, vacuum_on_exit_if_small) { try { block_log_fixture t(enable_read, reopen_on_mark, remove_index_on_reopen, vacuum_on_exit_if_small, 0); @@ -573,4 +573,35 @@ BOOST_DATA_TEST_CASE(not_generate_block_log_nongenesis, bdata::xrange(2) * bdata t.check_not_present(13); } FC_LOG_AND_RETHROW() } +void no_block_log_public_functions_test( block_log_fixture& t) { + BOOST_REQUIRE_NO_THROW(t.log->flush()); + BOOST_REQUIRE(t.log->read_block(1) == nullptr); + BOOST_REQUIRE_NO_THROW( + eosio::chain::block_header bh; + t.log->read_block_header(bh, 1); + ); + BOOST_REQUIRE(t.log->read_block_by_num(1) == nullptr); + BOOST_REQUIRE(t.log->read_block_id_by_num(1) == eosio::chain::block_id_type{}); + BOOST_REQUIRE(t.log->get_block_pos(1) == eosio::chain::block_log::npos); + BOOST_REQUIRE(t.log->read_head() == nullptr); +} + +// Test when prune_blocks is set to 0, block_log's public methods work +BOOST_DATA_TEST_CASE(no_block_log_public_functions_genesis, bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2), + enable_read, reopen_on_mark, remove_index_on_reopen, vacuum_on_exit_if_small) { try { + block_log_fixture t(false, false, remove_index_on_reopen, vacuum_on_exit_if_small, 0); + + t.startup(1); + no_block_log_public_functions_test(t); +} FC_LOG_AND_RETHROW() } + +// Test when prune_blocks is set to 0, block_log's public methods work +BOOST_DATA_TEST_CASE(no_block_log_public_functions_nogenesis, bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2) * bdata::xrange(2), + enable_read, reopen_on_mark, remove_index_on_reopen, vacuum_on_exit_if_small) { try { + block_log_fixture t(enable_read, reopen_on_mark, remove_index_on_reopen, vacuum_on_exit_if_small, 0); + + t.startup(10); + no_block_log_public_functions_test(t); +} FC_LOG_AND_RETHROW() } + BOOST_AUTO_TEST_SUITE_END() diff --git a/tests/block_log_retain_blocks_test.py b/tests/block_log_retain_blocks_test.py index d415b6edf7..8ded489009 100755 --- a/tests/block_log_retain_blocks_test.py +++ b/tests/block_log_retain_blocks_test.py @@ -1,101 +1,98 @@ #!/usr/bin/env python3 from testUtils import Utils +from Cluster import Cluster +from WalletMgr import WalletMgr from TestHelper import TestHelper -import subprocess -import signal -import time +import random import os -import shutil +import signal ############################################################### -# block-log-retain-blocks test +# terminate-scenarios-test # -# A basic test for --block-log-retain-blocks option. It validates no blocks.log -# is generated when the option is set to 0 and blocks.log is generated when the -# option is set to greater than 0. +# A basic test for --block-log-retain-blocks option. It validates +# * no blocks.log is generated when the option is set to 0 +# * blocks.log is generated when the option is set to greater than 0 +# * blocks.log is generated when the option is not present. # ############################################################### Print=Utils.Print errorExit=Utils.errorExit -args = TestHelper.parse_args({"--keep-logs", "--dump-error-details", "-v", "--leave-running", "--clean-run"}) +args=TestHelper.parse_args({"--keep-logs" ,"--dump-error-details","-v","--leave-running","--clean-run" }) debug=args.v -keepLogs=args.keep_logs +killEosInstances= not args.leave_running dumpErrorDetails=args.dump_error_details +keepLogs=args.keep_logs killAll=args.clean_run -killEosInstances= not args.leave_running +seed=1 Utils.Debug=debug - -def is_in_file(string, filename): - with open(filename) as myfile: - if string in myfile.read(): - return True - else: - return False - -def start_and_stop_nodeos(retain_blocks=0, produced_block=10): - cmd=f'programs/nodeos/nodeos -e -p eosio --plugin eosio::producer_api_plugin --plugin eosio::producer_plugin --plugin eosio::chain_api_plugin --plugin eosio::chain_plugin --data-dir {data_dir} --block-log-retain-blocks {retain_blocks}' - Print(f'cmd to launch nodeos: {cmd}') - - stdout_filename=data_dir + "stdout.out" - stderr_filename=data_dir + "stderr.out" - stdout_file=open(stdout_filename, "w") - stderr_file=open(stderr_filename, "w") - proc=subprocess.Popen(cmd.split(), stdout=stdout_file, stderr=stderr_file) - - for i in range(1,20): - if is_in_file(f'lib: {produced_block}', stderr_filename): - Print(f'nodeos launched and lib: {produced_block} produced') - break; - elif i == 19: - os.system('killall nodeos') - errorExit("nodeos failed to start after 20 seconds") - time.sleep(1) - - proc.send_signal(signal.SIGINT) - proc.wait() - stdout_file.close() - stderr_file.close() - Print(f'nodeos stopped') - -def expect_no_block_log_file(): - if os.path.exists(blocklog_file): - errorExit(f'{blocklog_file} not expected to exist. Test failed') - -def expect_block_log_file(): - if not os.path.exists(blocklog_file): - errorExit(f'{blocklog_file} expected to exist. Test failed') - -TestHelper.printSystemInfo("BEGIN") - -if killAll: - os.system('killall nodeos') -data_dir="var/lib/node/" # OK to be a global -blocklog_file=data_dir + "blocks/blocks.log" -shutil.rmtree(data_dir, ignore_errors=True) -os.makedirs(data_dir, exist_ok=True) - -Print("Start nodeos with retain_blocks as 0 the first time") -start_and_stop_nodeos(retain_blocks=0, produced_block=10) -expect_no_block_log_file() - -Print("Start nodeos with retain_blocks as 0 the second time") -start_and_stop_nodeos(retain_blocks=0, produced_block=20) -expect_no_block_log_file() - -shutil.rmtree(data_dir, ignore_errors=True) -os.makedirs(data_dir, exist_ok=True) - -Print("Start nodeos with retain_blocks as 10") -start_and_stop_nodeos(retain_blocks=5, produced_block=10) -expect_block_log_file() - -Print("Test succeeded") -if not keepLogs: - shutil.rmtree(data_dir, ignore_errors=True) - -exit(0) +testSuccessful=False + +random.seed(seed) # Use a fixed seed for repeatability. +cluster=Cluster(walletd=True) +walletMgr=WalletMgr(True) + +# the first node for --block-log-retain-blocks 0, +# the second for --block-log-retain-blocks 10, +# the third for -block-log-retain-blocks not configured +pnodes=1 +total_nodes=pnodes + 2 + +try: + TestHelper.printSystemInfo("BEGIN") + cluster.setWalletMgr(walletMgr) + + cluster.setWalletMgr(walletMgr) + + cluster.killall(allInstances=killAll) + cluster.cleanup() + walletMgr.killall(allInstances=killAll) + walletMgr.cleanup() + + specificExtraNodeosArgs={} + specificExtraNodeosArgs[0]=f' --block-log-retain-blocks 0 ' + specificExtraNodeosArgs[1]=f' --block-log-retain-blocks 10 ' + extraNodeosArgs=" --plugin eosio::trace_api_plugin --trace-no-abis " + + Print("Stand up cluster") + if cluster.launch(pnodes=pnodes, totalNodes=total_nodes, extraNodeosArgs=extraNodeosArgs, specificExtraNodeosArgs=specificExtraNodeosArgs) is False: + errorExit("Failed to stand up eos cluster.") + + Print ("Wait for Cluster stabilization") + # wait for cluster to start producing blocks + if not cluster.waitOnClusterBlockNumSync(3): + errorExit("Cluster never stabilized") + Print ("Cluster stabilized") + + # node 0 started with --block-log-retain-blocks 0. no blocks.log should + # be generated + blocksLog0=os.path.join(Utils.getNodeDataDir(0), "blocks", "blocks.log") + if os.path.exists(blocksLog0): + errorExit(f'{blocksLog0} not expected to exist. Test failed') + Print ("Verified no blocks.log existed for --block-log-retain-blocks 0"); + + # node 1 started with --block-log-retain-blocks 10. blocks.log should + # be generated + blocksLog1=os.path.join(Utils.getNodeDataDir(1), "blocks", "blocks.log") + if not os.path.exists(blocksLog1): + errorExit(f'{blocksLog1} expected to exist. Test failed') + Print ("Verified blocks.log existed for --block-log-retain-blocks 10"); + + # node 2 started without --block-log-retain-blocks. blocks.log should + # be generated + blocksLog2=os.path.join(Utils.getNodeDataDir(2), "blocks", "blocks.log") + if not os.path.exists(blocksLog2): + errorExit(f'{blocksLog2} expected to exist. Test failed') + Print ("Verified blocks.log existed for no --block-log-retain-blocks configured"); + + testSuccessful=True +finally: + TestHelper.shutdown(cluster, walletMgr, testSuccessful=testSuccessful, killEosInstances=killEosInstances, killWallet=killEosInstances, keepLogs=keepLogs, cleanRun=killAll, dumpErrorDetails=dumpErrorDetails) + +exitCode = 0 if testSuccessful else 1 +exit(exitCode) From 09938266e78ba87c3a5b852907f5ae4a21c573a2 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Tue, 19 Jul 2022 11:19:35 -0400 Subject: [PATCH 3/4] add optional parameter id to update_head and remove unnecessary logging --- libraries/chain/block_log.cpp | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index cc4a7e8cef..d3afa77977 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -116,7 +116,7 @@ namespace eosio { namespace chain { void append(const signed_block_ptr& b, const block_id_type& id, const std::vector& packed_block); - void update_head(const signed_block_ptr& b); + void update_head(const signed_block_ptr& b, const std::optional& id={}); void prune(); @@ -374,7 +374,7 @@ namespace eosio { namespace chain { EOS_ASSERT( genesis_written_to_block_log, block_log_append_fail, "Cannot append to block log until the genesis is first written" ); if (not_generate_block_log) { - update_head(b); + update_head(b, id); return; } @@ -397,8 +397,7 @@ namespace eosio { namespace chain { const uint64_t end = block_file.tellp(); index_file.write((char*)&pos, sizeof(pos)); - head = b; - head_id = id; + update_head(b, id); if(prune_config) { if((pos&prune_config->prune_threshold) != (end&prune_config->prune_threshold)) @@ -413,12 +412,16 @@ namespace eosio { namespace chain { FC_LOG_AND_RETHROW() } - void detail::block_log_impl::update_head(const signed_block_ptr& b) { + void detail::block_log_impl::update_head(const signed_block_ptr& b, const std::optional& id) { head = b; - if (head) { - head_id = b->calculate_id(); + if (id) { + head_id = *id; } else { - head_id = {}; + if (head) { + head_id = b->calculate_id(); + } else { + head_id = {}; + } } } @@ -447,7 +450,6 @@ namespace eosio { namespace chain { void block_log::flush() { if (my->not_generate_block_log) { - ilog("Not possible to flush in no blocks.log mode (block-log-retain-blocks=0)"); return; } my->flush(); @@ -652,7 +654,6 @@ namespace eosio { namespace chain { signed_block_ptr block_log::read_block(uint64_t pos)const { if (my->not_generate_block_log) { - ilog("Not possible to read_block in no blocks.log mode (block-log-retain-blocks=0)"); return nullptr; } @@ -667,7 +668,6 @@ namespace eosio { namespace chain { void block_log::read_block_header(block_header& bh, uint64_t pos)const { if (my->not_generate_block_log) { - ilog("Not possible to read_block_header in no blocks.log mode (block-log-retain-blocks=0)"); return; } @@ -684,7 +684,6 @@ namespace eosio { namespace chain { if (my->not_generate_block_log) { // No blocks exist. Avoid cascading failures if going further. - ilog("Not possible to read_block_by_num in no blocks.log mode (block-log-retain-blocks=0)"); return b; } @@ -701,7 +700,6 @@ namespace eosio { namespace chain { block_id_type block_log::read_block_id_by_num(uint32_t block_num)const { try { if (my->not_generate_block_log) { - ilog("Not possible to read_block_id_by_num in no blocks.log mode (block-log-retain-blocks=0)"); return {}; } uint64_t pos = get_block_pos(block_num); @@ -728,7 +726,6 @@ namespace eosio { namespace chain { uint64_t block_log::get_block_pos(uint32_t block_num) const { if (my->not_generate_block_log) { - ilog("Not possible to get_block_pos in no blocks.log mode (block-log-retain-blocks=0)"); return block_log::npos; } return my->get_block_pos(block_num); @@ -736,7 +733,6 @@ namespace eosio { namespace chain { signed_block_ptr block_log::read_head()const { if (my->not_generate_block_log) { - ilog("Not possible to read_head in no blocks.log mode (block-log-retain-blocks=0)"); return {}; } From b64aa18acd682a8ddf94bd7be349922432a0a038 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Fri, 22 Jul 2022 11:45:20 -0400 Subject: [PATCH 4/4] Remove duplicate setWalletMgr and correct test title --- tests/block_log_retain_blocks_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/block_log_retain_blocks_test.py b/tests/block_log_retain_blocks_test.py index 8ded489009..0dff258fcb 100755 --- a/tests/block_log_retain_blocks_test.py +++ b/tests/block_log_retain_blocks_test.py @@ -10,7 +10,7 @@ import signal ############################################################### -# terminate-scenarios-test +# block_log_retain_blocks_test # # A basic test for --block-log-retain-blocks option. It validates # * no blocks.log is generated when the option is set to 0 @@ -45,7 +45,6 @@ try: TestHelper.printSystemInfo("BEGIN") - cluster.setWalletMgr(walletMgr) cluster.setWalletMgr(walletMgr)