Merge pull request AntelopeIO#535 from eosnetworkfoundation/add-extra…

…ct-blocks-to-eosio-blocklog-tool

[3.2] Add extract-blocks option to eosio-blocklog

libraries/chain/block_log.cpp

@@ -1185,31 +1185,29 @@ namespace eosio { namespace chain {
       }
    }
 
-   bool block_log::trim_blocklog_front(const fc::path& block_dir, const fc::path& temp_dir, uint32_t truncate_at_block) {
-      using namespace std;
-      EOS_ASSERT( block_dir != temp_dir, block_log_exception, "block_dir and temp_dir need to be different directories" );
-      ilog("In directory ${dir} will trim all blocks before block ${n} from blocks.log and blocks.index.",
-           ("dir", block_dir.generic_string())("n", truncate_at_block));
+   bool block_log::extract_block_range(const fc::path& block_dir, const fc::path&output_dir, block_num_type& start, block_num_type& end, bool rename_input) {
+      EOS_ASSERT( block_dir != output_dir, block_log_exception, "block_dir and output_dir need to be different directories" );
       trim_data original_block_log(block_dir);
-      if (truncate_at_block <= original_block_log.first_block) {
-         ilog("There are no blocks before block ${n} so do nothing.", ("n", truncate_at_block));
-         return false;
+      if(start < original_block_log.first_block) {
+         dlog("Requested start block of ${start} is less than the first available block ${n}; adjusting to ${n}", ("start", start)("n", original_block_log.first_block));
+         start = original_block_log.first_block;
       }
-      if (truncate_at_block > original_block_log.last_block) {
-         ilog("All blocks are before block ${n} so do nothing (trim front would delete entire blocks.log).", ("n", truncate_at_block));
-         return false;
+      if(end > original_block_log.last_block) {
+         dlog("Requested end block of ${end} is greater than the last available block ${n}; adjusting to ${n}", ("end", end)("n", original_block_log.last_block));
+         end = original_block_log.last_block;
       }
-
+      ilog("In directory ${ouput} will create new block log with range ${start}-${end}",
+           ("output", output_dir.generic_string())("start", start)("end", end));
       // ****** create the new block log file and write out the header for the file
-      fc::create_directories(temp_dir);
-      fc::path new_block_filename = temp_dir / "blocks.log";
+      fc::create_directories(output_dir);
+      fc::path new_block_filename = output_dir / "blocks.log";
       if (fc::remove(new_block_filename)) {
-         ilog("Removing old blocks.out file");
+         ilog("Removing existing blocks.log file");
       }
       fc::cfile new_block_file;
       new_block_file.set_file_path(new_block_filename);
-      // need to open as append since the file doesn't already exist, then reopen without append to allow writing the
-      // file in any order
+      // need to open as write since the file doesn't already exist, then reopen
+      // with read/write to allow writing the file in any order
       new_block_file.open( LOG_WRITE_C );
       new_block_file.close();
       new_block_file.open( LOG_RW_C );
@@ -1219,37 +1217,49 @@ namespace eosio { namespace chain {
       uint32_t version = block_log::max_supported_version;
       new_block_file.seek(0);
       new_block_file.write((char*)&version, sizeof(version));
-      new_block_file.write((char*)&truncate_at_block, sizeof(truncate_at_block));
+      new_block_file.write((char*)&start, sizeof(start));
 
-      new_block_file << original_block_log.chain_id;
+      if (start > 1) {
+         new_block_file << original_block_log.chain_id;
+      } else {
+         fc::raw::pack(new_block_file, original_block_log.gs);
+      }
 
       // append a totem to indicate the division between blocks and header
       auto totem = block_log::npos;
       new_block_file.write((char*)&totem, sizeof(totem));
+      // ****** end of new block log header
 
       const auto new_block_file_first_block_pos = new_block_file.tellp();
-      // ****** end of new block log header
 
       // copy over remainder of block log to new block log
-      auto buffer =  make_unique<char[]>(detail::reverse_iterator::_buf_len);
+      auto buffer =  std::make_unique<char[]>(detail::reverse_iterator::_buf_len);
       char* buf =  buffer.get();
 
       // offset bytes to shift from old blocklog position to new blocklog position
-      const uint64_t original_file_block_pos = original_block_log.block_pos(truncate_at_block);
-      const uint64_t pos_delta = original_file_block_pos - new_block_file_first_block_pos;
-      auto status = fseek(original_block_log.blk_in, 0, SEEK_END);
-      EOS_ASSERT( status == 0, block_log_exception, "blocks.log seek failed" );
+      const uint64_t original_file_start_block_pos = original_block_log.block_pos(start);
+      const uint64_t pos_delta = original_file_start_block_pos - new_block_file_first_block_pos;
+      uint64_t original_file_end_block_pos;
+      if (end == original_block_log.last_block) {
+         auto status = fseek(original_block_log.blk_in, 0, SEEK_END);
+         EOS_ASSERT( status == 0, block_log_exception, "blocks.log seek failed" );
+         original_file_end_block_pos = ftell(original_block_log.blk_in);
+      } else {
+         original_file_end_block_pos = original_block_log.block_pos(end+1);
+         auto status = fseek(original_block_log.blk_in, original_file_end_block_pos, SEEK_SET);
+         EOS_ASSERT( status == 0, block_log_exception, "blocks.log seek failed" );
+      }
 
-      // all blocks to copy to the new blocklog
-      const uint64_t to_write = ftell(original_block_log.blk_in) - original_file_block_pos;
-      const auto pos_size = sizeof(uint64_t);
+      // all bytes to copy to the new blocklog
+      const uint64_t to_write = original_file_end_block_pos - original_file_start_block_pos;
 
       // start with the last block's position stored at the end of the block
-      uint64_t original_pos = ftell(original_block_log.blk_in) - pos_size;
+      const auto pos_size = sizeof(uint64_t);
+      uint64_t original_pos = original_file_end_block_pos - pos_size;
 
-      const auto num_blocks = original_block_log.last_block - truncate_at_block + 1;
+      const auto num_blocks = end - start + 1;
 
-      fc::path new_index_filename = temp_dir / "blocks.index";
+      fc::path new_index_filename = output_dir / "blocks.index";
       detail::index_writer index(new_index_filename, num_blocks);
 
       uint64_t read_size = 0;
@@ -1261,10 +1271,11 @@ namespace eosio { namespace chain {
          }
 
          // read in the previous contiguous memory into the read buffer
-         const auto start_of_blk_buffer_pos = original_file_block_pos + to_write_remaining - read_size;
-         status = fseek(original_block_log.blk_in, start_of_blk_buffer_pos, SEEK_SET);
+         const auto start_of_blk_buffer_pos = original_file_start_block_pos + to_write_remaining - read_size;
+         auto status = fseek(original_block_log.blk_in, start_of_blk_buffer_pos, SEEK_SET);
+         EOS_ASSERT( status == 0, block_log_exception, "original blocks.log seek failed" );
          const auto num_read = fread(buf, read_size, 1, original_block_log.blk_in);
-         EOS_ASSERT( num_read == 1, block_log_exception, "blocks.log read failed" );
+         EOS_ASSERT( num_read == 1, block_log_exception, "original blocks.log read failed" );
 
          // walk this memory section to adjust block position to match the adjusted location
          // of the block start and store in the new index file
@@ -1293,12 +1304,14 @@ namespace eosio { namespace chain {
       new_block_file.flush();
       new_block_file.close();
 
-      fc::path old_log = temp_dir / "old.log";
-      rename(original_block_log.block_file_name, old_log);
-      rename(new_block_filename, original_block_log.block_file_name);
-      fc::path old_ind = temp_dir / "old.index";
-      rename(original_block_log.index_file_name, old_ind);
-      rename(new_index_filename, original_block_log.index_file_name);
+      if (rename_input) {
+         fc::path old_log = output_dir / "old.log";
+         rename(original_block_log.block_file_name, old_log);
+         rename(new_block_filename, original_block_log.block_file_name);
+         fc::path old_ind = output_dir / "old.index";
+         rename(original_block_log.index_file_name, old_ind);
+         rename(new_index_filename, original_block_log.index_file_name);
+      }
 
       return true;
    }
@@ -1333,7 +1346,6 @@ namespace eosio { namespace chain {
          EOS_ASSERT(size == 1, block_log_exception, "invalid format for file ${file}",
                     ("file", block_file_name.string()));
          if (block_log::contains_genesis_state(version, first_block)) {
-            genesis_state gs;
             fc::raw::unpack(ds, gs);
             chain_id = gs.compute_chain_id();
          }
@@ -1429,4 +1441,4 @@ namespace eosio { namespace chain {
 // used only for unit test to adjust the buffer length
 void block_log_set_buff_len(uint64_t len){
     eosio::chain::detail::reverse_iterator::_buf_len = len;
-}
+}

libraries/chain/include/eosio/chain/block_log.hpp

@@ -91,7 +91,7 @@ namespace eosio { namespace chain {
 
          static bool is_pruned_log(const fc::path& data_dir);
 
-         static bool trim_blocklog_front(const fc::path& block_dir, const fc::path& temp_dir, uint32_t truncate_at_block);
+         static bool extract_block_range(const fc::path& block_dir, const fc::path&output_dir, block_num_type& start, block_num_type& end, bool rename_input=false);
 
    private:
          void open(const fc::path& data_dir);
@@ -119,6 +119,7 @@ namespace eosio { namespace chain {
       FILE* ind_in = nullptr;                            //C style files for reading blocks.log and blocks.index
       //we use low level file IO because it is distinctly faster than C++ filebuf or iostream
       uint64_t first_block_pos = 0;                      //file position in blocks.log for the first block in the log
+      genesis_state gs;
       chain_id_type chain_id;
 
       static constexpr int blknum_offset{14};            //offset from start of block to 4 byte block number, valid for the only allowed versions

programs/eosio-blocklog/main.cpp

@@ -47,6 +47,7 @@ struct blocklog {
    bool                             as_json_array = false;
    bool                             make_index = false;
    bool                             trim_log = false;
+   bool                             extract_blocks = false;
    bool                             smoke_test = false;
    bool                             vacuum = false;
    bool                             help = false;
@@ -193,7 +194,11 @@ void blocklog::set_program_options(options_description& cli)
          ("make-index", bpo::bool_switch(&make_index)->default_value(false),
           "Create blocks.index from blocks.log. Must give 'blocks-dir'. Give 'output-file' relative to current directory or absolute path (default is <blocks-dir>/blocks.index).")
          ("trim-blocklog", bpo::bool_switch(&trim_log)->default_value(false),
-          "Trim blocks.log and blocks.index. Must give 'blocks-dir' and 'first and/or 'last'.")
+          "Trim blocks.log and blocks.index. Must give 'blocks-dir' and 'first' and/or 'last'.")
+         ("extract-blocks", bpo::bool_switch(&extract_blocks)->default_value(false),
+          "Extract range of blocks from blocks.log and write to output-dir.  Must give 'first' and/or 'last'.")
+         ("output-dir", bpo::value<bfs::path>(),
+          "the output directory for the block log extracted from blocks-dir")
          ("smoke-test", bpo::bool_switch(&smoke_test)->default_value(false),
           "Quick test that blocks.log and blocks.index are well formed and agree with each other.")
          ("vacuum", bpo::bool_switch(&vacuum)->default_value(false),
@@ -253,7 +258,16 @@ int trim_blocklog_end(bfs::path block_dir, uint32_t n) {       //n is last block
 
 bool trim_blocklog_front(bfs::path block_dir, uint32_t n) {        //n is first block to keep (remove prior blocks)
    report_time rt("trimming blocklog start");
-   const bool status = block_log::trim_blocklog_front(block_dir, block_dir / "old", n);
+   block_num_type end = std::numeric_limits<block_num_type>::max();
+   const bool status = block_log::extract_block_range(block_dir, block_dir / "old", n, end, true);
+   rt.report();
+   return status;
+}
+
+bool extract_block_range(bfs::path block_dir, bfs::path output_dir, uint32_t start, uint32_t end) {
+   report_time rt("extracting block range");
+   EOS_ASSERT( end > start, block_log_exception, "extract range end must be greater than start");
+   const bool status = block_log::extract_block_range(block_dir, output_dir, start, end, false);
    rt.report();
    return status;
 }
@@ -308,7 +322,7 @@ int main(int argc, char** argv) {
       }
       if (blog.trim_log) {
          if (blog.first_block == 0 && blog.last_block == std::numeric_limits<uint32_t>::max()) {
-            std::cerr << "trim-blocklog does nothing unless specify first and/or last block.";
+            std::cerr << "trim-blocklog does nothing unless first and/or last block are specified.";
             return -1;
          }
          if (blog.last_block != std::numeric_limits<uint32_t>::max()) {
@@ -321,6 +335,15 @@ int main(int argc, char** argv) {
          }
          return 0;
       }
+      if (blog.extract_blocks) {
+         if (blog.first_block == 0 && blog.last_block == std::numeric_limits<uint32_t>::max()) {
+            std::cerr << "extract-blocklog does nothing unless first and/or last block are specified.";
+            return -1;
+         }
+         if (!extract_block_range(vmap.at("blocks-dir").as<bfs::path>(), vmap.at("output-dir").as<bfs::path>(), blog.first_block, blog.last_block))
+            return -1;
+         return 0;
+      }
       if (blog.vacuum) {
          blog.initialize(vmap);
          blog.do_vacuum();

unittests/block_log_extract.cpp

@@ -0,0 +1,100 @@
+#include <boost/test/unit_test.hpp>
+
+#include <fc/bitutil.hpp>
+
+#include <eosio/chain/block_log.hpp>
+#include <eosio/chain/block.hpp>
+
+using namespace eosio::chain;
+
+struct block_log_extract_fixture {
+   block_log_extract_fixture() {
+      log.emplace(dir.path(), std::optional<block_log_prune_config>());
+      log->reset(genesis_state(), std::make_shared<signed_block>());
+      BOOST_REQUIRE_EQUAL(log->first_block_num(), 1);
+      BOOST_REQUIRE_EQUAL(log->head()->block_num(), 1);
+      for(uint32_t i = 2; i < 13; ++i) {
+         add(i);
+      }
+      BOOST_REQUIRE_EQUAL(log->head()->block_num(), 12);
+   };
+
+   void add(uint32_t index) {
+      signed_block_ptr p = std::make_shared<signed_block>();
+      p->previous._hash[0] = fc::endian_reverse_u32(index-1);
+      log->append(p);
+   }
+
+   genesis_state gs;
+   fc::temp_directory dir;
+   std::optional<block_log> log;
+};
+
+BOOST_AUTO_TEST_SUITE(block_log_extraction_tests)
+
+BOOST_FIXTURE_TEST_CASE(extract_from_middle, block_log_extract_fixture) try {
+
+   fc::temp_directory output_dir;
+   block_num_type start=3, end=7;
+   block_log::extract_block_range(dir.path(), output_dir.path(), start, end);
+
+   block_log new_log(output_dir.path(), std::optional<block_log_prune_config>());
+
+   auto id = gs.compute_chain_id();
+   BOOST_REQUIRE_EQUAL(new_log.extract_chain_id(output_dir.path()), id);
+   BOOST_REQUIRE_EQUAL(new_log.first_block_num(), 3);
+   BOOST_REQUIRE_EQUAL(new_log.head()->block_num(), 7);
+
+
+} FC_LOG_AND_RETHROW()
+
+BOOST_FIXTURE_TEST_CASE(extract_from_start, block_log_extract_fixture) try {
+
+   fc::temp_directory output_dir;
+   block_num_type start=0, end=7;
+   block_log::extract_block_range(dir.path(), output_dir.path(), start, end);
+
+   block_log new_log(output_dir.path(), std::optional<block_log_prune_config>());
+
+   auto id = gs.compute_chain_id();
+   BOOST_REQUIRE_EQUAL(new_log.extract_chain_id(output_dir.path()), id);
+   BOOST_REQUIRE_EQUAL(new_log.first_block_num(), 1);
+   BOOST_REQUIRE_EQUAL(new_log.head()->block_num(), 7);
+
+} FC_LOG_AND_RETHROW()
+
+BOOST_FIXTURE_TEST_CASE(reextract_from_start, block_log_extract_fixture) try {
+
+   fc::temp_directory output_dir;
+   block_num_type start=0, end=9;
+   block_log::extract_block_range(dir.path(), output_dir.path(), start, end);
+
+   fc::temp_directory output_dir2;
+   end=6;
+   block_log::extract_block_range(output_dir.path(), output_dir2.path(), start, end);
+
+   block_log new_log(output_dir2.path(), std::optional<block_log_prune_config>());
+
+   auto id = gs.compute_chain_id();
+   BOOST_REQUIRE_EQUAL(new_log.extract_chain_id(output_dir2.path()), id);
+   BOOST_REQUIRE_EQUAL(new_log.first_block_num(), 1);
+   BOOST_REQUIRE_EQUAL(new_log.head()->block_num(), 6);
+
+} FC_LOG_AND_RETHROW()
+
+BOOST_FIXTURE_TEST_CASE(extract_to_end, block_log_extract_fixture) try {
+
+   fc::temp_directory output_dir;
+   block_num_type start=5, end=std::numeric_limits<block_num_type>::max();
+   block_log::extract_block_range(dir.path(), output_dir.path(), start, end);
+
+   block_log new_log(output_dir.path(), std::optional<block_log_prune_config>());
+
+   auto id = gs.compute_chain_id();
+   BOOST_REQUIRE_EQUAL(new_log.extract_chain_id(output_dir.path()), id);
+   BOOST_REQUIRE_EQUAL(new_log.first_block_num(), 5);
+   BOOST_REQUIRE_EQUAL(new_log.head()->block_num(), 12);
+
+} FC_LOG_AND_RETHROW()
+
+BOOST_AUTO_TEST_SUITE_END()

unittests/restart_chain_tests.cpp

@@ -287,7 +287,8 @@ void trim_blocklog_front(uint32_t truncate_at_block, buf_len_type len_type) {
          return;
    }
 
-   BOOST_CHECK( block_log::trim_blocklog_front(temp1.path, temp2.path, truncate_at_block) == true);
+   block_num_type end = std::numeric_limits<block_num_type>::max();
+   BOOST_CHECK( block_log::extract_block_range(temp1.path, temp2.path, truncate_at_block, end, true) == true);
    trim_data new_log(temp1.path);
    BOOST_CHECK(new_log.first_block == truncate_at_block);
    BOOST_CHECK(new_log.last_block == old_log.last_block);