From 9d1ff5f8a804695d9c2407b0413567654ed091a9 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 2 Feb 2022 13:22:16 -0600 Subject: [PATCH 1/3] Add snapshot-to-json command line option --- .../chain/include/eosio/chain/snapshot.hpp | 15 ++++++ libraries/chain/snapshot.cpp | 39 ++++++++++++++ plugins/chain_plugin/chain_plugin.cpp | 54 ++++++++++++++++++- 3 files changed, 107 insertions(+), 1 deletion(-) diff --git a/libraries/chain/include/eosio/chain/snapshot.hpp b/libraries/chain/include/eosio/chain/snapshot.hpp index e5bfd26243..cce1292963 100644 --- a/libraries/chain/include/eosio/chain/snapshot.hpp +++ b/libraries/chain/include/eosio/chain/snapshot.hpp @@ -342,7 +342,22 @@ namespace eosio { namespace chain { std::streampos header_pos; std::streampos section_pos; uint64_t row_count; + }; + + class ostream_json_snapshot_writer : public snapshot_writer { + public: + explicit ostream_json_snapshot_writer(std::ostream& snapshot); + + void write_start_section( const std::string& section_name ) override; + void write_row( const detail::abstract_snapshot_row_writer& row_writer ) override; + void write_end_section() override; + void finalize(); + + static const uint32_t magic_number = 0x30510550; + private: + detail::ostream_wrapper snapshot; + uint64_t row_count; }; class istream_snapshot_reader : public snapshot_reader { diff --git a/libraries/chain/snapshot.cpp b/libraries/chain/snapshot.cpp index 66966ef6be..a3a6829639 100644 --- a/libraries/chain/snapshot.cpp +++ b/libraries/chain/snapshot.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace eosio { namespace chain { @@ -190,6 +191,44 @@ void ostream_snapshot_writer::finalize() { snapshot.write((char*)&end_marker, sizeof(end_marker)); } +ostream_json_snapshot_writer::ostream_json_snapshot_writer(std::ostream& snapshot) + :snapshot(snapshot) + ,row_count(0) +{ + snapshot << "{\n"; + // write magic number + auto totem = magic_number; + snapshot << "\"magic_number\":" << fc::json::to_string(totem, fc::time_point::maximum()) << "\n"; + + // write version + auto version = current_snapshot_version; + snapshot << ",\"version\":" << fc::json::to_string(version, fc::time_point::maximum()) << "\n"; +} + +void ostream_json_snapshot_writer::write_start_section( const std::string& section_name ) +{ + snapshot.inner << "," << fc::json::to_string(section_name, fc::time_point::maximum()) << ":{\n"; +} + +void ostream_json_snapshot_writer::write_row( const detail::abstract_snapshot_row_writer& row_writer ) { + const auto yield = [&](size_t s) {}; + + if(row_count != 0) snapshot.inner << ","; + snapshot.inner << "\"row_" << row_count << "\":" << fc::json::to_string(row_writer.to_variant(), yield) << "\n"; + ++row_count; +} + +void ostream_json_snapshot_writer::write_end_section( ) { + snapshot.inner << "}\n"; + row_count = 0; +} + +void ostream_json_snapshot_writer::finalize() { + snapshot.inner << "}\n"; + snapshot.inner.flush(); +} + + istream_snapshot_reader::istream_snapshot_reader(std::istream& snapshot) :snapshot(snapshot) ,header_pos(snapshot.tellg()) diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index 9141072cde..4619921cbb 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -360,6 +360,8 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip "print build environment information to console as JSON and exit") ("extract-build-info", bpo::value(), "extract build environment information as JSON, write into specified file, and exit") + ("snapshot-to-json", bpo::value(), + "convert snapshot to JSON format, and exit") ("force-all-checks", bpo::bool_switch()->default_value(false), "do not skip any validation checks while replaying blocks (useful for replaying blocks from untrusted source)") ("disable-replay-opts", bpo::bool_switch()->default_value(false), @@ -862,6 +864,57 @@ void chain_plugin::plugin_initialize(const variables_map& options) { EOS_THROW( extract_genesis_state_exception, "extracted genesis state from blocks.log" ); } + std::optional chain_id; + if( options.count("snapshot-to-json") ) { + my->snapshot_path = options.at( "snapshot-to-json" ).as(); + EOS_ASSERT( fc::exists(*my->snapshot_path), plugin_config_exception, + "Cannot load snapshot, ${name} does not exist", ("name", my->snapshot_path->generic_string()) ); + + // recover genesis information from the snapshot + // used for validation code below + auto infile = std::ifstream(my->snapshot_path->generic_string(), (std::ios::in | std::ios::binary)); + istream_snapshot_reader reader(infile); + reader.validate(); + chain_id = controller::extract_chain_id(reader); + infile.close(); + + boost::filesystem::path temp_dir = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path(); + my->chain_config->state_dir = temp_dir / "state"; + my->blocks_dir = temp_dir / "blocks"; + my->chain_config->blog.log_dir = my->blocks_dir; + try { + auto shutdown = [](){ return app().quit(); }; + auto check_shutdown = [](){ return app().is_quiting(); }; + auto infile = std::ifstream(my->snapshot_path->generic_string(), (std::ios::in | std::ios::binary)); + auto reader = std::make_shared(infile); + my->chain.emplace( *my->chain_config, std::move(pfs), *chain_id ); + my->chain->add_indices(); + my->chain->startup(shutdown, check_shutdown, reader); + infile.close(); + app().quit(); // shutdown as we will be finished after writing the snapshot + + ilog("Writing snapshot: ${s}", ("s", my->snapshot_path->generic_string() + ".json")); + auto snap_out = std::ofstream( my->snapshot_path->generic_string() + ".json", (std::ios::out) ); + auto writer = std::make_shared( snap_out ); + my->chain->write_snapshot( writer ); + writer->finalize(); + snap_out.flush(); + snap_out.close(); + } catch (const database_guard_exception& e) { + log_guard_exception(e); + // make sure to properly close the db + my->chain.reset(); + fc::remove_all(temp_dir); + throw; + } + my->chain.reset(); + fc::remove_all(temp_dir); + ilog("Completed writing snapshot: ${s}", ("s", my->snapshot_path->generic_string() + ".json")); + ilog("==== Ignore any additional log messages. ===="); + + EOS_THROW( node_management_success, "extracted json from snapshot" ); + } + // move fork_db to new location upgrade_from_reversible_to_fork_db( my.get() ); @@ -888,7 +941,6 @@ void chain_plugin::plugin_initialize(const variables_map& options) { wlog( "The --truncate-at-block option can only be used with --hard-replay-blockchain." ); } - std::optional chain_id; if (options.count( "snapshot" )) { my->snapshot_path = options.at( "snapshot" ).as(); EOS_ASSERT( fc::exists(*my->snapshot_path), plugin_config_exception, From 84a4eae3547b0693fce9ef31ef0e74ee1c1000e4 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 7 Feb 2022 13:32:45 -0600 Subject: [PATCH 2/3] Address PR comments, add better description to option --- libraries/chain/snapshot.cpp | 1 + plugins/chain_plugin/chain_plugin.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/chain/snapshot.cpp b/libraries/chain/snapshot.cpp index a3a6829639..ce6adec721 100644 --- a/libraries/chain/snapshot.cpp +++ b/libraries/chain/snapshot.cpp @@ -207,6 +207,7 @@ ostream_json_snapshot_writer::ostream_json_snapshot_writer(std::ostream& snapsho void ostream_json_snapshot_writer::write_start_section( const std::string& section_name ) { + row_count = 0; snapshot.inner << "," << fc::json::to_string(section_name, fc::time_point::maximum()) << ":{\n"; } diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index 4619921cbb..cea50ccf7f 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -361,7 +361,7 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip ("extract-build-info", bpo::value(), "extract build environment information as JSON, write into specified file, and exit") ("snapshot-to-json", bpo::value(), - "convert snapshot to JSON format, and exit") + "snapshot file to convert to JSON format, writes to .json (tmp state dir used), and exit") ("force-all-checks", bpo::bool_switch()->default_value(false), "do not skip any validation checks while replaying blocks (useful for replaying blocks from untrusted source)") ("disable-replay-opts", bpo::bool_switch()->default_value(false), From 4970c7f3011d689d96c3bc2e7d9d1ded6ad70bd6 Mon Sep 17 00:00:00 2001 From: Clayton Calabrese Date: Wed, 20 Jul 2022 15:52:39 -0500 Subject: [PATCH 3/3] fix build errors --- plugins/chain_plugin/chain_plugin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index cea50ccf7f..07e240b517 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -881,7 +881,7 @@ void chain_plugin::plugin_initialize(const variables_map& options) { boost::filesystem::path temp_dir = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path(); my->chain_config->state_dir = temp_dir / "state"; my->blocks_dir = temp_dir / "blocks"; - my->chain_config->blog.log_dir = my->blocks_dir; + my->chain_config->blocks_dir = my->blocks_dir; try { auto shutdown = [](){ return app().quit(); }; auto check_shutdown = [](){ return app().is_quiting(); };