From 9eb67cc41e29c0248ee1e621c06c734144690d2b Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Fri, 15 Mar 2024 20:54:27 -0400 Subject: [PATCH 01/18] implement controller::get_chain_head_finality_data() for SHiP --- libraries/chain/block_state.cpp | 11 +++++++++++ libraries/chain/controller.cpp | 12 ++++++++++++ .../chain/include/eosio/chain/block_state.hpp | 15 +++++++++++++++ .../chain/include/eosio/chain/controller.hpp | 1 + 4 files changed, 39 insertions(+) diff --git a/libraries/chain/block_state.cpp b/libraries/chain/block_state.cpp index 08e742df46..fd10897d3b 100644 --- a/libraries/chain/block_state.cpp +++ b/libraries/chain/block_state.cpp @@ -297,6 +297,17 @@ digest_type block_state::get_finality_mroot_claim(const qc_claim_t& qc_claim) co return get_validation_mroot(next_core_metadata.final_on_strong_qc_block_num); } +finality_data_t block_state::get_finality_data() const { + assert(action_mroot.has_value()); + finality_data_t finality_data { + // other fields take the default values set by finality_data_t definition + .action_mroot = *action_mroot, + .base_digest = compute_base_digest() // from block_header_state + }; + + return finality_data; +} + void inject_additional_signatures( signed_block& b, const std::vector& additional_signatures) { if (!additional_signatures.empty()) { diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index e392dbf727..c34def353f 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -753,6 +753,7 @@ struct building_block { // have one. if (!validating_bsp->valid) { validating_bsp->valid = bb.parent.new_valid(bhs, action_mroot); + validating_bsp->action_mroot = action_mroot; // caching for constructing finality_data. Only needed when block is commited. } } else { // Create the valid structure for producing @@ -4102,6 +4103,13 @@ struct controller_impl { } } + finality_data_t get_chain_head_finality_data(block_id_type block_id) const { + return apply_s(chain_head, [&](const auto& head) { + assert(head->id() == block_id); + return head->get_finality_data(); + }); + } + uint32_t earliest_available_block_num() const { return (blog.first_block_num() != 0) ? blog.first_block_num() : fork_db_root_block_num(); } @@ -4668,6 +4676,10 @@ uint32_t controller::if_irreversible_block_num() const { return block_header::num_from_id(my->if_irreversible_block_id); } +finality_data_t controller::get_chain_head_finality_data(block_id_type block_id) const { + return my->get_chain_head_finality_data(block_id); +} + uint32_t controller::last_irreversible_block_num() const { return my->fork_db_root_block_num(); } diff --git a/libraries/chain/include/eosio/chain/block_state.hpp b/libraries/chain/include/eosio/chain/block_state.hpp index 2c9c26e212..c19937383b 100644 --- a/libraries/chain/include/eosio/chain/block_state.hpp +++ b/libraries/chain/include/eosio/chain/block_state.hpp @@ -53,6 +53,15 @@ struct valid_t { std::vector validation_mroots; }; +// This is mostly used by SHiP to stream finality_data +struct finality_data_t { + uint32_t major_version{light_header_protocol_version_major}; + uint32_t minor_version{light_header_protocol_version_minor}; + uint32_t active_finalizer_policy_generation{0}; + digest_type action_mroot{}; + digest_type base_digest{}; +}; + struct block_state : public block_header_state { // block_header_state provides parent link // ------ data members ------------------------------------------------------------- signed_block_ptr block; @@ -69,6 +78,7 @@ struct block_state : public block_header_state { // block_header_state provi // ------ data members caching information available elsewhere ---------------------- bool pub_keys_recovered = false; deque cached_trxs; + std::optional action_mroot{std::nullopt}; // ------ private methods ----------------------------------------------------------- bool is_valid() const { return validated; } @@ -81,6 +91,7 @@ struct block_state : public block_header_state { // block_header_state provi friend struct fc::reflector; friend struct controller_impl; friend struct completed_block; + friend struct building_block; public: // ------ functions ----------------------------------------------------------------- const block_id_type& id() const { return block_header_state::id(); } @@ -107,6 +118,9 @@ struct block_state : public block_header_state { // block_header_state provi // Returns finality_mroot_claim of the current block digest_type get_finality_mroot_claim(const qc_claim_t& qc_claim) const; + // Returns finality_data of the current block + finality_data_t get_finality_data() const; + // vote_status vote_status aggregate_vote(const vote_message& vote); // aggregate vote into pending_qc void verify_qc(const valid_quorum_certificate& qc) const; // verify given qc is valid with respect block_state @@ -147,4 +161,5 @@ using block_state_ptr = std::shared_ptr; // not exporting pending_qc or valid_qc FC_REFLECT( eosio::chain::valid_t::finality_leaf_node_t, (major_version)(minor_version)(block_num)(finality_digest)(action_mroot) ) FC_REFLECT( eosio::chain::valid_t, (validation_tree)(validation_mroots)) +FC_REFLECT( eosio::chain::finality_data_t, (major_version)(minor_version)(active_finalizer_policy_generation)(action_mroot)(base_digest)) FC_REFLECT_DERIVED( eosio::chain::block_state, (eosio::chain::block_header_state), (block)(strong_digest)(weak_digest)(pending_qc)(valid_qc)(valid)(validated) ) diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index e45cb75deb..2e1771940a 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -263,6 +263,7 @@ namespace eosio::chain { void set_if_irreversible_block_id(const block_id_type& id); uint32_t if_irreversible_block_num() const; + finality_data_t get_chain_head_finality_data(block_id_type block_id) const; uint32_t last_irreversible_block_num() const; block_id_type last_irreversible_block_id() const; From d08e321f2b75f550bc54b24f6c03102437da2a84 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Sat, 16 Mar 2024 15:59:55 -0400 Subject: [PATCH 02/18] update ABI definition and types for the new finality_data log --- libraries/state_history/abi.cpp | 6 ++++-- .../include/eosio/state_history/serialization.hpp | 1 + .../state_history/include/eosio/state_history/types.hpp | 6 ++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/libraries/state_history/abi.cpp b/libraries/state_history/abi.cpp index d778d0c7fc..649ca3a7a2 100644 --- a/libraries/state_history/abi.cpp +++ b/libraries/state_history/abi.cpp @@ -30,7 +30,8 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "irreversible_only", "type": "bool" }, { "name": "fetch_block", "type": "bool" }, { "name": "fetch_traces", "type": "bool" }, - { "name": "fetch_deltas", "type": "bool" } + { "name": "fetch_deltas", "type": "bool" }, + { "name": "fetch_finality_data", "type": "bool" } ] }, { @@ -46,7 +47,8 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "prev_block", "type": "block_position?" }, { "name": "block", "type": "bytes?" }, { "name": "traces", "type": "bytes?" }, - { "name": "deltas", "type": "bytes?" } + { "name": "deltas", "type": "bytes?" }, + { "name": "finality_data", "type": "bytes?" } ] }, { diff --git a/libraries/state_history/include/eosio/state_history/serialization.hpp b/libraries/state_history/include/eosio/state_history/serialization.hpp index b39698291c..50d90bb431 100644 --- a/libraries/state_history/include/eosio/state_history/serialization.hpp +++ b/libraries/state_history/include/eosio/state_history/serialization.hpp @@ -716,6 +716,7 @@ datastream& operator<<(datastream& ds, const eosio::state_history::get_b history_pack_big_bytes(ds, obj.block); history_pack_big_bytes(ds, obj.traces); history_pack_big_bytes(ds, obj.deltas); + history_pack_big_bytes(ds, obj.finality_data); return ds; } diff --git a/libraries/state_history/include/eosio/state_history/types.hpp b/libraries/state_history/include/eosio/state_history/types.hpp index e86275ad8d..990fb8fdd4 100644 --- a/libraries/state_history/include/eosio/state_history/types.hpp +++ b/libraries/state_history/include/eosio/state_history/types.hpp @@ -102,6 +102,7 @@ struct get_blocks_request_v0 { bool fetch_block = false; bool fetch_traces = false; bool fetch_deltas = false; + bool fetch_finality_data = false; }; struct get_blocks_ack_request_v0 { @@ -119,6 +120,7 @@ struct get_blocks_result_base { struct get_blocks_result_v0 : get_blocks_result_base { std::optional traces; std::optional deltas; + std::optional finality_data; }; using state_request = std::variant; @@ -132,8 +134,8 @@ FC_REFLECT(eosio::state_history::table_delta, (struct_version)(name)(rows)); FC_REFLECT(eosio::state_history::block_position, (block_num)(block_id)); FC_REFLECT_EMPTY(eosio::state_history::get_status_request_v0); FC_REFLECT(eosio::state_history::get_status_result_v0, (head)(last_irreversible)(trace_begin_block)(trace_end_block)(chain_state_begin_block)(chain_state_end_block)(chain_id)); -FC_REFLECT(eosio::state_history::get_blocks_request_v0, (start_block_num)(end_block_num)(max_messages_in_flight)(have_positions)(irreversible_only)(fetch_block)(fetch_traces)(fetch_deltas)); +FC_REFLECT(eosio::state_history::get_blocks_request_v0, (start_block_num)(end_block_num)(max_messages_in_flight)(have_positions)(irreversible_only)(fetch_block)(fetch_traces)(fetch_deltas)(fetch_finality_data)); FC_REFLECT(eosio::state_history::get_blocks_ack_request_v0, (num_messages)); FC_REFLECT(eosio::state_history::get_blocks_result_base, (head)(last_irreversible)(this_block)(prev_block)(block)); -FC_REFLECT_DERIVED(eosio::state_history::get_blocks_result_v0, (eosio::state_history::get_blocks_result_base), (traces)(deltas)); +FC_REFLECT_DERIVED(eosio::state_history::get_blocks_result_v0, (eosio::state_history::get_blocks_result_base), (traces)(deltas)(finality_data)); // clang-format on From 900fa225f4ab3a90eb420cfc8e5efdc5ce55140b Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Sun, 17 Mar 2024 18:04:17 -0400 Subject: [PATCH 03/18] Support finality_data in state_history_plugin --- .../eosio/state_history_plugin/session.hpp | 53 ++++++++++++------- .../state_history_plugin.hpp | 1 + .../state_history_plugin.cpp | 36 ++++++++++++- 3 files changed, 71 insertions(+), 19 deletions(-) diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/session.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/session.hpp index 94da1e3e1d..d6ac250628 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/session.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/session.hpp @@ -237,7 +237,7 @@ class blocks_result_send_queue_entry : public send_queue_entry_base, public std: } template - void send_log(uint64_t entry_size, bool is_deltas, Next&& next) { + void send_log(uint64_t entry_size, bool fin, Next&& next) { if (entry_size) { data.resize(16); // should be at least for 1 byte (optional) + 10 bytes (variable sized uint64_t) fc::datastream ds(data.data(), data.size()); @@ -248,10 +248,10 @@ class blocks_result_send_queue_entry : public send_queue_entry_base, public std: data = {'\0'}; // optional false } - async_send(is_deltas && entry_size == 0, data, - [is_deltas, entry_size, next = std::forward(next), me=this->shared_from_this()]() mutable { + async_send(fin && entry_size == 0, data, + [fin, entry_size, next = std::forward(next), me=this->shared_from_this()]() mutable { if (entry_size) { - me->async_send_buf(is_deltas, [me, next = std::move(next)]() { + me->async_send_buf(fin, [me, next = std::move(next)]() { next(); }); } else @@ -259,14 +259,24 @@ class blocks_result_send_queue_entry : public send_queue_entry_base, public std: }); } - void send_deltas() { + // last to be sent + void send_finality_data() { stream.reset(); - send_log(session->get_delta_log_entry(r, stream), true, [me=this->shared_from_this()]() { + send_log(session->get_finality_data_log_entry(r, stream), true, [me=this->shared_from_this()]() { me->stream.reset(); me->session->session_mgr.pop_entry(); }); } + // second to be sent + void send_deltas() { + stream.reset(); + send_log(session->get_delta_log_entry(r, stream), false, [me=this->shared_from_this()]() { + me->send_finality_data(); + }); + } + + // first to be sent void send_traces() { stream.reset(); send_log(session->get_trace_log_entry(r, stream), false, [me=this->shared_from_this()]() { @@ -379,10 +389,11 @@ struct session : session_base, std::enable_shared_from_this& buf) { - if (result.traces.has_value()) { - auto& optional_log = plugin.get_trace_log(); + uint64_t get_log_entry_impl(const eosio::state_history::get_blocks_result_v0& result, + bool has_value, + std::optional& optional_log, + std::optional& buf) { + if (has_value) { if( optional_log ) { buf.emplace( optional_log->create_locked_decompress_stream() ); return optional_log->get_unpacked_entry( result.this_block->block_num, *buf ); @@ -391,16 +402,19 @@ struct session : session_base, std::enable_shared_from_this& buf) { + return get_log_entry_impl(result, result.traces.has_value(), plugin.get_trace_log(), buf); + } + uint64_t get_delta_log_entry(const eosio::state_history::get_blocks_result_v0& result, std::optional& buf) { - if (result.deltas.has_value()) { - auto& optional_log = plugin.get_chain_state_log(); - if( optional_log ) { - buf.emplace( optional_log->create_locked_decompress_stream() ); - return optional_log->get_unpacked_entry( result.this_block->block_num, *buf ); - } - } - return 0; + return get_log_entry_impl(result, result.deltas.has_value(), plugin.get_chain_state_log(), buf); + } + + uint64_t get_finality_data_log_entry(const eosio::state_history::get_blocks_result_v0& result, + std::optional& buf) { + return get_log_entry_impl(result, result.finality_data.has_value(), plugin.get_finality_data_log(), buf); } void process(state_history::get_status_request_v0&) { @@ -535,6 +549,9 @@ struct session : session_base, std::enable_shared_from_thisfetch_deltas && plugin.get_chain_state_log()) result.deltas.emplace(); + if (current_request->fetch_finality_data && plugin.get_finality_data_log()) { + result.finality_data.emplace(); // create finality_data (it's an optional field) + } } ++to_send_block_num; diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp index 017dda0b12..a9070115d1 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp @@ -31,6 +31,7 @@ class state_history_plugin : public plugin { const state_history_log* trace_log() const; const state_history_log* chain_state_log() const; + const state_history_log* finality_data_log() const; private: state_history_ptr my; diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index a676249a14..e80b983463 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -54,6 +54,7 @@ struct state_history_plugin_impl : std::enable_shared_from_this trace_log; std::optional chain_state_log; + std::optional finality_data_log; uint32_t first_available_block = 0; bool trace_debug_mode = false; std::optional applied_transaction_connection; @@ -84,6 +85,7 @@ struct state_history_plugin_impl : std::enable_shared_from_this& get_trace_log() { return trace_log; } std::optional& get_chain_state_log(){ return chain_state_log; } + std::optional& get_finality_data_log(){ return finality_data_log; } boost::asio::io_context& get_ship_executor() { return thread_pool.get_executor(); } @@ -126,6 +128,11 @@ struct state_history_plugin_impl : std::enable_shared_from_thisget_block_id( block_num ); + if( id ) + return id; + } try { return chain_plug->chain().get_block_id_for_num(block_num); } catch (...) { @@ -207,6 +214,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thisprevious, block->block_num()); + store_finality_data(block, id); } catch (const fc::exception& e) { fc_elog(_log, "fc::exception: ${details}", ("details", e.to_detail_string())); // Both app().quit() and exception throwing are required. Without app().quit(), @@ -264,12 +272,25 @@ struct state_history_plugin_impl : std::enable_shared_from_thispack_and_write_entry(header, previous_id, [this, fresh](auto&& buf) { pack_deltas(buf, chain_plug->chain().db(), fresh); }); } // store_chain_state + // called from main thread + void store_finality_data(const signed_block_ptr& block, const block_id_type& id) { + if (!finality_data_log) + return; + + state_history_log_header header{ + .magic = ship_magic(ship_current_version, 0), .block_id = id, .payload_size = 0}; + finality_data_log->pack_and_write_entry(header, block->previous, [this, id](auto&& buf) { + fc::datastream ds{buf}; + fc::raw::pack(ds, chain_plug->chain().get_chain_head_finality_data(id)); + }); + } + ~state_history_plugin_impl() { } @@ -302,6 +323,7 @@ void state_history_plugin::set_program_options(options_description& cli, options cli.add_options()("delete-state-history", bpo::bool_switch()->default_value(false), "clear state history files"); options("trace-history", bpo::bool_switch()->default_value(false), "enable trace history"); options("chain-state-history", bpo::bool_switch()->default_value(false), "enable chain state history"); + options("finality-data-history", bpo::bool_switch()->default_value(false), "enable finality data history"); options("state-history-endpoint", bpo::value()->default_value("127.0.0.1:8080"), "the endpoint upon which to listen for incoming connections. Caution: only expose this port to " "your internal network."); @@ -393,6 +415,8 @@ void state_history_plugin_impl::plugin_initialize(const variables_map& options) trace_log.emplace("trace_history", state_history_dir , ship_log_conf); if (options.at("chain-state-history").as()) chain_state_log.emplace("chain_state_history", state_history_dir, ship_log_conf); + if (options.at("finality-data-history").as()) + finality_data_log.emplace("finality_data_history", state_history_dir, ship_log_conf); } FC_LOG_AND_RETHROW() } // state_history_plugin::plugin_initialize @@ -423,6 +447,11 @@ void state_history_plugin_impl::plugin_startup() { if( first_state_block > 0 ) first_available_block = std::min( first_available_block, first_state_block ); } + if (finality_data_log) { + auto first_state_block = finality_data_log->block_range().first; + if( first_state_block > 0 ) + first_available_block = std::min( first_available_block, first_state_block ); + } fc_ilog(_log, "First available block for SHiP ${b}", ("b", first_available_block)); listen(); // use of executor assumes only one thread @@ -465,4 +494,9 @@ const state_history_log* state_history_plugin::chain_state_log() const { return log ? std::addressof(*log) : nullptr; } +const state_history_log* state_history_plugin::finality_data_log() const { + const auto& log = my->get_finality_data_log(); + return log ? std::addressof(*log) : nullptr; +} + } // namespace eosio From 0bda7666e4a3e4b0e13e6c823d97bfc028b4e59a Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Sun, 17 Mar 2024 18:09:36 -0400 Subject: [PATCH 04/18] add new finality_data tests to state_history_plugin test --- .../tests/session_test.cpp | 119 +++++++++++------- 1 file changed, 75 insertions(+), 44 deletions(-) diff --git a/plugins/state_history_plugin/tests/session_test.cpp b/plugins/state_history_plugin/tests/session_test.cpp index bcfb2a219d..83629de10d 100644 --- a/plugins/state_history_plugin/tests/session_test.cpp +++ b/plugins/state_history_plugin/tests/session_test.cpp @@ -73,6 +73,7 @@ fc::datastream& operator>>(fc::datastream& ds, eosio::state_history::get unpack_big_bytes(ds, obj.block); unpack_big_bytes(ds, obj.traces); unpack_big_bytes(ds, obj.deltas); + unpack_big_bytes(ds, obj.finality_data); return ds; } } // namespace eosio::state_history @@ -103,6 +104,7 @@ struct mock_state_history_plugin { fc::temp_directory log_dir; std::optional trace_log; std::optional state_log; + std::optional finality_data_log; std::atomic stopping = false; eosio::session_manager session_mgr{ship_ioc}; @@ -110,6 +112,7 @@ struct mock_state_history_plugin { std::optional& get_trace_log() { return trace_log; } std::optional& get_chain_state_log() { return state_log; } + std::optional& get_finality_data_log() { return finality_data_log; } fc::sha256 get_chain_id() const { return {}; } boost::asio::io_context& get_ship_executor() { return ship_ioc; } @@ -117,6 +120,7 @@ struct mock_state_history_plugin { void setup_state_history_log(eosio::state_history_log_config conf = {}) { trace_log.emplace("ship_trace", log_dir.path(), conf); state_log.emplace("ship_state", log_dir.path(), conf); + finality_data_log.emplace("ship_finality_data", log_dir.path(), conf); } fc::logger logger = fc::logger::get(DEFAULT_LOGGER); @@ -144,6 +148,11 @@ struct mock_state_history_plugin { if( id ) return id; } + if( finality_data_log ) { + id = finality_data_log->get_block_id( block_num ); + if( id ) + return id; + } return block_id_for(block_num); } @@ -299,24 +308,21 @@ struct state_history_test_fixture { header.payload_size += sizeof(uint64_t); } - std::unique_lock gt(server.trace_log->_mx); - server.trace_log->write_entry(header, block_id_for(index - 1), [&](auto& f) { - f.write((const char*)&type, sizeof(type)); - if (type == 1) { - f.write((const char*)&decompressed_byte_count, sizeof(decompressed_byte_count)); - } - f.write(compressed.data(), compressed.size()); - }); - gt.unlock(); - std::unique_lock gs(server.state_log->_mx); - server.state_log->write_entry(header, block_id_for(index - 1), [&](auto& f) { - f.write((const char*)&type, sizeof(type)); - if (type == 1) { - f.write((const char*)&decompressed_byte_count, sizeof(decompressed_byte_count)); - } - f.write(compressed.data(), compressed.size()); - }); - gs.unlock(); + auto write_log = [&](std::optional& log) { + std::unique_lock lk(log->_mx); + log->write_entry(header, block_id_for(index - 1), [&](auto& f) { + f.write((const char*)&type, sizeof(type)); + if (type == 1) { + f.write((const char*)&decompressed_byte_count, sizeof(decompressed_byte_count)); + } + f.write(compressed.data(), compressed.size()); + }); + lk.unlock(); + }; + + write_log(server.trace_log); + write_log(server.state_log); + write_log(server.finality_data_log); if (written_data.size() < index) written_data.resize(index); @@ -330,7 +336,6 @@ void store_read_test_case(uint64_t data_size, eosio::state_history_log_config co fc::temp_directory log_dir; eosio::state_history_log log("ship", log_dir.path(), config); - eosio::state_history_log_header header; header.block_id = block_id_for(1); header.payload_size = 0; @@ -406,7 +411,7 @@ BOOST_FIXTURE_TEST_CASE(test_session_no_prune, state_history_test_fixture) { uint32_t head_block_num = 3; server.block_head = {head_block_num, block_id_for(head_block_num)}; - // generate the log data used for traces and deltas + // generate the log data used for traces, deltas and finality_data uint32_t n = mock_state_history_plugin::default_frame_size; add_to_log(1, n * sizeof(uint32_t), generate_data(n)); // original data format add_to_log(2, 0, generate_data(n)); // format to accommodate the compressed size greater than 4GB @@ -429,7 +434,8 @@ BOOST_FIXTURE_TEST_CASE(test_session_no_prune, state_history_test_fixture) { .irreversible_only = false, .fetch_block = true, .fetch_traces = true, - .fetch_deltas = true}); + .fetch_deltas = true, + .fetch_finality_data = true}); eosio::state_history::state_result result; // we should get 3 consecutive block result @@ -440,15 +446,19 @@ BOOST_FIXTURE_TEST_CASE(test_session_no_prune, state_history_test_fixture) { BOOST_REQUIRE_EQUAL(r.head.block_num, server.block_head.block_num); BOOST_REQUIRE(r.traces.has_value()); BOOST_REQUIRE(r.deltas.has_value()); - auto traces = r.traces.value(); - auto deltas = r.deltas.value(); + BOOST_REQUIRE(r.finality_data.has_value()); + auto traces = r.traces.value(); + auto deltas = r.deltas.value(); + auto finality_data = r.finality_data.value(); auto& data = written_data[i]; auto data_size = data.size() * sizeof(int32_t); BOOST_REQUIRE_EQUAL(traces.size(), data_size); BOOST_REQUIRE_EQUAL(deltas.size(), data_size); + BOOST_REQUIRE_EQUAL(finality_data.size(), data_size); BOOST_REQUIRE(std::equal(traces.begin(), traces.end(), (const char*)data.data())); BOOST_REQUIRE(std::equal(deltas.begin(), deltas.end(), (const char*)data.data())); + BOOST_REQUIRE(std::equal(finality_data.begin(), finality_data.end(), (const char*)data.data())); } } FC_LOG_AND_RETHROW() @@ -464,7 +474,7 @@ BOOST_FIXTURE_TEST_CASE(test_split_log, state_history_test_fixture) { uint32_t head_block_num = head; server.block_head = {head_block_num, block_id_for(head_block_num)}; - // generate the log data used for traces and deltas + // generate the log data used for traces, deltas and finality_data uint32_t n = mock_state_history_plugin::default_frame_size; add_to_log(1, n * sizeof(uint32_t), generate_data(n)); // original data format add_to_log(2, 0, generate_data(n)); // format to accommodate the compressed size greater than 4GB @@ -480,7 +490,8 @@ BOOST_FIXTURE_TEST_CASE(test_split_log, state_history_test_fixture) { .irreversible_only = false, .fetch_block = true, .fetch_traces = true, - .fetch_deltas = true}); + .fetch_deltas = true, + .fetch_finality_data = true}); eosio::state_history::state_result result; // we should get 1023 consecutive block result @@ -496,15 +507,19 @@ BOOST_FIXTURE_TEST_CASE(test_split_log, state_history_test_fixture) { prev_id = r.this_block->block_id; BOOST_REQUIRE(r.traces.has_value()); BOOST_REQUIRE(r.deltas.has_value()); - auto traces = r.traces.value(); - auto deltas = r.deltas.value(); + BOOST_REQUIRE(r.finality_data.has_value()); + auto traces = r.traces.value(); + auto deltas = r.deltas.value(); + auto finality_data = r.finality_data.value(); auto& data = written_data[i]; auto data_size = data.size() * sizeof(int32_t); BOOST_REQUIRE_EQUAL(traces.size(), data_size); BOOST_REQUIRE_EQUAL(deltas.size(), data_size); + BOOST_REQUIRE_EQUAL(finality_data.size(), data_size); BOOST_REQUIRE(std::equal(traces.begin(), traces.end(), (const char*)data.data())); BOOST_REQUIRE(std::equal(deltas.begin(), deltas.end(), (const char*)data.data())); + BOOST_REQUIRE(std::equal(finality_data.begin(), finality_data.end(), (const char*)data.data())); } } FC_LOG_AND_RETHROW() @@ -519,7 +534,7 @@ BOOST_FIXTURE_TEST_CASE(test_session_with_prune, state_history_test_fixture) { uint32_t head_block_num = 3; server.block_head = {head_block_num, block_id_for(head_block_num)}; - // generate the log data used for traces and deltas + // generate the log data used for traces, deltas and finality_data uint32_t n = mock_state_history_plugin::default_frame_size; add_to_log(1, n * sizeof(uint32_t), generate_data(n)); // original data format add_to_log(2, 0, generate_data(n)); // format to accommodate the compressed size greater than 4GB @@ -542,7 +557,8 @@ BOOST_FIXTURE_TEST_CASE(test_session_with_prune, state_history_test_fixture) { .irreversible_only = false, .fetch_block = true, .fetch_traces = true, - .fetch_deltas = true}); + .fetch_deltas = true, + .fetch_finality_data = true}); eosio::state_history::state_result result; // we should get 3 consecutive block result @@ -553,6 +569,7 @@ BOOST_FIXTURE_TEST_CASE(test_session_with_prune, state_history_test_fixture) { BOOST_REQUIRE_EQUAL(r.head.block_num, server.block_head.block_num); BOOST_REQUIRE(!r.traces.has_value()); BOOST_REQUIRE(!r.deltas.has_value()); + BOOST_REQUIRE(!r.finality_data.has_value()); for (int i = 1; i < 3; ++i) { receive_result(result); @@ -561,15 +578,19 @@ BOOST_FIXTURE_TEST_CASE(test_session_with_prune, state_history_test_fixture) { BOOST_REQUIRE_EQUAL(r.head.block_num, server.block_head.block_num); BOOST_REQUIRE(r.traces.has_value()); BOOST_REQUIRE(r.deltas.has_value()); - auto traces = r.traces.value(); - auto deltas = r.deltas.value(); - auto& data = written_data[i]; - auto data_size = data.size() * sizeof(int32_t); + BOOST_REQUIRE(r.finality_data.has_value()); + auto traces = r.traces.value(); + auto deltas = r.deltas.value(); + auto finality_data = r.finality_data.value(); + auto& data = written_data[i]; + auto data_size = data.size() * sizeof(int32_t); BOOST_REQUIRE_EQUAL(traces.size(), data_size); BOOST_REQUIRE_EQUAL(deltas.size(), data_size); + BOOST_REQUIRE_EQUAL(finality_data.size(), data_size); BOOST_REQUIRE(std::equal(traces.begin(), traces.end(), (const char*)data.data())); BOOST_REQUIRE(std::equal(deltas.begin(), deltas.end(), (const char*)data.data())); + BOOST_REQUIRE(std::equal(finality_data.begin(), finality_data.end(), (const char*)data.data())); } } FC_LOG_AND_RETHROW() @@ -582,7 +603,7 @@ BOOST_FIXTURE_TEST_CASE(test_session_fork, state_history_test_fixture) { uint32_t head_block_num = 4; server.block_head = {head_block_num, block_id_for(head_block_num)}; - // generate the log data used for traces and deltas + // generate the log data used for traces, deltas and finality_data uint32_t n = mock_state_history_plugin::default_frame_size; add_to_log(1, n * sizeof(uint32_t), generate_data(n)); // original data format add_to_log(2, 0, generate_data(n)); // format to accommodate the compressed size greater than 4GB @@ -607,7 +628,8 @@ BOOST_FIXTURE_TEST_CASE(test_session_fork, state_history_test_fixture) { .irreversible_only = false, .fetch_block = true, .fetch_traces = true, - .fetch_deltas = true}); + .fetch_deltas = true, + .fetch_finality_data = true}); std::vector have_positions; eosio::state_history::state_result result; @@ -619,18 +641,22 @@ BOOST_FIXTURE_TEST_CASE(test_session_fork, state_history_test_fixture) { BOOST_REQUIRE_EQUAL(r.head.block_num, server.block_head.block_num); BOOST_REQUIRE(r.traces.has_value()); BOOST_REQUIRE(r.deltas.has_value()); - auto traces = r.traces.value(); - auto deltas = r.deltas.value(); - auto& data = written_data[i]; - auto data_size = data.size() * sizeof(int32_t); + BOOST_REQUIRE(r.finality_data.has_value()); + auto traces = r.traces.value(); + auto deltas = r.deltas.value(); + auto finality_data = r.finality_data.value(); + auto& data = written_data[i]; + auto data_size = data.size() * sizeof(int32_t); BOOST_REQUIRE_EQUAL(traces.size(), data_size); BOOST_REQUIRE_EQUAL(deltas.size(), data_size); + BOOST_REQUIRE_EQUAL(finality_data.size(), data_size); BOOST_REQUIRE(r.this_block.has_value()); BOOST_REQUIRE_EQUAL(r.this_block->block_num, i+1); have_positions.push_back(*r.this_block); BOOST_REQUIRE(std::equal(traces.begin(), traces.end(), (const char*)data.data())); BOOST_REQUIRE(std::equal(deltas.begin(), deltas.end(), (const char*)data.data())); + BOOST_REQUIRE(std::equal(finality_data.begin(), finality_data.end(), (const char*)data.data())); } // generate a fork that includes blocks 3,4 and verify new data retrieved @@ -658,7 +684,8 @@ BOOST_FIXTURE_TEST_CASE(test_session_fork, state_history_test_fixture) { .irreversible_only = false, .fetch_block = true, .fetch_traces = true, - .fetch_deltas = true}); + .fetch_deltas = true, + .fetch_finality_data = true}); eosio::state_history::state_result fork_result; // we should now get data for fork 3,4 @@ -671,15 +698,19 @@ BOOST_FIXTURE_TEST_CASE(test_session_fork, state_history_test_fixture) { BOOST_REQUIRE_EQUAL(r.this_block->block_num, i+1); BOOST_REQUIRE(r.traces.has_value()); BOOST_REQUIRE(r.deltas.has_value()); - auto traces = r.traces.value(); - auto deltas = r.deltas.value(); - auto& data = written_data[i]; - auto data_size = data.size() * sizeof(int32_t); + BOOST_REQUIRE(r.finality_data.has_value()); + auto traces = r.traces.value(); + auto deltas = r.deltas.value(); + auto finality_data = r.finality_data.value(); + auto& data = written_data[i]; + auto data_size = data.size() * sizeof(int32_t); BOOST_REQUIRE_EQUAL(traces.size(), data_size); BOOST_REQUIRE_EQUAL(deltas.size(), data_size); + BOOST_REQUIRE_EQUAL(finality_data.size(), data_size); BOOST_REQUIRE(std::equal(traces.begin(), traces.end(), (const char*)data.data())); BOOST_REQUIRE(std::equal(deltas.begin(), deltas.end(), (const char*)data.data())); + BOOST_REQUIRE(std::equal(finality_data.begin(), finality_data.end(), (const char*)data.data())); } } FC_LOG_AND_RETHROW() From 54797c0365aa0c35ef9dae2daba3f3502622a25b Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Sun, 17 Mar 2024 18:10:26 -0400 Subject: [PATCH 05/18] update ship_streamer tests for finality_data --- tests/ship_streamer.cpp | 5 +++++ tests/ship_streamer_test.py | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/ship_streamer.cpp b/tests/ship_streamer.cpp index 94a3c40fc9..ff2843c55d 100644 --- a/tests/ship_streamer.cpp +++ b/tests/ship_streamer.cpp @@ -32,6 +32,7 @@ int main(int argc, char* argv[]) { bool fetch_block = false; bool fetch_traces = false; bool fetch_deltas = false; + bool fetch_finality_data = false; cli.add_options() ("help,h", bpo::bool_switch(&help)->default_value(false), "Print this help message and exit.") @@ -42,6 +43,7 @@ int main(int argc, char* argv[]) { ("fetch-block", bpo::bool_switch(&fetch_block)->default_value(fetch_block), "Fetch blocks") ("fetch-traces", bpo::bool_switch(&fetch_traces)->default_value(fetch_traces), "Fetch traces") ("fetch-deltas", bpo::bool_switch(&fetch_deltas)->default_value(fetch_deltas), "Fetch deltas") + ("fetch-finality-data", bpo::bool_switch(&fetch_finality_data)->default_value(fetch_finality_data), "Fetch finality data") ; bpo::variables_map varmap; bpo::store(bpo::parse_command_line(argc, argv, cli), varmap); @@ -85,6 +87,7 @@ int main(int argc, char* argv[]) { // bool fetch_block = false; // bool fetch_traces = false; // bool fetch_deltas = false; + // bool fetch_finality_data = false; //}; request_writer.StartArray(); request_writer.String("get_blocks_request_v0"); @@ -106,6 +109,8 @@ int main(int argc, char* argv[]) { request_writer.Bool(fetch_traces); request_writer.Key("fetch_deltas"); request_writer.Bool(fetch_deltas); + request_writer.Key("fetch_finality_data"); + request_writer.Bool(fetch_finality_data); request_writer.EndObject(); request_writer.EndArray(); diff --git a/tests/ship_streamer_test.py b/tests/ship_streamer_test.py index 26422c5943..49a8370cbf 100755 --- a/tests/ship_streamer_test.py +++ b/tests/ship_streamer_test.py @@ -71,7 +71,7 @@ def getLatestSnapshot(nodeId): shipNodeNum = 3 specificExtraNodeosArgs={} - specificExtraNodeosArgs[shipNodeNum]="--plugin eosio::state_history_plugin --trace-history --chain-state-history --state-history-stride 200 --plugin eosio::net_api_plugin --plugin eosio::producer_api_plugin " + specificExtraNodeosArgs[shipNodeNum]="--plugin eosio::state_history_plugin --trace-history --chain-state-history --finality-data-history --state-history-stride 200 --plugin eosio::net_api_plugin --plugin eosio::producer_api_plugin " # producer nodes will be mapped to 0 through totalProducerNodes-1, so the number totalProducerNodes will be the non-producing node specificExtraNodeosArgs[totalProducerNodes]="--plugin eosio::test_control_api_plugin " @@ -134,7 +134,7 @@ def getLatestSnapshot(nodeId): end_block_num = start_block_num + block_range shipClient = "tests/ship_streamer" - cmd = f"{shipClient} --start-block-num {start_block_num} --end-block-num {end_block_num} --fetch-block --fetch-traces --fetch-deltas" + cmd = f"{shipClient} --start-block-num {start_block_num} --end-block-num {end_block_num} --fetch-block --fetch-traces --fetch-deltas --fetch-finality-data" if Utils.Debug: Utils.Print(f"cmd: {cmd}") clients = [] files = [] @@ -234,7 +234,7 @@ def getLatestSnapshot(nodeId): start_block_num = afterSnapshotBlockNum block_range = 0 end_block_num = start_block_num + block_range - cmd = f"{shipClient} --start-block-num {start_block_num} --end-block-num {end_block_num} --fetch-block --fetch-traces --fetch-deltas" + cmd = f"{shipClient} --start-block-num {start_block_num} --end-block-num {end_block_num} --fetch-block --fetch-traces --fetch-deltas --fetch-finality-data" if Utils.Debug: Utils.Print(f"cmd: {cmd}") clients = [] files = [] From 31e41006abd218143b2b05f7834719646ee490b6 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Sun, 17 Mar 2024 19:01:40 -0400 Subject: [PATCH 06/18] rename get_chain_head_finality_data() to head_finality_data() --- libraries/chain/controller.cpp | 12 +++++------- libraries/chain/include/eosio/chain/controller.hpp | 3 ++- .../state_history_plugin/state_history_plugin.cpp | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index c34def353f..b291e9471c 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -4103,9 +4103,8 @@ struct controller_impl { } } - finality_data_t get_chain_head_finality_data(block_id_type block_id) const { + finality_data_t head_finality_data() const { return apply_s(chain_head, [&](const auto& head) { - assert(head->id() == block_id); return head->get_finality_data(); }); } @@ -4633,6 +4632,10 @@ const signed_block_ptr& controller::head_block()const { return my->chain_head.block(); } +finality_data_t controller::head_finality_data() const { + return my->head_finality_data(); +} + uint32_t controller::fork_db_head_block_num()const { return my->fork_db_head_block_num(); } @@ -4676,10 +4679,6 @@ uint32_t controller::if_irreversible_block_num() const { return block_header::num_from_id(my->if_irreversible_block_id); } -finality_data_t controller::get_chain_head_finality_data(block_id_type block_id) const { - return my->get_chain_head_finality_data(block_id); -} - uint32_t controller::last_irreversible_block_num() const { return my->fork_db_root_block_num(); } @@ -4692,7 +4691,6 @@ time_point controller::last_irreversible_block_time() const { return my->fork_db_root_timestamp().to_time_point(); } - const dynamic_global_property_object& controller::get_dynamic_global_properties()const { return my->db.get(); } diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index 2e1771940a..b12056eb2a 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -240,6 +240,8 @@ namespace eosio::chain { const signed_block_ptr& head_block()const; // returns nullptr after instant finality enabled block_state_legacy_ptr head_block_state_legacy()const; + // returns finality_data associated with chain head for SHiP + finality_data_t head_finality_data() const; uint32_t fork_db_head_block_num()const; block_id_type fork_db_head_block_id()const; @@ -263,7 +265,6 @@ namespace eosio::chain { void set_if_irreversible_block_id(const block_id_type& id); uint32_t if_irreversible_block_num() const; - finality_data_t get_chain_head_finality_data(block_id_type block_id) const; uint32_t last_irreversible_block_num() const; block_id_type last_irreversible_block_id() const; diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index e80b983463..c84bdb48a0 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -287,7 +287,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thispack_and_write_entry(header, block->previous, [this, id](auto&& buf) { fc::datastream ds{buf}; - fc::raw::pack(ds, chain_plug->chain().get_chain_head_finality_data(id)); + fc::raw::pack(ds, chain_plug->chain().head_finality_data()); }); } From 3db6e06c64570d4856d3b545c789fc68a3f0558e Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Sun, 17 Mar 2024 22:33:53 -0400 Subject: [PATCH 07/18] cache action_mroot when producing blocks --- libraries/chain/block_state.cpp | 7 ++++--- libraries/chain/controller.cpp | 6 ++++-- libraries/chain/include/eosio/chain/block_state.hpp | 5 +++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/libraries/chain/block_state.cpp b/libraries/chain/block_state.cpp index fd10897d3b..297abc0529 100644 --- a/libraries/chain/block_state.cpp +++ b/libraries/chain/block_state.cpp @@ -31,7 +31,8 @@ block_state::block_state(const block_header_state& bhs, const std::optional& valid, const std::optional& qc, const signer_callback_type& signer, - const block_signing_authority& valid_block_signing_authority) + const block_signing_authority& valid_block_signing_authority, + const digest_type& action_mroot) : block_header_state(bhs) , block(std::make_shared(signed_block_header{bhs.header})) , strong_digest(compute_finality_digest()) @@ -40,6 +41,7 @@ block_state::block_state(const block_header_state& bhs, , valid(valid) , pub_keys_recovered(true) // called by produce_block so signature recovery of trxs must have been done , cached_trxs(std::move(trx_metas)) + , action_mroot(action_mroot) { block->transactions = std::move(trx_receipts); @@ -298,10 +300,9 @@ digest_type block_state::get_finality_mroot_claim(const qc_claim_t& qc_claim) co } finality_data_t block_state::get_finality_data() const { - assert(action_mroot.has_value()); finality_data_t finality_data { // other fields take the default values set by finality_data_t definition - .action_mroot = *action_mroot, + .action_mroot = action_mroot, .base_digest = compute_base_digest() // from block_header_state }; diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 083926818e..975f0106e7 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -241,6 +241,7 @@ struct assembled_block { deque trx_receipts; // Comes from building_block::pending_trx_receipts std::optional valid; // Comes from assemble_block std::optional qc; // QC to add as block extension to new block + digest_type action_mroot; block_header_state& get_bhs() { return bhs; } }; @@ -370,7 +371,7 @@ struct assembled_block { [&](assembled_block_if& ab) { auto bsp = std::make_shared(ab.bhs, std::move(ab.trx_metas), std::move(ab.trx_receipts), ab.valid, ab.qc, signer, - valid_block_signing_authority); + valid_block_signing_authority, ab.action_mroot); return completed_block{block_handle{std::move(bsp)}, {}}; }}, v); @@ -766,7 +767,8 @@ struct building_block { std::move(bb.pending_trx_metas), std::move(bb.pending_trx_receipts), valid, - qc_data.qc + qc_data.qc, + action_mroot // caching for constructing finality_data. }; return assembled_block{.v = std::move(ab)}; diff --git a/libraries/chain/include/eosio/chain/block_state.hpp b/libraries/chain/include/eosio/chain/block_state.hpp index d3b29166db..0e42b448fb 100644 --- a/libraries/chain/include/eosio/chain/block_state.hpp +++ b/libraries/chain/include/eosio/chain/block_state.hpp @@ -78,7 +78,7 @@ struct block_state : public block_header_state { // block_header_state provi // ------ data members caching information available elsewhere ---------------------- bool pub_keys_recovered = false; deque cached_trxs; - std::optional action_mroot{std::nullopt}; + digest_type action_mroot; // ------ private methods ----------------------------------------------------------- bool is_valid() const { return validated; } @@ -142,7 +142,8 @@ struct block_state : public block_header_state { // block_header_state provi const std::optional& valid, const std::optional& qc, const signer_callback_type& signer, - const block_signing_authority& valid_block_signing_authority); + const block_signing_authority& valid_block_signing_authority, + const digest_type& action_mroot); block_state(const block_state_legacy& bsp, const digest_type& action_mroot_svnn); From 55717d12b8cbe65d92a6d29935bf3ec0eac86c74 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Mon, 18 Mar 2024 11:55:32 -0400 Subject: [PATCH 08/18] make head_finality_data() return std::optional --- libraries/chain/controller.cpp | 10 +++++----- libraries/chain/include/eosio/chain/controller.hpp | 5 +++-- plugins/state_history_plugin/state_history_plugin.cpp | 8 ++++++-- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 975f0106e7..914f149965 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -4110,10 +4110,10 @@ struct controller_impl { } } - finality_data_t head_finality_data() const { - return apply_s(chain_head, [&](const auto& head) { - return head->get_finality_data(); - }); + std::optional head_finality_data() const { + return apply>(chain_head, + overloaded{ [](const block_state_legacy_ptr& head) { return std::nullopt; }, + [](const block_state_ptr& head) { return head->get_finality_data(); }}); } uint32_t earliest_available_block_num() const { @@ -4639,7 +4639,7 @@ const signed_block_ptr& controller::head_block()const { return my->chain_head.block(); } -finality_data_t controller::head_finality_data() const { +std::optional controller::head_finality_data() const { return my->head_finality_data(); } diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index b12056eb2a..c1bc7a8d1b 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -240,8 +240,9 @@ namespace eosio::chain { const signed_block_ptr& head_block()const; // returns nullptr after instant finality enabled block_state_legacy_ptr head_block_state_legacy()const; - // returns finality_data associated with chain head for SHiP - finality_data_t head_finality_data() const; + // returns finality_data associated with chain head for SHiP when in Savanna, + // std::nullopt in Legacy + std::optional head_finality_data() const; uint32_t fork_db_head_block_num()const; block_id_type fork_db_head_block_id()const; diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index c84bdb48a0..6a167f8d0c 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -283,11 +283,15 @@ struct state_history_plugin_impl : std::enable_shared_from_this finality_data = chain_plug->chain().head_finality_data(); + if (!finality_data) + return; + state_history_log_header header{ .magic = ship_magic(ship_current_version, 0), .block_id = id, .payload_size = 0}; - finality_data_log->pack_and_write_entry(header, block->previous, [this, id](auto&& buf) { + finality_data_log->pack_and_write_entry(header, block->previous, [this, finality_data](auto&& buf) { fc::datastream ds{buf}; - fc::raw::pack(ds, chain_plug->chain().head_finality_data()); + fc::raw::pack(ds, *finality_data); }); } From a3a0a6f1a14cbd2e6c69dcd2375ad18ae86a39fa Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Mon, 18 Mar 2024 13:23:14 -0400 Subject: [PATCH 09/18] small refactoring in state_history_plugin.cpp --- .../state_history_plugin/state_history_plugin.cpp | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 6a167f8d0c..797031890f 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -117,20 +117,16 @@ struct state_history_plugin_impl : std::enable_shared_from_this get_block_id(uint32_t block_num) { - std::optional id; if( trace_log ) { - id = trace_log->get_block_id( block_num ); - if( id ) + if ( auto id = trace_log->get_block_id( block_num ); id ) return id; } if( chain_state_log ) { - id = chain_state_log->get_block_id( block_num ); - if( id ) + if( auto id = chain_state_log->get_block_id( block_num ); id ) return id; } if( finality_data_log ) { - id = finality_data_log->get_block_id( block_num ); - if( id ) + if( auto id = finality_data_log->get_block_id( block_num ); id ) return id; } try { @@ -284,12 +280,12 @@ struct state_history_plugin_impl : std::enable_shared_from_this finality_data = chain_plug->chain().head_finality_data(); - if (!finality_data) + if (!finality_data.has_value()) return; state_history_log_header header{ .magic = ship_magic(ship_current_version, 0), .block_id = id, .payload_size = 0}; - finality_data_log->pack_and_write_entry(header, block->previous, [this, finality_data](auto&& buf) { + finality_data_log->pack_and_write_entry(header, block->previous, [finality_data](auto&& buf) { fc::datastream ds{buf}; fc::raw::pack(ds, *finality_data); }); From bbe87357892d5f42cd9826a17e1ab5d8608f7239 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Wed, 20 Mar 2024 14:03:11 -0400 Subject: [PATCH 10/18] Introduce get_blocks_request_v1 to support fetch_finality_data parameter; add tests for previous SHiP clients interacting with new format --- libraries/state_history/abi.cpp | 14 +- .../include/eosio/state_history/types.hpp | 11 +- .../eosio/state_history_plugin/session.hpp | 105 ++- .../tests/session_test.cpp | 647 ++++++++++-------- tests/CMakeLists.txt | 2 + tests/ship_streamer.cpp | 23 +- tests/ship_streamer_test.py | 17 +- 7 files changed, 490 insertions(+), 329 deletions(-) diff --git a/libraries/state_history/abi.cpp b/libraries/state_history/abi.cpp index 649ca3a7a2..20e5118dd8 100644 --- a/libraries/state_history/abi.cpp +++ b/libraries/state_history/abi.cpp @@ -23,6 +23,18 @@ extern const char* const state_history_plugin_abi = R"({ }, { "name": "get_blocks_request_v0", "fields": [ + { "name": "start_block_num", "type": "uint32" }, + { "name": "end_block_num", "type": "uint32" }, + { "name": "max_messages_in_flight", "type": "uint32" }, + { "name": "have_positions", "type": "block_position[]" }, + { "name": "irreversible_only", "type": "bool" }, + { "name": "fetch_block", "type": "bool" }, + { "name": "fetch_traces", "type": "bool" }, + { "name": "fetch_deltas", "type": "bool" } + ] + }, + { + "name": "get_blocks_request_v1", "fields": [ { "name": "start_block_num", "type": "uint32" }, { "name": "end_block_num", "type": "uint32" }, { "name": "max_messages_in_flight", "type": "uint32" }, @@ -554,7 +566,7 @@ extern const char* const state_history_plugin_abi = R"({ { "new_type_name": "transaction_id", "type": "checksum256" } ], "variants": [ - { "name": "request", "types": ["get_status_request_v0", "get_blocks_request_v0", "get_blocks_ack_request_v0"] }, + { "name": "request", "types": ["get_status_request_v0", "get_blocks_request_v0", "get_blocks_request_v1", "get_blocks_ack_request_v0"] }, { "name": "result", "types": ["get_status_result_v0", "get_blocks_result_v0"] }, { "name": "action_receipt", "types": ["action_receipt_v0"] }, diff --git a/libraries/state_history/include/eosio/state_history/types.hpp b/libraries/state_history/include/eosio/state_history/types.hpp index 990fb8fdd4..338b152d3e 100644 --- a/libraries/state_history/include/eosio/state_history/types.hpp +++ b/libraries/state_history/include/eosio/state_history/types.hpp @@ -102,7 +102,10 @@ struct get_blocks_request_v0 { bool fetch_block = false; bool fetch_traces = false; bool fetch_deltas = false; - bool fetch_finality_data = false; +}; + +struct get_blocks_request_v1 : get_blocks_request_v0 { + bool fetch_finality_data = false;; }; struct get_blocks_ack_request_v0 { @@ -123,7 +126,8 @@ struct get_blocks_result_v0 : get_blocks_result_base { std::optional finality_data; }; -using state_request = std::variant; +using state_request = std::variant; +using get_blocks_request = std::variant; using state_result = std::variant; } // namespace state_history @@ -134,7 +138,8 @@ FC_REFLECT(eosio::state_history::table_delta, (struct_version)(name)(rows)); FC_REFLECT(eosio::state_history::block_position, (block_num)(block_id)); FC_REFLECT_EMPTY(eosio::state_history::get_status_request_v0); FC_REFLECT(eosio::state_history::get_status_result_v0, (head)(last_irreversible)(trace_begin_block)(trace_end_block)(chain_state_begin_block)(chain_state_end_block)(chain_id)); -FC_REFLECT(eosio::state_history::get_blocks_request_v0, (start_block_num)(end_block_num)(max_messages_in_flight)(have_positions)(irreversible_only)(fetch_block)(fetch_traces)(fetch_deltas)(fetch_finality_data)); +FC_REFLECT(eosio::state_history::get_blocks_request_v0, (start_block_num)(end_block_num)(max_messages_in_flight)(have_positions)(irreversible_only)(fetch_block)(fetch_traces)(fetch_deltas)); +FC_REFLECT_DERIVED(eosio::state_history::get_blocks_request_v1, (eosio::state_history::get_blocks_request_v0), (fetch_finality_data)); FC_REFLECT(eosio::state_history::get_blocks_ack_request_v0, (num_messages)); FC_REFLECT(eosio::state_history::get_blocks_result_base, (head)(last_irreversible)(this_block)(prev_block)(block)); FC_REFLECT_DERIVED(eosio::state_history::get_blocks_result_v0, (eosio::state_history::get_blocks_result_base), (traces)(deltas)(finality_data)); diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/session.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/session.hpp index d6ac250628..5f8d45f97f 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/session.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/session.hpp @@ -28,7 +28,7 @@ struct session_base { virtual void send_update(const chain::signed_block_ptr& block, const chain::block_id_type& id) = 0; virtual ~session_base() = default; - std::optional current_request; + std::optional current_request; bool need_to_send_update = false; }; @@ -165,7 +165,14 @@ class blocks_ack_request_send_queue_entry : public send_queue_entry_base { , req(std::move(r)) {} void send_entry() override { - session->current_request->max_messages_in_flight += req.num_messages; + assert(session->current_request); + assert(std::holds_alternative(*session->current_request) || + std::holds_alternative(*session->current_request)); + + std::visit(chain::overloaded{ + [&](eosio::state_history::get_blocks_request_v0& request) { request.max_messages_in_flight += req.num_messages;}, + [&](eosio::state_history::get_blocks_request_v1& request) { request.max_messages_in_flight += req.num_messages;} }, + *session->current_request); session->send_update(false); } }; @@ -173,10 +180,10 @@ class blocks_ack_request_send_queue_entry : public send_queue_entry_base { template class blocks_request_send_queue_entry : public send_queue_entry_base { std::shared_ptr session; - eosio::state_history::get_blocks_request_v0 req; + eosio::state_history::get_blocks_request req; public: - blocks_request_send_queue_entry(std::shared_ptr s, state_history::get_blocks_request_v0&& r) + blocks_request_send_queue_entry(std::shared_ptr s, state_history::get_blocks_request&& r) : session(std::move(s)) , req(std::move(r)) {} @@ -433,6 +440,14 @@ struct session : session_base, std::enable_shared_from_thisshared_from_this(); + auto entry_ptr = std::make_unique>(self, std::move(req)); + session_mgr.add_send_queue(std::move(self), std::move(entry_ptr)); + } + void process(state_history::get_blocks_ack_request_v0& req) { fc_dlog(plugin.get_logger(), "received get_blocks_ack_request_v0 = ${req}", ("req", req)); if (!current_request) { @@ -468,7 +483,7 @@ struct session : session_base, std::enable_shared_from_this(req) || + std::holds_alternative(req)); - current_request = std::move(req); + std::visit(chain::overloaded{ + [&](state_history::get_blocks_request_v0& request) { + update_current_request_impl(request); + current_request = std::move(req);}, + [&](state_history::get_blocks_request_v1& request) { + update_current_request_impl(request); + fc_dlog(plugin.get_logger(), "replying get_blocks_request_v1, fetch_finality_data = ${fetch_finality_data}", ("fetch_finality_data", request.fetch_finality_data)); + current_request = std::move(req);} }, + req); } - void send_update(state_history::get_blocks_result_v0 result, const chain::signed_block_ptr& block, const chain::block_id_type& id) { + void send_update(state_history::get_blocks_request_v0& request, bool fetch_finality_data, state_history::get_blocks_result_v0 result, const chain::signed_block_ptr& block, const chain::block_id_type& id) { need_to_send_update = true; - if (!current_request || !current_request->max_messages_in_flight) { - session_mgr.pop_entry(false); - return; - } result.last_irreversible = plugin.get_last_irreversible(); uint32_t current = - current_request->irreversible_only ? result.last_irreversible.block_num : result.head.block_num; + request.irreversible_only ? result.last_irreversible.block_num : result.head.block_num; - if (to_send_block_num > current || to_send_block_num >= current_request->end_block_num) { - fc_dlog( plugin.get_logger(), "Not sending, to_send_block_num: ${s}, current: ${c} current_request.end_block_num: ${b}", - ("s", to_send_block_num)("c", current)("b", current_request->end_block_num) ); + fc_dlog( plugin.get_logger(), "irreversible_only: ${i}, last_irreversible: ${p}, head.block_num: ${h}", ("i", request.irreversible_only)("p", result.last_irreversible.block_num)("h", result.head.block_num)); + fc_dlog( plugin.get_logger(), "recved result: ${r}", ("r", result)); + if (to_send_block_num > current || to_send_block_num >= request.end_block_num) { + fc_dlog( plugin.get_logger(), "Not sending, to_send_block_num: ${s}, current: ${c} request.end_block_num: ${b}", + ("s", to_send_block_num)("c", current)("b", request.end_block_num) ); session_mgr.pop_entry(false); return; } @@ -526,7 +552,7 @@ struct session : session_base, std::enable_shared_from_thisblock_id; ++itr; - if (itr == current_request->have_positions.end()) + if (itr == request.have_positions.end()) position_it.reset(); if(block_id_seen_by_client == *block_id) { @@ -541,15 +567,15 @@ struct session : session_base, std::enable_shared_from_thisfetch_block) { + if (request.fetch_block) { uint32_t block_num = block ? block->block_num() : 0; // block can be nullptr in testing plugin.get_block(to_send_block_num, block_num, block, result.block); } - if (current_request->fetch_traces && plugin.get_trace_log()) + if (request.fetch_traces && plugin.get_trace_log()) result.traces.emplace(); - if (current_request->fetch_deltas && plugin.get_chain_state_log()) + if (request.fetch_deltas && plugin.get_chain_state_log()) result.deltas.emplace(); - if (current_request->fetch_finality_data && plugin.get_finality_data_log()) { + if (fetch_finality_data && plugin.get_finality_data_log()) { result.finality_data.emplace(); // create finality_data (it's an optional field) } } @@ -567,15 +593,48 @@ struct session : session_base, std::enable_shared_from_thisblock_id} : fc::variant{})); } - --current_request->max_messages_in_flight; + --request.max_messages_in_flight; need_to_send_update = to_send_block_num <= current && - to_send_block_num < current_request->end_block_num; + to_send_block_num < request.end_block_num; std::make_shared>(this->shared_from_this(), std::move(result))->send_entry(); } + bool no_request_or_not_max_messages_in_flight() { + if (!current_request) + return true; + + uint32_t max_messages_in_flight = std::visit( + chain::overloaded{ + [&](state_history::get_blocks_request_v0& request) -> uint32_t { + return request.max_messages_in_flight; }, + [&](state_history::get_blocks_request_v1& request) -> uint32_t { + return request.max_messages_in_flight; }}, + *current_request); + + return !max_messages_in_flight; + } + + void send_update(state_history::get_blocks_result_v0 result, const chain::signed_block_ptr& block, const chain::block_id_type& id) { + if (no_request_or_not_max_messages_in_flight()) { + session_mgr.pop_entry(false); + return; + } + + assert(current_request); + assert(std::holds_alternative(*current_request) || + std::holds_alternative(*current_request)); + + std::visit(eosio::chain::overloaded{ + [&](eosio::state_history::get_blocks_request_v0& request) { + send_update(request, false, result, block, id); }, + [&](eosio::state_history::get_blocks_request_v1& request) { + send_update(request, request.fetch_finality_data, result, block, id); } }, + *current_request); + } + void send_update(const chain::signed_block_ptr& block, const chain::block_id_type& id) override { - if (!current_request || !current_request->max_messages_in_flight) { + if (no_request_or_not_max_messages_in_flight()) { session_mgr.pop_entry(false); return; } diff --git a/plugins/state_history_plugin/tests/session_test.cpp b/plugins/state_history_plugin/tests/session_test.cpp index 83629de10d..0a35bbaa55 100644 --- a/plugins/state_history_plugin/tests/session_test.cpp +++ b/plugins/state_history_plugin/tests/session_test.cpp @@ -30,6 +30,8 @@ namespace net = boost::asio; // from namespace bio = boost::iostreams; using tcp = boost::asio::ip::tcp; // from +using namespace eosio::state_history; + namespace eosio::state_history { template @@ -80,15 +82,6 @@ fc::datastream& operator>>(fc::datastream& ds, eosio::state_history::get //------------------------------------------------------------------------------ -std::unordered_map block_ids; -fc::sha256 block_id_for(const uint32_t bnum, const std::string& nonce = {}) { - if (auto it = block_ids.find(bnum); it != block_ids.end()) - return it->second; - fc::sha256 m = fc::sha256::hash(fc::sha256::hash(std::to_string(bnum)+nonce)); - m._hash[0] = fc::endian_reverse_u32(bnum); - block_ids[bnum] = m; - return m; -} // Report a failure void fail(beast::error_code ec, char const* what) { std::cerr << what << ": " << ec.message() << "\n"; } @@ -107,6 +100,7 @@ struct mock_state_history_plugin { std::optional finality_data_log; std::atomic stopping = false; eosio::session_manager session_mgr{ship_ioc}; + std::unordered_map block_ids; constexpr static uint32_t default_frame_size = 1024; @@ -117,10 +111,12 @@ struct mock_state_history_plugin { boost::asio::io_context& get_ship_executor() { return ship_ioc; } - void setup_state_history_log(eosio::state_history_log_config conf = {}) { + void setup_state_history_log(bool fetch_finality_data, eosio::state_history_log_config conf = {}) { trace_log.emplace("ship_trace", log_dir.path(), conf); state_log.emplace("ship_state", log_dir.path(), conf); - finality_data_log.emplace("ship_finality_data", log_dir.path(), conf); + if( fetch_finality_data ) { + finality_data_log.emplace("ship_finality_data", log_dir.path(), conf); + } } fc::logger logger = fc::logger::get(DEFAULT_LOGGER); @@ -136,6 +132,15 @@ struct mock_state_history_plugin { return fc::time_point{}; } + fc::sha256 block_id_for(const uint32_t bnum, const std::string& nonce = {}) { + if (auto it = block_ids.find(bnum); it != block_ids.end()) + return it->second; + fc::sha256 m = fc::sha256::hash(fc::sha256::hash(std::to_string(bnum)+nonce)); + m._hash[0] = fc::endian_reverse_u32(bnum); + block_ids[bnum] = m; + return m; + } + std::optional get_block_id(uint32_t block_num) { std::optional id; if( trace_log ) { @@ -296,21 +301,22 @@ struct state_history_test_fixture { BOOST_CHECK(fc::raw::pack(status) == fc::raw::pack(received_status)); } - void add_to_log(uint32_t index, uint32_t type, std::vector&& decompressed_data) { + void add_to_log(uint32_t index, uint32_t type, std::vector&& decompressed_data, bool fetch_finality_data) { uint64_t decompressed_byte_count = decompressed_data.size() * sizeof(int32_t); auto compressed = zlib_compress((const char*)decompressed_data.data(), decompressed_byte_count); eosio::state_history_log_header header; - header.block_id = block_id_for(index); + header.block_id = server.block_id_for(index); header.payload_size = compressed.size() + sizeof(type); if (type == 1) { header.payload_size += sizeof(uint64_t); } auto write_log = [&](std::optional& log) { + assert(log); std::unique_lock lk(log->_mx); - log->write_entry(header, block_id_for(index - 1), [&](auto& f) { + log->write_entry(header, server.block_id_for(index - 1), [&](auto& f) { f.write((const char*)&type, sizeof(type)); if (type == 1) { f.write((const char*)&decompressed_byte_count, sizeof(decompressed_byte_count)); @@ -322,7 +328,9 @@ struct state_history_test_fixture { write_log(server.trace_log); write_log(server.state_log); - write_log(server.finality_data_log); + if( fetch_finality_data ) { + write_log(server.finality_data_log); + } if (written_data.size() < index) written_data.resize(index); @@ -332,16 +340,16 @@ struct state_history_test_fixture { ~state_history_test_fixture() { ws.close(websocket::close_code::normal); } }; -void store_read_test_case(uint64_t data_size, eosio::state_history_log_config config) { +void store_read_test_case(test_server& server, uint64_t data_size, eosio::state_history_log_config config) { fc::temp_directory log_dir; eosio::state_history_log log("ship", log_dir.path(), config); eosio::state_history_log_header header; - header.block_id = block_id_for(1); + header.block_id = server.block_id_for(1); header.payload_size = 0; auto data = generate_data(data_size); - log.pack_and_write_entry(header, block_id_for(0), + log.pack_and_write_entry(header, server.block_id_for(0), [&](auto&& buf) { bio::write(buf, (const char*)data.data(), data.size() * sizeof(data[0])); }); // make sure the current file position is at the end of file @@ -349,7 +357,6 @@ void store_read_test_case(uint64_t data_size, eosio::state_history_log_config co log.get_log_file().seek_end(0); BOOST_REQUIRE_EQUAL(log.get_log_file().tellp(), pos); - eosio::locked_decompress_stream buf = log.create_locked_decompress_stream(); log.get_unpacked_entry(1, buf); @@ -363,40 +370,40 @@ void store_read_test_case(uint64_t data_size, eosio::state_history_log_config co BOOST_CHECK(std::equal(decompressed.begin(), decompressed.end(), (const char*)data.data())); } -BOOST_AUTO_TEST_CASE(store_read_entry_no_prune) { - store_read_test_case(1024, {}); +BOOST_FIXTURE_TEST_CASE(store_read_entry_no_prune, state_history_test_fixture) { + store_read_test_case(server, 1024, {}); } -BOOST_AUTO_TEST_CASE(store_read_big_entry_no_prune) { +BOOST_FIXTURE_TEST_CASE(store_read_big_entry_no_prune, state_history_test_fixture) { // test the case where the uncompressed data size exceeds 4GB - store_read_test_case( (1ULL<< 32) + (1ULL << 20), {}); + store_read_test_case(server, (1ULL<< 32) + (1ULL << 20), {}); } -BOOST_AUTO_TEST_CASE(store_read_entry_prune_enabled) { - store_read_test_case(1024, eosio::state_history::prune_config{.prune_blocks = 100}); +BOOST_FIXTURE_TEST_CASE(store_read_entry_prune_enabled, state_history_test_fixture) { + store_read_test_case(server, 1024, eosio::state_history::prune_config{.prune_blocks = 100}); } -BOOST_AUTO_TEST_CASE(store_with_existing) { +BOOST_FIXTURE_TEST_CASE(store_with_existing, state_history_test_fixture) { uint64_t data_size = 512; fc::temp_directory log_dir; eosio::state_history_log log("ship", log_dir.path(), {}); eosio::state_history_log_header header; - header.block_id = block_id_for(1); + header.block_id = server.block_id_for(1); header.payload_size = 0; auto data = generate_data(data_size); - log.pack_and_write_entry(header, block_id_for(0), + log.pack_and_write_entry(header, server.block_id_for(0), [&](auto&& buf) { bio::write(buf, (const char*)data.data(), data.size() * sizeof(data[0])); }); - header.block_id = block_id_for(2); - log.pack_and_write_entry(header, block_id_for(1), + header.block_id = server.block_id_for(2); + log.pack_and_write_entry(header, server.block_id_for(1), [&](auto&& buf) { bio::write(buf, (const char*)data.data(), data.size() * sizeof(data[0])); }); // Do not allow starting from scratch for existing - header.block_id = block_id_for(1); + header.block_id = server.block_id_for(1); BOOST_CHECK_EXCEPTION( - log.pack_and_write_entry(header, block_id_for(0), [&](auto&& buf) { bio::write(buf, (const char*)data.data(), data.size() * sizeof(data[0])); }), + log.pack_and_write_entry(header, server.block_id_for(0), [&](auto&& buf) { bio::write(buf, (const char*)data.data(), data.size() * sizeof(data[0])); }), eosio::chain::plugin_exception, []( const auto& e ) { return e.to_detail_string().find( "Existing ship log" ) != std::string::npos; @@ -404,314 +411,362 @@ BOOST_AUTO_TEST_CASE(store_with_existing) { ); } -BOOST_FIXTURE_TEST_CASE(test_session_no_prune, state_history_test_fixture) { - try { - // setup block head for the server - server.setup_state_history_log(); - uint32_t head_block_num = 3; - server.block_head = {head_block_num, block_id_for(head_block_num)}; - - // generate the log data used for traces, deltas and finality_data - uint32_t n = mock_state_history_plugin::default_frame_size; - add_to_log(1, n * sizeof(uint32_t), generate_data(n)); // original data format - add_to_log(2, 0, generate_data(n)); // format to accommodate the compressed size greater than 4GB - add_to_log(3, 1, generate_data(n)); // format to encode decompressed size to avoid decompress entire data upfront. - - // send a get_status_request and verify the result is what we expected - verify_status(eosio::state_history::get_status_result_v0{ - .head = {head_block_num, block_id_for(head_block_num)}, - .last_irreversible = {head_block_num, block_id_for(head_block_num)}, - .trace_begin_block = 1, - .trace_end_block = head_block_num + 1, - .chain_state_begin_block = 1, - .chain_state_end_block = head_block_num + 1}); - - // send a get_blocks_request to server - send_request(eosio::state_history::get_blocks_request_v0{.start_block_num = 1, - .end_block_num = UINT32_MAX, - .max_messages_in_flight = UINT32_MAX, - .have_positions = {}, - .irreversible_only = false, - .fetch_block = true, - .fetch_traces = true, - .fetch_deltas = true, - .fetch_finality_data = true}); +void send_request(state_history_test_fixture& fixture, bool fetch_finality_data, uint32_t start_block_num, const std::vector& have_positions) { + if( fetch_finality_data ) { + get_blocks_request_v1 req { .fetch_finality_data = true }; - eosio::state_history::state_result result; - // we should get 3 consecutive block result - for (int i = 0; i < 3; ++i) { - receive_result(result); - BOOST_REQUIRE(std::holds_alternative(result)); - auto r = std::get(result); - BOOST_REQUIRE_EQUAL(r.head.block_num, server.block_head.block_num); - BOOST_REQUIRE(r.traces.has_value()); - BOOST_REQUIRE(r.deltas.has_value()); + req.start_block_num = start_block_num; + req.end_block_num = UINT32_MAX; + req.max_messages_in_flight = UINT32_MAX; + req.have_positions = have_positions; + req.irreversible_only = false; + req.fetch_block = true; + req.fetch_traces = true; + req.fetch_deltas = true; + + fixture.send_request(req); + } else { + fixture.send_request(eosio::state_history::get_blocks_request_v0{ + .start_block_num = start_block_num, + .end_block_num = UINT32_MAX, + .max_messages_in_flight = UINT32_MAX, + .have_positions = have_positions, + .irreversible_only = false, + .fetch_block = true, + .fetch_traces = true, + .fetch_deltas = true} + ); + } +} + +void test_session_no_prune_impl(state_history_test_fixture& fixture, bool fetch_finality_data) { + // setup block head for the server + fixture.server.setup_state_history_log(fetch_finality_data); + uint32_t head_block_num = 3; + fixture.server.block_head = {head_block_num, fixture.server.block_id_for(head_block_num)}; + + // generate the log data used for traces, deltas, and finality_data if required + uint32_t n = mock_state_history_plugin::default_frame_size; + fixture.add_to_log(1, n * sizeof(uint32_t), generate_data(n), fetch_finality_data); // original data format + fixture.add_to_log(2, 0, generate_data(n), fetch_finality_data); // format to accommodate the compressed size greater than 4GB + fixture.add_to_log(3, 1, generate_data(n), fetch_finality_data); // format to encode decompressed size to avoid decompress entire data upfront. + + // send a get_status_request and verify the result is what we expected + fixture.verify_status(eosio::state_history::get_status_result_v0{ + .head = {head_block_num, fixture.server.block_id_for(head_block_num)}, + .last_irreversible = {head_block_num, fixture.server.block_id_for(head_block_num)}, + .trace_begin_block = 1, + .trace_end_block = head_block_num + 1, + .chain_state_begin_block = 1, + .chain_state_end_block = head_block_num + 1}); + + // send a get_blocks_request to server + send_request(fixture, fetch_finality_data, 1, {}); + + eosio::state_history::state_result result; + // we should get 3 consecutive block result + for (int i = 0; i < 3; ++i) { + fixture.receive_result(result); + BOOST_REQUIRE(std::holds_alternative(result)); + auto r = std::get(result); + BOOST_REQUIRE_EQUAL(r.head.block_num, fixture.server.block_head.block_num); + BOOST_REQUIRE(r.traces.has_value()); + BOOST_REQUIRE(r.deltas.has_value()); + if( fetch_finality_data ) { BOOST_REQUIRE(r.finality_data.has_value()); - auto traces = r.traces.value(); - auto deltas = r.deltas.value(); - auto finality_data = r.finality_data.value(); - auto& data = written_data[i]; - auto data_size = data.size() * sizeof(int32_t); - BOOST_REQUIRE_EQUAL(traces.size(), data_size); - BOOST_REQUIRE_EQUAL(deltas.size(), data_size); + } + auto traces = r.traces.value(); + auto deltas = r.deltas.value(); + auto& data = fixture.written_data[i]; + auto data_size = data.size() * sizeof(int32_t); + BOOST_REQUIRE_EQUAL(traces.size(), data_size); + BOOST_REQUIRE_EQUAL(deltas.size(), data_size); + BOOST_REQUIRE(std::equal(traces.begin(), traces.end(), (const char*)data.data())); + BOOST_REQUIRE(std::equal(deltas.begin(), deltas.end(), (const char*)data.data())); + + if( fetch_finality_data ) { + auto finality_data = r.finality_data.value(); BOOST_REQUIRE_EQUAL(finality_data.size(), data_size); - - BOOST_REQUIRE(std::equal(traces.begin(), traces.end(), (const char*)data.data())); - BOOST_REQUIRE(std::equal(deltas.begin(), deltas.end(), (const char*)data.data())); BOOST_REQUIRE(std::equal(finality_data.begin(), finality_data.end(), (const char*)data.data())); } } +} + +BOOST_FIXTURE_TEST_CASE(test_session_no_prune, state_history_test_fixture) { + try { + test_session_no_prune_impl(*this, false); + } FC_LOG_AND_RETHROW() } -BOOST_FIXTURE_TEST_CASE(test_split_log, state_history_test_fixture) { +BOOST_FIXTURE_TEST_CASE(test_session_no_prune_fetch_finality_data, state_history_test_fixture) { try { - // setup block head for the server - constexpr uint32_t head = 1023; - eosio::state_history::partition_config conf; - conf.stride = 25; - server.setup_state_history_log(conf); - uint32_t head_block_num = head; - server.block_head = {head_block_num, block_id_for(head_block_num)}; - - // generate the log data used for traces, deltas and finality_data - uint32_t n = mock_state_history_plugin::default_frame_size; - add_to_log(1, n * sizeof(uint32_t), generate_data(n)); // original data format - add_to_log(2, 0, generate_data(n)); // format to accommodate the compressed size greater than 4GB - add_to_log(3, 1, generate_data(n)); // format to encode decompressed size to avoid decompress entire data upfront. - for (size_t i = 4; i <= head; ++i) { - add_to_log(i, 1, generate_data(n)); - } + test_session_no_prune_impl(*this, true); + } + FC_LOG_AND_RETHROW() +} - send_request(eosio::state_history::get_blocks_request_v0{.start_block_num = 1, - .end_block_num = UINT32_MAX, - .max_messages_in_flight = UINT32_MAX, - .have_positions = {}, - .irreversible_only = false, - .fetch_block = true, - .fetch_traces = true, - .fetch_deltas = true, - .fetch_finality_data = true}); +void test_split_log_impl(state_history_test_fixture& fixture, bool fetch_finality_data) { + // setup block head for the server + constexpr uint32_t head = 1023; + eosio::state_history::partition_config conf; + conf.stride = 25; + fixture.server.setup_state_history_log(fetch_finality_data, conf); + uint32_t head_block_num = head; + fixture.server.block_head = {head_block_num, fixture.server.block_id_for(head_block_num)}; + + // generate the log data used for traces, deltas and finality_data + uint32_t n = mock_state_history_plugin::default_frame_size; + fixture.add_to_log(1, n * sizeof(uint32_t), generate_data(n), fetch_finality_data); // original data format + fixture.add_to_log(2, 0, generate_data(n), fetch_finality_data); // format to accommodate the compressed size greater than 4GB + fixture.add_to_log(3, 1, generate_data(n), fetch_finality_data); // format to encode decompressed size to avoid decompress entire data upfront. + for (size_t i = 4; i <= head; ++i) { + fixture.add_to_log(i, 1, generate_data(n), fetch_finality_data); + } - eosio::state_history::state_result result; - // we should get 1023 consecutive block result - eosio::chain::block_id_type prev_id; - for (uint32_t i = 0; i < head; ++i) { - receive_result(result); - BOOST_REQUIRE(std::holds_alternative(result)); - auto r = std::get(result); - BOOST_REQUIRE_EQUAL(r.head.block_num, server.block_head.block_num); - if (i > 0) { - BOOST_TEST(prev_id.str() == r.prev_block->block_id.str()); - } - prev_id = r.this_block->block_id; - BOOST_REQUIRE(r.traces.has_value()); - BOOST_REQUIRE(r.deltas.has_value()); - BOOST_REQUIRE(r.finality_data.has_value()); - auto traces = r.traces.value(); - auto deltas = r.deltas.value(); + send_request(fixture, fetch_finality_data, 1, {}); + + eosio::state_history::state_result result; + // we should get 1023 consecutive block result + eosio::chain::block_id_type prev_id; + for (uint32_t i = 0; i < head; ++i) { + fixture.receive_result(result); + BOOST_REQUIRE(std::holds_alternative(result)); + auto r = std::get(result); + BOOST_REQUIRE_EQUAL(r.head.block_num, fixture.server.block_head.block_num); + if (i > 0) { + BOOST_TEST(prev_id.str() == r.prev_block->block_id.str()); + } + prev_id = r.this_block->block_id; + BOOST_REQUIRE(r.traces.has_value()); + BOOST_REQUIRE(r.deltas.has_value()); + auto traces = r.traces.value(); + auto deltas = r.deltas.value(); + auto& data = fixture.written_data[i]; + auto data_size = data.size() * sizeof(int32_t); + BOOST_REQUIRE_EQUAL(traces.size(), data_size); + BOOST_REQUIRE_EQUAL(deltas.size(), data_size); + + BOOST_REQUIRE(std::equal(traces.begin(), traces.end(), (const char*)data.data())); + BOOST_REQUIRE(std::equal(deltas.begin(), deltas.end(), (const char*)data.data())); + + if( fetch_finality_data ) { auto finality_data = r.finality_data.value(); - auto& data = written_data[i]; - auto data_size = data.size() * sizeof(int32_t); - BOOST_REQUIRE_EQUAL(traces.size(), data_size); - BOOST_REQUIRE_EQUAL(deltas.size(), data_size); + BOOST_REQUIRE(r.finality_data.has_value()); BOOST_REQUIRE_EQUAL(finality_data.size(), data_size); - - BOOST_REQUIRE(std::equal(traces.begin(), traces.end(), (const char*)data.data())); - BOOST_REQUIRE(std::equal(deltas.begin(), deltas.end(), (const char*)data.data())); BOOST_REQUIRE(std::equal(finality_data.begin(), finality_data.end(), (const char*)data.data())); } } +} + +BOOST_FIXTURE_TEST_CASE(test_split_log, state_history_test_fixture) { + try { + test_split_log_impl(*this, false); + } FC_LOG_AND_RETHROW() } -BOOST_FIXTURE_TEST_CASE(test_session_with_prune, state_history_test_fixture) { +BOOST_FIXTURE_TEST_CASE(test_split_log_fetch_finality_data, state_history_test_fixture) { try { - // setup block head for the server - server.setup_state_history_log( - eosio::state_history::prune_config{.prune_blocks = 2, .prune_threshold = 4 * 1024}); - - uint32_t head_block_num = 3; - server.block_head = {head_block_num, block_id_for(head_block_num)}; - - // generate the log data used for traces, deltas and finality_data - uint32_t n = mock_state_history_plugin::default_frame_size; - add_to_log(1, n * sizeof(uint32_t), generate_data(n)); // original data format - add_to_log(2, 0, generate_data(n)); // format to accommodate the compressed size greater than 4GB - add_to_log(3, 1, generate_data(n)); // format to encode decompressed size to avoid decompress entire data upfront. - - // send a get_status_request and verify the result is what we expected - verify_status(eosio::state_history::get_status_result_v0{ - .head = {head_block_num, block_id_for(head_block_num)}, - .last_irreversible = {head_block_num, block_id_for(head_block_num)}, - .trace_begin_block = 2, - .trace_end_block = head_block_num + 1, - .chain_state_begin_block = 2, - .chain_state_end_block = head_block_num + 1}); - - // send a get_blocks_request to server - send_request(eosio::state_history::get_blocks_request_v0{.start_block_num = 1, - .end_block_num = UINT32_MAX, - .max_messages_in_flight = UINT32_MAX, - .have_positions = {}, - .irreversible_only = false, - .fetch_block = true, - .fetch_traces = true, - .fetch_deltas = true, - .fetch_finality_data = true}); + test_split_log_impl(*this, true); + } + FC_LOG_AND_RETHROW() +} - eosio::state_history::state_result result; - // we should get 3 consecutive block result +void test_session_with_prune_impl(state_history_test_fixture& fixture, bool fetch_finality_data) { + // setup block head for the server + fixture.server.setup_state_history_log(fetch_finality_data, + eosio::state_history::prune_config{.prune_blocks = 2, .prune_threshold = 4 * 1024}); + + uint32_t head_block_num = 3; + fixture.server.block_head = {head_block_num, fixture.server.block_id_for(head_block_num)}; + + // generate the log data used for traces, deltas and finality_data + uint32_t n = mock_state_history_plugin::default_frame_size; + fixture.add_to_log(1, n * sizeof(uint32_t), generate_data(n), fetch_finality_data); // original data format + fixture.add_to_log(2, 0, generate_data(n), fetch_finality_data); // format to accommodate the compressed size greater than 4GB + fixture.add_to_log(3, 1, generate_data(n), fetch_finality_data); // format to encode decompressed size to avoid decompress entire data upfront. + + // send a get_status_request and verify the result is what we expected + fixture.verify_status(eosio::state_history::get_status_result_v0{ + .head = {head_block_num, fixture.server.block_id_for(head_block_num)}, + .last_irreversible = {head_block_num, fixture.server.block_id_for(head_block_num)}, + .trace_begin_block = 2, + .trace_end_block = head_block_num + 1, + .chain_state_begin_block = 2, + .chain_state_end_block = head_block_num + 1}); + + // send a get_blocks_request to fixture.server + send_request(fixture, fetch_finality_data, 1, {}); + + eosio::state_history::state_result result; + // we should get 3 consecutive block result + + fixture.receive_result(result); + BOOST_REQUIRE(std::holds_alternative(result)); + auto r = std::get(result); + BOOST_REQUIRE_EQUAL(r.head.block_num, fixture.server.block_head.block_num); + BOOST_REQUIRE(!r.traces.has_value()); + BOOST_REQUIRE(!r.deltas.has_value()); + if( fetch_finality_data ) { + BOOST_REQUIRE(!r.finality_data.has_value()); + } - receive_result(result); + for (int i = 1; i < 3; ++i) { + fixture.receive_result(result); BOOST_REQUIRE(std::holds_alternative(result)); auto r = std::get(result); - BOOST_REQUIRE_EQUAL(r.head.block_num, server.block_head.block_num); - BOOST_REQUIRE(!r.traces.has_value()); - BOOST_REQUIRE(!r.deltas.has_value()); - BOOST_REQUIRE(!r.finality_data.has_value()); - - for (int i = 1; i < 3; ++i) { - receive_result(result); - BOOST_REQUIRE(std::holds_alternative(result)); - auto r = std::get(result); - BOOST_REQUIRE_EQUAL(r.head.block_num, server.block_head.block_num); - BOOST_REQUIRE(r.traces.has_value()); - BOOST_REQUIRE(r.deltas.has_value()); + BOOST_REQUIRE_EQUAL(r.head.block_num, fixture.server.block_head.block_num); + BOOST_REQUIRE(r.traces.has_value()); + BOOST_REQUIRE(r.deltas.has_value()); + auto traces = r.traces.value(); + auto deltas = r.deltas.value(); + auto& data = fixture.written_data[i]; + auto data_size = data.size() * sizeof(int32_t); + BOOST_REQUIRE_EQUAL(traces.size(), data_size); + BOOST_REQUIRE_EQUAL(deltas.size(), data_size); + + BOOST_REQUIRE(std::equal(traces.begin(), traces.end(), (const char*)data.data())); + BOOST_REQUIRE(std::equal(deltas.begin(), deltas.end(), (const char*)data.data())); + + if( fetch_finality_data ) { BOOST_REQUIRE(r.finality_data.has_value()); - auto traces = r.traces.value(); - auto deltas = r.deltas.value(); auto finality_data = r.finality_data.value(); - auto& data = written_data[i]; - auto data_size = data.size() * sizeof(int32_t); - BOOST_REQUIRE_EQUAL(traces.size(), data_size); - BOOST_REQUIRE_EQUAL(deltas.size(), data_size); BOOST_REQUIRE_EQUAL(finality_data.size(), data_size); - - BOOST_REQUIRE(std::equal(traces.begin(), traces.end(), (const char*)data.data())); - BOOST_REQUIRE(std::equal(deltas.begin(), deltas.end(), (const char*)data.data())); BOOST_REQUIRE(std::equal(finality_data.begin(), finality_data.end(), (const char*)data.data())); } } +} + +BOOST_FIXTURE_TEST_CASE(test_session_with_prune, state_history_test_fixture) { + try { + test_session_with_prune_impl(*this, false); + } FC_LOG_AND_RETHROW() } -BOOST_FIXTURE_TEST_CASE(test_session_fork, state_history_test_fixture) { +BOOST_FIXTURE_TEST_CASE(test_session_with_prune_fetch_finality_data, state_history_test_fixture) { try { - // setup block head for the server - server.setup_state_history_log(); - uint32_t head_block_num = 4; - server.block_head = {head_block_num, block_id_for(head_block_num)}; - - // generate the log data used for traces, deltas and finality_data - uint32_t n = mock_state_history_plugin::default_frame_size; - add_to_log(1, n * sizeof(uint32_t), generate_data(n)); // original data format - add_to_log(2, 0, generate_data(n)); // format to accommodate the compressed size greater than 4GB - add_to_log(3, 1, generate_data(n)); // format to encode decompressed size to avoid decompress entire data upfront. - add_to_log(4, 1, generate_data(n)); // format to encode decompressed size to avoid decompress entire data upfront. - - // send a get_status_request and verify the result is what we expected - verify_status(eosio::state_history::get_status_result_v0{ - .head = {head_block_num, block_id_for(head_block_num)}, - .last_irreversible = {head_block_num, block_id_for(head_block_num)}, - .trace_begin_block = 1, - .trace_end_block = head_block_num + 1, - .chain_state_begin_block = 1, - .chain_state_end_block = head_block_num + 1}); - - // send a get_blocks_request to server - send_request(eosio::state_history::get_blocks_request_v0{ - .start_block_num = 1, - .end_block_num = UINT32_MAX, - .max_messages_in_flight = UINT32_MAX, - .have_positions = {}, - .irreversible_only = false, - .fetch_block = true, - .fetch_traces = true, - .fetch_deltas = true, - .fetch_finality_data = true}); - - std::vector have_positions; - eosio::state_history::state_result result; - // we should get 4 consecutive block result - for (uint32_t i = 0; i < 4; ++i) { - receive_result(result); - BOOST_REQUIRE(std::holds_alternative(result)); - auto r = std::get(result); - BOOST_REQUIRE_EQUAL(r.head.block_num, server.block_head.block_num); - BOOST_REQUIRE(r.traces.has_value()); - BOOST_REQUIRE(r.deltas.has_value()); + test_session_with_prune_impl(*this, true); + } + FC_LOG_AND_RETHROW() +} + +void test_session_fork_impl(state_history_test_fixture& fixture, bool fetch_finality_data) { + fixture.server.setup_state_history_log(fetch_finality_data); + uint32_t head_block_num = 4; + fixture.server.block_head = {head_block_num, fixture.server.block_id_for(head_block_num)}; + + // generate the log data used for traces, deltas and finality_data + uint32_t n = mock_state_history_plugin::default_frame_size; + fixture.add_to_log(1, n * sizeof(uint32_t), generate_data(n), fetch_finality_data); // original data format + fixture.add_to_log(2, 0, generate_data(n), fetch_finality_data); // format to accommodate the compressed size greater than 4GB + fixture.add_to_log(3, 1, generate_data(n), fetch_finality_data); // format to encode decompressed size to avoid decompress entire data upfront. + fixture.add_to_log(4, 1, generate_data(n), fetch_finality_data); // format to encode decompressed size to avoid decompress entire data upfront. + + // send a get_status_request and verify the result is what we expected + fixture.verify_status(eosio::state_history::get_status_result_v0{ + .head = {head_block_num, fixture.server.block_id_for(head_block_num)}, + .last_irreversible = {head_block_num, fixture.server.block_id_for(head_block_num)}, + .trace_begin_block = 1, + .trace_end_block = head_block_num + 1, + .chain_state_begin_block = 1, + .chain_state_end_block = head_block_num + 1}); + + // send a get_blocks_request to fixture.server + send_request(fixture, fetch_finality_data, 1, {}); + + std::vector have_positions; + eosio::state_history::state_result result; + // we should get 4 consecutive block result + for (uint32_t i = 0; i < 4; ++i) { + fixture.receive_result(result); + BOOST_REQUIRE(std::holds_alternative(result)); + auto r = std::get(result); + BOOST_REQUIRE_EQUAL(r.head.block_num, fixture.server.block_head.block_num); + BOOST_REQUIRE(r.traces.has_value()); + BOOST_REQUIRE(r.deltas.has_value()); + auto traces = r.traces.value(); + auto deltas = r.deltas.value(); + auto& data = fixture.written_data[i]; + auto data_size = data.size() * sizeof(int32_t); + BOOST_REQUIRE_EQUAL(traces.size(), data_size); + BOOST_REQUIRE_EQUAL(deltas.size(), data_size); + BOOST_REQUIRE(r.this_block.has_value()); + BOOST_REQUIRE_EQUAL(r.this_block->block_num, i+1); + have_positions.push_back(*r.this_block); + + BOOST_REQUIRE(std::equal(traces.begin(), traces.end(), (const char*)data.data())); + BOOST_REQUIRE(std::equal(deltas.begin(), deltas.end(), (const char*)data.data())); + + if( fetch_finality_data ) { BOOST_REQUIRE(r.finality_data.has_value()); - auto traces = r.traces.value(); - auto deltas = r.deltas.value(); - auto finality_data = r.finality_data.value(); - auto& data = written_data[i]; - auto data_size = data.size() * sizeof(int32_t); - BOOST_REQUIRE_EQUAL(traces.size(), data_size); - BOOST_REQUIRE_EQUAL(deltas.size(), data_size); + auto finality_data = r.finality_data.value(); BOOST_REQUIRE_EQUAL(finality_data.size(), data_size); - BOOST_REQUIRE(r.this_block.has_value()); - BOOST_REQUIRE_EQUAL(r.this_block->block_num, i+1); - have_positions.push_back(*r.this_block); - - BOOST_REQUIRE(std::equal(traces.begin(), traces.end(), (const char*)data.data())); - BOOST_REQUIRE(std::equal(deltas.begin(), deltas.end(), (const char*)data.data())); BOOST_REQUIRE(std::equal(finality_data.begin(), finality_data.end(), (const char*)data.data())); } + } - // generate a fork that includes blocks 3,4 and verify new data retrieved - block_ids.extract(3); block_id_for(3, "fork"); - block_ids.extract(4); block_id_for(4, "fork"); - server.block_head = {head_block_num, block_id_for(head_block_num)}; - add_to_log(3, 0, generate_data(n)); - add_to_log(4, 1, generate_data(n)); - - // send a get_status_request and verify the result is what we expected - verify_status(eosio::state_history::get_status_result_v0{ - .head = {head_block_num, block_id_for(head_block_num)}, - .last_irreversible = {head_block_num, block_id_for(head_block_num)}, - .trace_begin_block = 1, - .trace_end_block = head_block_num + 1, - .chain_state_begin_block = 1, - .chain_state_end_block = head_block_num + 1}); - - // send a get_blocks_request to server starting at 5, will send 3,4 because of fork - send_request(eosio::state_history::get_blocks_request_v0{ - .start_block_num = 5, - .end_block_num = UINT32_MAX, - .max_messages_in_flight = UINT32_MAX, - .have_positions = std::move(have_positions), - .irreversible_only = false, - .fetch_block = true, - .fetch_traces = true, - .fetch_deltas = true, - .fetch_finality_data = true}); - - eosio::state_history::state_result fork_result; - // we should now get data for fork 3,4 - for (uint32_t i = 2; i < 4; ++i) { - receive_result(fork_result); - BOOST_REQUIRE(std::holds_alternative(fork_result)); - auto r = std::get(fork_result); - BOOST_REQUIRE_EQUAL(r.head.block_num, server.block_head.block_num); - BOOST_REQUIRE(r.this_block.has_value()); - BOOST_REQUIRE_EQUAL(r.this_block->block_num, i+1); - BOOST_REQUIRE(r.traces.has_value()); - BOOST_REQUIRE(r.deltas.has_value()); + // generate a fork that includes blocks 3,4 and verify new data retrieved + // setup block head for the server + fixture.server.block_ids.extract(3); fixture.server.block_id_for(3, "fork"); + fixture.server.block_ids.extract(4); fixture.server.block_id_for(4, "fork"); + fixture.server.block_head = {head_block_num, fixture.server.block_id_for(head_block_num)}; + fixture.add_to_log(3, 0, generate_data(n), fetch_finality_data); + fixture.add_to_log(4, 1, generate_data(n), fetch_finality_data); + + // send a get_status_request and verify the result is what we expected + fixture.verify_status(eosio::state_history::get_status_result_v0{ + .head = {head_block_num, fixture.server.block_id_for(head_block_num)}, + .last_irreversible = {head_block_num, fixture.server.block_id_for(head_block_num)}, + .trace_begin_block = 1, + .trace_end_block = head_block_num + 1, + .chain_state_begin_block = 1, + .chain_state_end_block = head_block_num + 1}); + + // send a get_blocks_request to fixture.server starting at 5, will send 3,4 because of fork + send_request(fixture, fetch_finality_data, 5, std::move(have_positions)); + + eosio::state_history::state_result fork_result; + // we should now get data for fork 3,4 + for (uint32_t i = 2; i < 4; ++i) { + fixture.receive_result(fork_result); + BOOST_REQUIRE(std::holds_alternative(fork_result)); + auto r = std::get(fork_result); + BOOST_REQUIRE_EQUAL(r.head.block_num, fixture.server.block_head.block_num); + BOOST_REQUIRE(r.this_block.has_value()); + BOOST_REQUIRE_EQUAL(r.this_block->block_num, i+1); + BOOST_REQUIRE(r.traces.has_value()); + BOOST_REQUIRE(r.deltas.has_value()); + auto traces = r.traces.value(); + auto deltas = r.deltas.value(); + auto& data = fixture.written_data[i]; + auto data_size = data.size() * sizeof(int32_t); + BOOST_REQUIRE_EQUAL(traces.size(), data_size); + BOOST_REQUIRE_EQUAL(deltas.size(), data_size); + + BOOST_REQUIRE(std::equal(traces.begin(), traces.end(), (const char*)data.data())); + BOOST_REQUIRE(std::equal(deltas.begin(), deltas.end(), (const char*)data.data())); + + if( fetch_finality_data ) { BOOST_REQUIRE(r.finality_data.has_value()); - auto traces = r.traces.value(); - auto deltas = r.deltas.value(); - auto finality_data = r.finality_data.value(); - auto& data = written_data[i]; - auto data_size = data.size() * sizeof(int32_t); - BOOST_REQUIRE_EQUAL(traces.size(), data_size); - BOOST_REQUIRE_EQUAL(deltas.size(), data_size); + auto finality_data = r.finality_data.value(); BOOST_REQUIRE_EQUAL(finality_data.size(), data_size); - - BOOST_REQUIRE(std::equal(traces.begin(), traces.end(), (const char*)data.data())); - BOOST_REQUIRE(std::equal(deltas.begin(), deltas.end(), (const char*)data.data())); BOOST_REQUIRE(std::equal(finality_data.begin(), finality_data.end(), (const char*)data.data())); } } +} + +BOOST_FIXTURE_TEST_CASE(test_session_fork, state_history_test_fixture) { + try { + test_session_fork_impl(*this, false); + } + FC_LOG_AND_RETHROW() +} + +BOOST_FIXTURE_TEST_CASE(test_session_fork_fetch_finality_data, state_history_test_fixture) { + try { + test_session_fork_impl(*this, true); + } FC_LOG_AND_RETHROW() } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4a12e8a90b..0419a3ab71 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -152,6 +152,8 @@ add_test(NAME ship_streamer_test COMMAND tests/ship_streamer_test.py -v --num-cl set_property(TEST ship_streamer_test PROPERTY LABELS long_running_tests) add_test(NAME ship_streamer_if_test COMMAND tests/ship_streamer_test.py -v --num-clients 10 --activate-if ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST ship_streamer_if_test PROPERTY LABELS long_running_tests) +add_test(NAME ship_streamer_if_fetch_finality_data_test COMMAND tests/ship_streamer_test.py -v --num-clients 10 --activate-if --finality-data-history --fetch-finality-data ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +set_property(TEST ship_streamer_if_fetch_finality_data_test PROPERTY LABELS long_running_tests) add_test(NAME p2p_dawn515_test COMMAND tests/p2p_tests/dawn_515/test.sh WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST p2p_dawn515_test PROPERTY LABELS nonparallelizable_tests) diff --git a/tests/ship_streamer.cpp b/tests/ship_streamer.cpp index ff2843c55d..cb1d141d91 100644 --- a/tests/ship_streamer.cpp +++ b/tests/ship_streamer.cpp @@ -87,10 +87,17 @@ int main(int argc, char* argv[]) { // bool fetch_block = false; // bool fetch_traces = false; // bool fetch_deltas = false; + //}; + //struct get_blocks_request_v1 : get_blocks_request_v0 { // bool fetch_finality_data = false; //}; request_writer.StartArray(); - request_writer.String("get_blocks_request_v0"); + + if( fetch_finality_data ) { + request_writer.String("get_blocks_request_v1"); + } else { + request_writer.String("get_blocks_request_v0"); + } request_writer.StartObject(); request_writer.Key("start_block_num"); request_writer.Uint(start_block_num); @@ -109,14 +116,20 @@ int main(int argc, char* argv[]) { request_writer.Bool(fetch_traces); request_writer.Key("fetch_deltas"); request_writer.Bool(fetch_deltas); - request_writer.Key("fetch_finality_data"); - request_writer.Bool(fetch_finality_data); + if( fetch_finality_data ) { + request_writer.Key("fetch_finality_data"); + request_writer.Bool(fetch_finality_data); + } request_writer.EndObject(); request_writer.EndArray(); + std::cerr << "01\n"; stream.binary(true); + std::cerr << "02\n"; stream.write(boost::asio::buffer(request_type.json_to_bin(request_sb.GetString(), [](){}))); + std::cerr << "10\n"; stream.read_message_max(0); + std::cerr << "11\n"; // Each block_num can have multiple block_ids since forks are possible // block_num, block_id @@ -125,10 +138,14 @@ int main(int argc, char* argv[]) { for(;;) { boost::beast::flat_buffer buffer; stream.read(buffer); + std::cerr << "1\n"; eosio::input_stream is((const char*)buffer.data().data(), buffer.data().size()); + std::cerr << "2\n"; rapidjson::Document result_document; + //std::cerr << result_type.bin_to_json(is).c_str() << std::endl; result_document.Parse(result_type.bin_to_json(is).c_str()); + std::cerr << "3\n"; eosio::check(!result_document.HasParseError(), "Failed to parse result JSON from abieos"); eosio::check(result_document.IsArray(), "result should have been an array (variant) but it's not"); diff --git a/tests/ship_streamer_test.py b/tests/ship_streamer_test.py index 49a8370cbf..8fb0d0fa84 100755 --- a/tests/ship_streamer_test.py +++ b/tests/ship_streamer_test.py @@ -31,8 +31,13 @@ appArgs = AppArgs() extraArgs = appArgs.add(flag="--num-clients", type=int, help="How many ship_streamers should be started", default=1) +extraArgs = appArgs.add_bool(flag="--finality-data-history", help="Enable finality data history", action='store_true') +extraArgs = appArgs.add_bool(flag="--fetch-finality-data", help="Fetch finality data", action='store_true') args = TestHelper.parse_args({"--activate-if","--dump-error-details","--keep-logs","-v","--leave-running","--unshared"}, applicationSpecificArgs=appArgs) +if args.fetch_finality_data: + assert args.finality_data_history is not None and args.finality_data_history is not False, "ERROR: --finality-data-history is required for --fetch-finality-data" + Utils.Debug=args.v cluster=Cluster(unshared=args.unshared, keepRunning=args.leave_running, keepLogs=args.keep_logs) activateIF=args.activate_if @@ -71,7 +76,9 @@ def getLatestSnapshot(nodeId): shipNodeNum = 3 specificExtraNodeosArgs={} - specificExtraNodeosArgs[shipNodeNum]="--plugin eosio::state_history_plugin --trace-history --chain-state-history --finality-data-history --state-history-stride 200 --plugin eosio::net_api_plugin --plugin eosio::producer_api_plugin " + specificExtraNodeosArgs[shipNodeNum]="--plugin eosio::state_history_plugin --trace-history --chain-state-history --state-history-stride 200 --plugin eosio::net_api_plugin --plugin eosio::producer_api_plugin " + if args.finality_data_history: + specificExtraNodeosArgs[shipNodeNum]+=" --finality-data-history" # producer nodes will be mapped to 0 through totalProducerNodes-1, so the number totalProducerNodes will be the non-producing node specificExtraNodeosArgs[totalProducerNodes]="--plugin eosio::test_control_api_plugin " @@ -134,7 +141,9 @@ def getLatestSnapshot(nodeId): end_block_num = start_block_num + block_range shipClient = "tests/ship_streamer" - cmd = f"{shipClient} --start-block-num {start_block_num} --end-block-num {end_block_num} --fetch-block --fetch-traces --fetch-deltas --fetch-finality-data" + cmd = f"{shipClient} --start-block-num {start_block_num} --end-block-num {end_block_num} --fetch-block --fetch-traces --fetch-deltas" + if args.fetch_finality_data: + cmd += " --fetch-finality-data" if Utils.Debug: Utils.Print(f"cmd: {cmd}") clients = [] files = [] @@ -234,7 +243,9 @@ def getLatestSnapshot(nodeId): start_block_num = afterSnapshotBlockNum block_range = 0 end_block_num = start_block_num + block_range - cmd = f"{shipClient} --start-block-num {start_block_num} --end-block-num {end_block_num} --fetch-block --fetch-traces --fetch-deltas --fetch-finality-data" + cmd = f"{shipClient} --start-block-num {start_block_num} --end-block-num {end_block_num} --fetch-block --fetch-traces --fetch-deltas" + if args.fetch_finality_data: + cmd += " --fetch-finality-data" if Utils.Debug: Utils.Print(f"cmd: {cmd}") clients = [] files = [] From b1b964279dcec1c4dde1bd8af45a53b6d3feb5eb Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Wed, 20 Mar 2024 15:42:02 -0400 Subject: [PATCH 11/18] remove leftover debugging statements --- tests/ship_streamer.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/ship_streamer.cpp b/tests/ship_streamer.cpp index cb1d141d91..039b0f9566 100644 --- a/tests/ship_streamer.cpp +++ b/tests/ship_streamer.cpp @@ -122,14 +122,10 @@ int main(int argc, char* argv[]) { } request_writer.EndObject(); request_writer.EndArray(); - std::cerr << "01\n"; stream.binary(true); - std::cerr << "02\n"; stream.write(boost::asio::buffer(request_type.json_to_bin(request_sb.GetString(), [](){}))); - std::cerr << "10\n"; stream.read_message_max(0); - std::cerr << "11\n"; // Each block_num can have multiple block_ids since forks are possible // block_num, block_id @@ -138,14 +134,10 @@ int main(int argc, char* argv[]) { for(;;) { boost::beast::flat_buffer buffer; stream.read(buffer); - std::cerr << "1\n"; eosio::input_stream is((const char*)buffer.data().data(), buffer.data().size()); - std::cerr << "2\n"; rapidjson::Document result_document; - //std::cerr << result_type.bin_to_json(is).c_str() << std::endl; result_document.Parse(result_type.bin_to_json(is).c_str()); - std::cerr << "3\n"; eosio::check(!result_document.HasParseError(), "Failed to parse result JSON from abieos"); eosio::check(result_document.IsArray(), "result should have been an array (variant) but it's not"); From 0c6d290c624682697dcd6067c77987abc2d80473 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Thu, 21 Mar 2024 11:29:56 -0400 Subject: [PATCH 12/18] Add finality_data ABI definition so clients have a standard type definition --- libraries/state_history/abi.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/libraries/state_history/abi.cpp b/libraries/state_history/abi.cpp index 20e5118dd8..f3a2a43ff3 100644 --- a/libraries/state_history/abi.cpp +++ b/libraries/state_history/abi.cpp @@ -560,6 +560,15 @@ extern const char* const state_history_plugin_abi = R"({ { "type": "uint32", "name": "account_cpu_usage_average_window" }, { "type": "uint32", "name": "account_net_usage_average_window" } ] + }, + { + "name": "finality_data", "fields": [ + { "name": "major_version", "type": "uint32" }, + { "name": "minor_version", "type": "uint32" }, + { "name": "active_finalizer_policy_generation", "type": "uint32" }, + { "name": "action_mroot", "type": "checksum256" }, + { "name": "base_digest", "type": "checksum256" } + ] } ], "types": [ From 79142ab4ebdfe0553549afcce9c12de854117add Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Thu, 21 Mar 2024 16:44:29 -0400 Subject: [PATCH 13/18] simplify std::visitor uses; cache base_digest; and minor changes for reviewing comments --- libraries/chain/block_state.cpp | 4 ++++ .../chain/include/eosio/chain/block_state.hpp | 3 ++- .../eosio/state_history_plugin/session.hpp | 22 ++++--------------- .../state_history_plugin.cpp | 6 ++--- tests/CMakeLists.txt | 2 +- tests/ship_streamer_test.py | 8 ++----- 6 files changed, 16 insertions(+), 29 deletions(-) diff --git a/libraries/chain/block_state.cpp b/libraries/chain/block_state.cpp index 297abc0529..1e11816d6d 100644 --- a/libraries/chain/block_state.cpp +++ b/libraries/chain/block_state.cpp @@ -16,6 +16,7 @@ block_state::block_state(const block_header_state& prev, signed_block_ptr b, con , strong_digest(compute_finality_digest()) , weak_digest(create_weak_digest(strong_digest)) , pending_qc(prev.active_finalizer_policy->finalizers.size(), prev.active_finalizer_policy->threshold, prev.active_finalizer_policy->max_weak_sum_before_weak_final()) + , base_digest(compute_base_digest()) { // ASSUMPTION FROM controller_impl::apply_block = all untrusted blocks will have their signatures pre-validated here if( !skip_validate_signee ) { @@ -42,6 +43,7 @@ block_state::block_state(const block_header_state& bhs, , pub_keys_recovered(true) // called by produce_block so signature recovery of trxs must have been done , cached_trxs(std::move(trx_metas)) , action_mroot(action_mroot) + , base_digest(compute_base_digest()) { block->transactions = std::move(trx_receipts); @@ -92,6 +94,8 @@ block_state::block_state(const block_state_legacy& bsp, const digest_type& actio validated = bsp.is_valid(); pub_keys_recovered = bsp._pub_keys_recovered; cached_trxs = bsp._cached_trxs; + action_mroot = action_mroot_svnn; + base_digest = compute_base_digest(); } block_state::block_state(snapshot_detail::snapshot_block_state_v7&& sbs) diff --git a/libraries/chain/include/eosio/chain/block_state.hpp b/libraries/chain/include/eosio/chain/block_state.hpp index 0e42b448fb..5aed16b5a7 100644 --- a/libraries/chain/include/eosio/chain/block_state.hpp +++ b/libraries/chain/include/eosio/chain/block_state.hpp @@ -78,7 +78,8 @@ struct block_state : public block_header_state { // block_header_state provi // ------ data members caching information available elsewhere ---------------------- bool pub_keys_recovered = false; deque cached_trxs; - digest_type action_mroot; + digest_type action_mroot; // For base_digest sent to SHiP + digest_type base_digest; // For base_digest sent to SHiP // ------ private methods ----------------------------------------------------------- bool is_valid() const { return validated; } diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/session.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/session.hpp index 5f8d45f97f..84ee1557a7 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/session.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/session.hpp @@ -169,9 +169,7 @@ class blocks_ack_request_send_queue_entry : public send_queue_entry_base { assert(std::holds_alternative(*session->current_request) || std::holds_alternative(*session->current_request)); - std::visit(chain::overloaded{ - [&](eosio::state_history::get_blocks_request_v0& request) { request.max_messages_in_flight += req.num_messages;}, - [&](eosio::state_history::get_blocks_request_v1& request) { request.max_messages_in_flight += req.num_messages;} }, + std::visit([&](auto& request) { request.max_messages_in_flight += req.num_messages; }, *session->current_request); session->send_update(false); } @@ -513,15 +511,8 @@ struct session : session_base, std::enable_shared_from_this(req) || std::holds_alternative(req)); - std::visit(chain::overloaded{ - [&](state_history::get_blocks_request_v0& request) { - update_current_request_impl(request); - current_request = std::move(req);}, - [&](state_history::get_blocks_request_v1& request) { - update_current_request_impl(request); - fc_dlog(plugin.get_logger(), "replying get_blocks_request_v1, fetch_finality_data = ${fetch_finality_data}", ("fetch_finality_data", request.fetch_finality_data)); - current_request = std::move(req);} }, - req); + std::visit( [&](auto& request) { update_current_request_impl(request); }, req ); + current_request = std::move(req); } void send_update(state_history::get_blocks_request_v0& request, bool fetch_finality_data, state_history::get_blocks_result_v0 result, const chain::signed_block_ptr& block, const chain::block_id_type& id) { @@ -532,7 +523,6 @@ struct session : session_base, std::enable_shared_from_this current || to_send_block_num >= request.end_block_num) { fc_dlog( plugin.get_logger(), "Not sending, to_send_block_num: ${s}, current: ${c} request.end_block_num: ${b}", ("s", to_send_block_num)("c", current)("b", request.end_block_num) ); @@ -605,11 +595,7 @@ struct session : session_base, std::enable_shared_from_this uint32_t { - return request.max_messages_in_flight; }, - [&](state_history::get_blocks_request_v1& request) -> uint32_t { - return request.max_messages_in_flight; }}, + [&](auto& request) -> uint32_t { return request.max_messages_in_flight; }, *current_request); return !max_messages_in_flight; diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 797031890f..7ce4088fd2 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -210,7 +210,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thisprevious, block->block_num()); - store_finality_data(block, id); + store_finality_data(id, block->previous); } catch (const fc::exception& e) { fc_elog(_log, "fc::exception: ${details}", ("details", e.to_detail_string())); // Both app().quit() and exception throwing are required. Without app().quit(), @@ -275,7 +275,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thispack_and_write_entry(header, block->previous, [finality_data](auto&& buf) { + finality_data_log->pack_and_write_entry(header, previous_id, [finality_data](auto&& buf) { fc::datastream ds{buf}; fc::raw::pack(ds, *finality_data); }); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0419a3ab71..7b31b59e21 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -152,7 +152,7 @@ add_test(NAME ship_streamer_test COMMAND tests/ship_streamer_test.py -v --num-cl set_property(TEST ship_streamer_test PROPERTY LABELS long_running_tests) add_test(NAME ship_streamer_if_test COMMAND tests/ship_streamer_test.py -v --num-clients 10 --activate-if ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST ship_streamer_if_test PROPERTY LABELS long_running_tests) -add_test(NAME ship_streamer_if_fetch_finality_data_test COMMAND tests/ship_streamer_test.py -v --num-clients 10 --activate-if --finality-data-history --fetch-finality-data ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME ship_streamer_if_fetch_finality_data_test COMMAND tests/ship_streamer_test.py -v --num-clients 10 --activate-if --finality-data-history ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST ship_streamer_if_fetch_finality_data_test PROPERTY LABELS long_running_tests) add_test(NAME p2p_dawn515_test COMMAND tests/p2p_tests/dawn_515/test.sh WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/tests/ship_streamer_test.py b/tests/ship_streamer_test.py index 8fb0d0fa84..4f4719efc9 100755 --- a/tests/ship_streamer_test.py +++ b/tests/ship_streamer_test.py @@ -32,12 +32,8 @@ appArgs = AppArgs() extraArgs = appArgs.add(flag="--num-clients", type=int, help="How many ship_streamers should be started", default=1) extraArgs = appArgs.add_bool(flag="--finality-data-history", help="Enable finality data history", action='store_true') -extraArgs = appArgs.add_bool(flag="--fetch-finality-data", help="Fetch finality data", action='store_true') args = TestHelper.parse_args({"--activate-if","--dump-error-details","--keep-logs","-v","--leave-running","--unshared"}, applicationSpecificArgs=appArgs) -if args.fetch_finality_data: - assert args.finality_data_history is not None and args.finality_data_history is not False, "ERROR: --finality-data-history is required for --fetch-finality-data" - Utils.Debug=args.v cluster=Cluster(unshared=args.unshared, keepRunning=args.leave_running, keepLogs=args.keep_logs) activateIF=args.activate_if @@ -142,7 +138,7 @@ def getLatestSnapshot(nodeId): shipClient = "tests/ship_streamer" cmd = f"{shipClient} --start-block-num {start_block_num} --end-block-num {end_block_num} --fetch-block --fetch-traces --fetch-deltas" - if args.fetch_finality_data: + if args.finality_data_history: cmd += " --fetch-finality-data" if Utils.Debug: Utils.Print(f"cmd: {cmd}") clients = [] @@ -244,7 +240,7 @@ def getLatestSnapshot(nodeId): block_range = 0 end_block_num = start_block_num + block_range cmd = f"{shipClient} --start-block-num {start_block_num} --end-block-num {end_block_num} --fetch-block --fetch-traces --fetch-deltas" - if args.fetch_finality_data: + if args.finality_data_history: cmd += " --fetch-finality-data" if Utils.Debug: Utils.Print(f"cmd: {cmd}") clients = [] From e421b74fe6c4ac3253304cf07bc1d6607e9ef8ed Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Tue, 26 Mar 2024 09:53:35 -0400 Subject: [PATCH 14/18] use cached base_digest actually; minor review changes --- libraries/chain/block_state.cpp | 6 ++---- libraries/chain/controller.cpp | 3 +++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/libraries/chain/block_state.cpp b/libraries/chain/block_state.cpp index 1e11816d6d..e519ac6533 100644 --- a/libraries/chain/block_state.cpp +++ b/libraries/chain/block_state.cpp @@ -304,13 +304,11 @@ digest_type block_state::get_finality_mroot_claim(const qc_claim_t& qc_claim) co } finality_data_t block_state::get_finality_data() const { - finality_data_t finality_data { + return { // other fields take the default values set by finality_data_t definition .action_mroot = action_mroot, - .base_digest = compute_base_digest() // from block_header_state + .base_digest = base_digest }; - - return finality_data; } void inject_additional_signatures( signed_block& b, const std::vector& additional_signatures) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 914f149965..8d475306cf 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -4111,6 +4111,9 @@ struct controller_impl { } std::optional head_finality_data() const { + // We cannot use apply_s here as it returns an empty `finality_data_t` in + // Legacy, which causes SHiP to generate a null `finality_data` log + // (we want no `finality_data` is generated at all). return apply>(chain_head, overloaded{ [](const block_state_legacy_ptr& head) { return std::nullopt; }, [](const block_state_ptr& head) { return head->get_finality_data(); }}); From 35a040d73295f47a20495f6db30469ef43f99515 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Tue, 26 Mar 2024 12:10:51 -0400 Subject: [PATCH 15/18] minor changes responding to review comments --- .../state_history/include/eosio/state_history/types.hpp | 2 +- plugins/state_history_plugin/state_history_plugin.cpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/state_history/include/eosio/state_history/types.hpp b/libraries/state_history/include/eosio/state_history/types.hpp index 338b152d3e..38cfb9af86 100644 --- a/libraries/state_history/include/eosio/state_history/types.hpp +++ b/libraries/state_history/include/eosio/state_history/types.hpp @@ -105,7 +105,7 @@ struct get_blocks_request_v0 { }; struct get_blocks_request_v1 : get_blocks_request_v0 { - bool fetch_finality_data = false;; + bool fetch_finality_data = false; }; struct get_blocks_ack_request_v0 { diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 7ce4088fd2..966fe464d9 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -118,15 +118,15 @@ struct state_history_plugin_impl : std::enable_shared_from_this get_block_id(uint32_t block_num) { if( trace_log ) { - if ( auto id = trace_log->get_block_id( block_num ); id ) + if ( auto id = trace_log->get_block_id( block_num ) ) return id; } if( chain_state_log ) { - if( auto id = chain_state_log->get_block_id( block_num ); id ) + if( auto id = chain_state_log->get_block_id( block_num ) ) return id; } if( finality_data_log ) { - if( auto id = finality_data_log->get_block_id( block_num ); id ) + if( auto id = finality_data_log->get_block_id( block_num ) ) return id; } try { From f41214726e8c7d9161b0c8438f2bcb0ba6a4f18d Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Tue, 26 Mar 2024 13:28:38 -0400 Subject: [PATCH 16/18] make cached base_digest as an optional --- libraries/chain/block_state.cpp | 9 +++++---- libraries/chain/include/eosio/chain/block_state.hpp | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/libraries/chain/block_state.cpp b/libraries/chain/block_state.cpp index e519ac6533..fdf585189c 100644 --- a/libraries/chain/block_state.cpp +++ b/libraries/chain/block_state.cpp @@ -16,7 +16,6 @@ block_state::block_state(const block_header_state& prev, signed_block_ptr b, con , strong_digest(compute_finality_digest()) , weak_digest(create_weak_digest(strong_digest)) , pending_qc(prev.active_finalizer_policy->finalizers.size(), prev.active_finalizer_policy->threshold, prev.active_finalizer_policy->max_weak_sum_before_weak_final()) - , base_digest(compute_base_digest()) { // ASSUMPTION FROM controller_impl::apply_block = all untrusted blocks will have their signatures pre-validated here if( !skip_validate_signee ) { @@ -43,7 +42,6 @@ block_state::block_state(const block_header_state& bhs, , pub_keys_recovered(true) // called by produce_block so signature recovery of trxs must have been done , cached_trxs(std::move(trx_metas)) , action_mroot(action_mroot) - , base_digest(compute_base_digest()) { block->transactions = std::move(trx_receipts); @@ -303,11 +301,14 @@ digest_type block_state::get_finality_mroot_claim(const qc_claim_t& qc_claim) co return get_validation_mroot(next_core_metadata.final_on_strong_qc_block_num); } -finality_data_t block_state::get_finality_data() const { +finality_data_t block_state::get_finality_data() { + if (!base_digest) { + base_digest = compute_base_digest(); // cache it + } return { // other fields take the default values set by finality_data_t definition .action_mroot = action_mroot, - .base_digest = base_digest + .base_digest = *base_digest }; } diff --git a/libraries/chain/include/eosio/chain/block_state.hpp b/libraries/chain/include/eosio/chain/block_state.hpp index 5aed16b5a7..76d19e1c0a 100644 --- a/libraries/chain/include/eosio/chain/block_state.hpp +++ b/libraries/chain/include/eosio/chain/block_state.hpp @@ -78,8 +78,8 @@ struct block_state : public block_header_state { // block_header_state provi // ------ data members caching information available elsewhere ---------------------- bool pub_keys_recovered = false; deque cached_trxs; - digest_type action_mroot; // For base_digest sent to SHiP - digest_type base_digest; // For base_digest sent to SHiP + digest_type action_mroot; // For finality_data sent to SHiP + std::optional base_digest; // For finality_data sent to SHiP // ------ private methods ----------------------------------------------------------- bool is_valid() const { return validated; } @@ -120,7 +120,7 @@ struct block_state : public block_header_state { // block_header_state provi digest_type get_finality_mroot_claim(const qc_claim_t& qc_claim) const; // Returns finality_data of the current block - finality_data_t get_finality_data() const; + finality_data_t get_finality_data(); // vote_status vote_status aggregate_vote(const vote_message& vote); // aggregate vote into pending_qc From 5c82eec6d980c6c9ea9f18b860f23c49ff93ba7f Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Tue, 26 Mar 2024 15:07:07 -0400 Subject: [PATCH 17/18] remove an unnecessary compute_base_digest() --- libraries/chain/block_state.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/chain/block_state.cpp b/libraries/chain/block_state.cpp index fdf585189c..ddc308469a 100644 --- a/libraries/chain/block_state.cpp +++ b/libraries/chain/block_state.cpp @@ -93,7 +93,6 @@ block_state::block_state(const block_state_legacy& bsp, const digest_type& actio pub_keys_recovered = bsp._pub_keys_recovered; cached_trxs = bsp._cached_trxs; action_mroot = action_mroot_svnn; - base_digest = compute_base_digest(); } block_state::block_state(snapshot_detail::snapshot_block_state_v7&& sbs) From 21de5c4bee2ddcbb701bf3c9a9ab5e8db84876b1 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Tue, 26 Mar 2024 18:53:49 -0400 Subject: [PATCH 18/18] use apply_s for head_finality_data to simplify code --- libraries/chain/controller.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index d70f408ba2..4aa89ce691 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -4123,12 +4123,7 @@ struct controller_impl { } std::optional head_finality_data() const { - // We cannot use apply_s here as it returns an empty `finality_data_t` in - // Legacy, which causes SHiP to generate a null `finality_data` log - // (we want no `finality_data` is generated at all). - return apply>(chain_head, - overloaded{ [](const block_state_legacy_ptr& head) { return std::nullopt; }, - [](const block_state_ptr& head) { return head->get_finality_data(); }}); + return apply_s>(chain_head, [](const block_state_ptr& head) { return head->get_finality_data(); }); } uint32_t earliest_available_block_num() const {