Skip to content

Commit

Permalink
Merge branch 'main' into GH-1072-refactor
Browse files Browse the repository at this point in the history
  • Loading branch information
@heifner
heifner authored May 18, 2023
2 parents 31983be + e9d1b3a commit caf601d
Show file tree
Hide file tree
Showing 15 changed files with 233 additions and 124 deletions.
2 changes: 1 addition & 1 deletion libraries/appbase
Submodule appbase updated 1 files
+15 −15 application_base.cpp
61 changes: 53 additions & 8 deletions libraries/chain/block_log.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1371,16 +1371,61 @@ namespace eosio { namespace chain {
}

// static
std::optional<genesis_state> block_log::extract_genesis_state(const std::filesystem::path& block_dir) {
std::filesystem::path p(block_dir / "blocks.log");
for_each_file_in_dir_matches(block_dir, R"(blocks-1-\d+\.log)",
[&p](std::filesystem::path log_path) { p = std::move(log_path); });
return block_log_data(p).get_genesis_state();
std::optional<block_log::chain_context> block_log::extract_chain_context(const std::filesystem::path& block_dir,
const std::filesystem::path& retained_dir) {
std::filesystem::path first_block_file;
if (!retained_dir.empty() && std::filesystem::exists(retained_dir)) {
for_each_file_in_dir_matches(retained_dir, R"(blocks-1-\d+\.log)",
[&](std::filesystem::path log_path) {
first_block_file = std::move(log_path);
});
}

if (first_block_file.empty() && std::filesystem::exists(block_dir / "blocks.log")) {
first_block_file = block_dir / "blocks.log";
}

if (!first_block_file.empty()) {
return block_log_data(first_block_file).get_preamble().chain_context;
}

if (!retained_dir.empty() && std::filesystem::exists(retained_dir)) {
const std::regex my_filter(R"(blocks-\d+-\d+\.log)");
std::smatch what;
std::filesystem::directory_iterator end_itr; // Default ctor yields past-the-end
for (std::filesystem::directory_iterator p(retained_dir); p != end_itr; ++p) {
// Skip if not a file
if (!std::filesystem::is_regular_file(p->status()))
continue;
// skip if it does not match the pattern
std::string file = p->path().filename().string();
if (!std::regex_match(file, what, my_filter))
continue;
return block_log_data(p->path()).chain_id();
}
}
return {};
}

// static
std::optional<genesis_state> block_log::extract_genesis_state(const std::filesystem::path& block_dir,
const std::filesystem::path& retained_dir) {
auto context = extract_chain_context(block_dir, retained_dir);
if (!context || std::holds_alternative<chain_id_type>(*context))
return {};
return std::get<genesis_state>(*context);
}

// static
chain_id_type block_log::extract_chain_id(const std::filesystem::path& data_dir) {
return block_log_data(data_dir / "blocks.log").chain_id();
std::optional<chain_id_type> block_log::extract_chain_id(const std::filesystem::path& block_dir,
const std::filesystem::path& retained_dir) {
auto context = extract_chain_context(block_dir, retained_dir);
if (!context)
return {};
return std::visit(overloaded{
[](const chain_id_type& id){ return id; },
[](const genesis_state& gs){ return gs.compute_chain_id(); }
} , *context);
}

// static
Expand Down Expand Up @@ -1538,7 +1583,7 @@ namespace eosio { namespace chain {
ilog("blocks.log and blocks.index agree on number of blocks");

if (interval == 0) {
interval = std::max((log_bundle.log_index.num_blocks() + 7u) >> 3, 1u);
interval = std::max((log_bundle.log_index.num_blocks() + 7) >> 3, 1U);
}
uint32_t expected_block_num = log_bundle.log_data.first_block_num();

Expand Down
11 changes: 4 additions & 7 deletions libraries/chain/controller.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -402,11 +402,13 @@ struct controller_impl {
}
}

void dmlog_applied_transaction(const transaction_trace_ptr& t) {
void dmlog_applied_transaction(const transaction_trace_ptr& t, const signed_transaction* trx = nullptr) {
// dmlog_applied_transaction is called by push_scheduled_transaction
// where transient transactions are not possible, and by push_transaction
// only when the transaction is not transient
if (auto dm_logger = get_deep_mind_logger(false)) {
if (trx && is_onblock(*t))
dm_logger->on_onblock(*trx);
dm_logger->on_applied_transaction(self.head_block_num() + 1, t);
}
}
Expand Down Expand Up @@ -1627,7 +1629,7 @@ struct controller_impl {
emit(self.accepted_transaction, trx);
}

dmlog_applied_transaction(trace);
dmlog_applied_transaction(trace, &trn);
emit(self.applied_transaction, std::tie(trace, trx->packed_trx()));
}
}
Expand Down Expand Up @@ -2665,11 +2667,6 @@ struct controller_impl {
trx.set_reference_block( self.head_block_id() );
}

// onblock transaction cannot be transient
if (auto dm_logger = get_deep_mind_logger(false)) {
dm_logger->on_onblock(trx);
}

return trx;
}

Expand Down
12 changes: 10 additions & 2 deletions libraries/chain/include/eosio/chain/block_log.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,9 +82,17 @@ namespace eosio { namespace chain {

static std::filesystem::path repair_log( const std::filesystem::path& data_dir, uint32_t truncate_at_block = 0, const char* reversible_block_dir_name="" );

static std::optional<genesis_state> extract_genesis_state( const std::filesystem::path& data_dir );
using chain_context = std::variant<genesis_state, chain_id_type>;
static std::optional<chain_context> extract_chain_context(const std::filesystem::path& data_dir,
const std::filesystem::path& retained_dir);

static chain_id_type extract_chain_id( const std::filesystem::path& data_dir );
static std::optional<genesis_state>
extract_genesis_state(const std::filesystem::path& data_dir,
const std::filesystem::path& retained_dir = std::filesystem::path{});

static std::optional<chain_id_type>
extract_chain_id(const std::filesystem::path& data_dir,
const std::filesystem::path& retained_dir = std::filesystem::path{});

static void construct_index(const std::filesystem::path& block_file_name, const std::filesystem::path& index_file_name);

Expand Down
2 changes: 1 addition & 1 deletion libraries/chain/include/eosio/chain/log_index.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ class log_index {
bool is_open() const { return file_.is_open(); }

uint64_t back() { return nth_block_position(num_blocks()-1); }
unsigned num_blocks() const { return num_blocks_; }
uint32_t num_blocks() const { return num_blocks_; }
uint64_t nth_block_position(uint32_t n) {
file_.seek(n*sizeof(uint64_t));
uint64_t r;
Expand Down
11 changes: 7 additions & 4 deletions libraries/testing/tester.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,11 +301,14 @@ namespace eosio { namespace testing {
if( !expected_chain_id ) {
expected_chain_id = controller::extract_chain_id_from_db( cfg.state_dir );
if( !expected_chain_id ) {
if( std::filesystem::is_regular_file( cfg.blocks_dir / "blocks.log" ) ) {
expected_chain_id = block_log::extract_chain_id( cfg.blocks_dir );
} else {
expected_chain_id = genesis_state().compute_chain_id();
std::filesystem::path retained_dir;
auto partitioned_config = std::get_if<partitioned_blocklog_config>(&cfg.blog);
if (partitioned_config) {
retained_dir = partitioned_config->retained_dir;
if (retained_dir.is_relative())
retained_dir = cfg.blocks_dir/retained_dir;
}
expected_chain_id = block_log::extract_chain_id( cfg.blocks_dir, retained_dir );
}
}

Expand Down
133 changes: 64 additions & 69 deletions plugins/chain_plugin/chain_plugin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -629,45 +629,6 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
if( options.count( "terminate-at-block" ))
my->chain_config->terminate_at_block = options.at( "terminate-at-block" ).as<uint32_t>();

if( options.count( "extract-genesis-json" ) || options.at( "print-genesis-json" ).as<bool>()) {
std::optional<genesis_state> gs;

if( std::filesystem::exists( my->blocks_dir / "blocks.log" )) {
gs = block_log::extract_genesis_state( my->blocks_dir );
EOS_ASSERT( gs,
plugin_config_exception,
"Block log at '${path}' does not contain a genesis state, it only has the chain-id.",
("path", (my->blocks_dir / "blocks.log"))
);
} else {
wlog( "No blocks.log found at '${p}'. Using default genesis state.",
("p", (my->blocks_dir / "blocks.log")));
gs.emplace();
}

if( options.at( "print-genesis-json" ).as<bool>()) {
ilog( "Genesis JSON:\n${genesis}", ("genesis", json::to_pretty_string( *gs )));
}

if( options.count( "extract-genesis-json" )) {
auto p = options.at( "extract-genesis-json" ).as<std::filesystem::path>();

if( p.is_relative()) {
p = std::filesystem::current_path() / p;
}

EOS_ASSERT( fc::json::save_to_file( *gs, p, true ),
misc_exception,
"Error occurred while writing genesis JSON to '${path}'",
("path", p)
);

ilog( "Saved genesis JSON to '${path}'", ("path", p) );
}

EOS_THROW( extract_genesis_state_exception, "extracted genesis state from blocks.log" );
}

// move fork_db to new location
upgrade_from_reversible_to_fork_db( my.get() );

Expand All @@ -678,18 +639,22 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
EOS_ASSERT(!has_partitioned_block_log_options || !has_retain_blocks_option, plugin_config_exception,
"block-log-retain-blocks cannot be specified together with blocks-retained-dir, blocks-archive-dir or blocks-log-stride or max-retained-block-files.");


std::filesystem::path retained_dir;
if (has_partitioned_block_log_options) {
retained_dir = options.count("blocks-retained-dir") ? options.at("blocks-retained-dir").as<std::filesystem::path>()
: std::filesystem::path("");
if (retained_dir.is_relative())
retained_dir = std::filesystem::path{my->blocks_dir}/retained_dir;

my->chain_config->blog = eosio::chain::partitioned_blocklog_config{
.retained_dir = options.count("blocks-retained-dir") ? options.at("blocks-retained-dir").as<std::filesystem::path>()
: std::filesystem::path(""),
.retained_dir = retained_dir,
.archive_dir = options.count("blocks-archive-dir") ? options.at("blocks-archive-dir").as<std::filesystem::path>()
: std::filesystem::path("archive"),
: std::filesystem::path("archive"),
.stride = options.count("blocks-log-stride") ? options.at("blocks-log-stride").as<uint32_t>()
: UINT32_MAX,
.max_retained_files = options.count("max-retained-block-files")
? options.at("max-retained-block-files").as<uint32_t>()
: UINT32_MAX,
? options.at("max-retained-block-files").as<uint32_t>()
: UINT32_MAX,
};
} else if(has_retain_blocks_option) {
uint32_t block_log_retain_blocks = options.at("block-log-retain-blocks").as<uint32_t>();
Expand All @@ -703,6 +668,42 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
}
}



if( options.count( "extract-genesis-json" ) || options.at( "print-genesis-json" ).as<bool>()) {
std::optional<genesis_state> gs;

gs = block_log::extract_genesis_state( my->blocks_dir, retained_dir );
EOS_ASSERT( gs,
plugin_config_exception,
"Block log at '${path}' does not contain a genesis state, it only has the chain-id.",
("path", (my->blocks_dir / "blocks.log").generic_string())
);


if( options.at( "print-genesis-json" ).as<bool>()) {
ilog( "Genesis JSON:\n${genesis}", ("genesis", json::to_pretty_string( *gs )));
}

if( options.count( "extract-genesis-json" )) {
auto p = options.at( "extract-genesis-json" ).as<std::filesystem::path>();

if( p.is_relative()) {
p = std::filesystem::current_path() / p;
}

EOS_ASSERT( fc::json::save_to_file( *gs, p, true ),
misc_exception,
"Error occurred while writing genesis JSON to '${path}'",
("path", p.generic_string())
);

ilog( "Saved genesis JSON to '${path}'", ("path", p.generic_string()) );
}

EOS_THROW( extract_genesis_state_exception, "extracted genesis state from blocks.log" );
}

if( options.at( "delete-all-blocks" ).as<bool>()) {
ilog( "Deleting state database and blocks" );
if( options.at( "truncate-at-block" ).as<uint32_t>() > 0 )
Expand Down Expand Up @@ -746,41 +747,35 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
plugin_config_exception,
"Snapshot can only be used to initialize an empty database." );

if( std::filesystem::is_regular_file( my->blocks_dir / "blocks.log" )) {
auto block_log_genesis = block_log::extract_genesis_state(my->blocks_dir);
if( block_log_genesis ) {
const auto& block_log_chain_id = block_log_genesis->compute_chain_id();
EOS_ASSERT( *chain_id == block_log_chain_id,
plugin_config_exception,
"snapshot chain ID (${snapshot_chain_id}) does not match the chain ID from the genesis state in the block log (${block_log_chain_id})",
("snapshot_chain_id", *chain_id)
("block_log_chain_id", block_log_chain_id)
);
} else {
const auto& block_log_chain_id = block_log::extract_chain_id(my->blocks_dir);
EOS_ASSERT( *chain_id == block_log_chain_id,
auto block_log_chain_id = block_log::extract_chain_id(my->blocks_dir, retained_dir);

if (block_log_chain_id) {
EOS_ASSERT( *chain_id == *block_log_chain_id,
plugin_config_exception,
"snapshot chain ID (${snapshot_chain_id}) does not match the chain ID (${block_log_chain_id}) in the block log",
("snapshot_chain_id", *chain_id)
("block_log_chain_id", block_log_chain_id)
("block_log_chain_id", *block_log_chain_id)
);
}
}

} else {

chain_id = controller::extract_chain_id_from_db( my->chain_config->state_dir );

auto chain_context = block_log::extract_chain_context( my->blocks_dir, retained_dir );
std::optional<genesis_state> block_log_genesis;
std::optional<chain_id_type> block_log_chain_id;

if( std::filesystem::is_regular_file( my->blocks_dir / "blocks.log" ) ) {
block_log_genesis = block_log::extract_genesis_state( my->blocks_dir );
if( block_log_genesis ) {
block_log_chain_id = block_log_genesis->compute_chain_id();
} else {
block_log_chain_id = block_log::extract_chain_id( my->blocks_dir );
}
std::optional<chain_id_type> block_log_chain_id;

if (chain_context) {
std::visit(overloaded {
[&](const genesis_state& gs) {
block_log_genesis = gs;
block_log_chain_id = gs.compute_chain_id();
},
[&](const chain_id_type& id) {
block_log_chain_id = id;
}
}, *chain_context);

if( chain_id ) {
EOS_ASSERT( *block_log_chain_id == *chain_id, block_log_exception,
Expand Down
2 changes: 2 additions & 0 deletions plugins/http_plugin/http_plugin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ namespace eosio {
// It's a programming error when the control flow reaches this point,
// please make sure all the category names are returned from above statements.
assert(false && "No correspding category name for the category value");
return "";
}

std::string category_plugin_name(api_category category) {
Expand All @@ -94,6 +95,7 @@ namespace eosio {
// It's a programming error when the control flow reaches this point,
// please make sure all the plugin names are returned from above statements.
assert(false && "No correspding plugin for the category value");
return {};
}

std::string category_names(api_category_set set) {
Expand Down
6 changes: 5 additions & 1 deletion plugins/producer_plugin/test/test_read_only_trx.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,11 @@ void test_configs_common(std::vector<const char*>& specific_args, app_init_statu
// app->initialize() returns a boolean. BOOST_CHECK_EQUAL cannot compare
// a boolean with a app_init_status directly
bool rc = (expected_status == app_init_status::succeeded) ? true : false;
BOOST_CHECK_EQUAL( app->initialize<producer_plugin>( argv.size(), (char**) &argv[0]), rc );
bool result = false;
try {
result = app->initialize<producer_plugin>( argv.size(), (char**) &argv[0]);
} catch(...) {}
BOOST_CHECK_EQUAL( result, rc );
}

// --read-only-thread not allowed on producer node
Expand Down
Loading

0 comments on commit caf601d

Please sign in to comment.