Skip to content

Commit

Permalink
Merge branch 'main' of github.com:AntelopeIO/leap into gh-1025
Browse files Browse the repository at this point in the history
  • Loading branch information
@greg7mdp
greg7mdp committed May 18, 2023
2 parents e1add68 + e9d1b3a commit 92144f5
Show file tree
Hide file tree
Showing 11 changed files with 219 additions and 111 deletions.
61 changes: 53 additions & 8 deletions libraries/chain/block_log.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1371,16 +1371,61 @@ namespace eosio { namespace chain {
}

// static
std::optional<genesis_state> block_log::extract_genesis_state(const std::filesystem::path& block_dir) {
std::filesystem::path p(block_dir / "blocks.log");
for_each_file_in_dir_matches(block_dir, R"(blocks-1-\d+\.log)",
[&p](std::filesystem::path log_path) { p = std::move(log_path); });
return block_log_data(p).get_genesis_state();
std::optional<block_log::chain_context> block_log::extract_chain_context(const std::filesystem::path& block_dir,
const std::filesystem::path& retained_dir) {
std::filesystem::path first_block_file;
if (!retained_dir.empty() && std::filesystem::exists(retained_dir)) {
for_each_file_in_dir_matches(retained_dir, R"(blocks-1-\d+\.log)",
[&](std::filesystem::path log_path) {
first_block_file = std::move(log_path);
});
}

if (first_block_file.empty() && std::filesystem::exists(block_dir / "blocks.log")) {
first_block_file = block_dir / "blocks.log";
}

if (!first_block_file.empty()) {
return block_log_data(first_block_file).get_preamble().chain_context;
}

if (!retained_dir.empty() && std::filesystem::exists(retained_dir)) {
const std::regex my_filter(R"(blocks-\d+-\d+\.log)");
std::smatch what;
std::filesystem::directory_iterator end_itr; // Default ctor yields past-the-end
for (std::filesystem::directory_iterator p(retained_dir); p != end_itr; ++p) {
// Skip if not a file
if (!std::filesystem::is_regular_file(p->status()))
continue;
// skip if it does not match the pattern
std::string file = p->path().filename().string();
if (!std::regex_match(file, what, my_filter))
continue;
return block_log_data(p->path()).chain_id();
}
}
return {};
}

// static
std::optional<genesis_state> block_log::extract_genesis_state(const std::filesystem::path& block_dir,
const std::filesystem::path& retained_dir) {
auto context = extract_chain_context(block_dir, retained_dir);
if (!context || std::holds_alternative<chain_id_type>(*context))
return {};
return std::get<genesis_state>(*context);
}

// static
chain_id_type block_log::extract_chain_id(const std::filesystem::path& data_dir) {
return block_log_data(data_dir / "blocks.log").chain_id();
std::optional<chain_id_type> block_log::extract_chain_id(const std::filesystem::path& block_dir,
const std::filesystem::path& retained_dir) {
auto context = extract_chain_context(block_dir, retained_dir);
if (!context)
return {};
return std::visit(overloaded{
[](const chain_id_type& id){ return id; },
[](const genesis_state& gs){ return gs.compute_chain_id(); }
} , *context);
}

// static
Expand Down Expand Up @@ -1538,7 +1583,7 @@ namespace eosio { namespace chain {
ilog("blocks.log and blocks.index agree on number of blocks");

if (interval == 0) {
interval = std::max((log_bundle.log_index.num_blocks() + 7u) >> 3, 1u);
interval = std::max((log_bundle.log_index.num_blocks() + 7) >> 3, 1U);
}
uint32_t expected_block_num = log_bundle.log_data.first_block_num();

Expand Down
12 changes: 10 additions & 2 deletions libraries/chain/include/eosio/chain/block_log.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,9 +82,17 @@ namespace eosio { namespace chain {

static std::filesystem::path repair_log( const std::filesystem::path& data_dir, uint32_t truncate_at_block = 0, const char* reversible_block_dir_name="" );

static std::optional<genesis_state> extract_genesis_state( const std::filesystem::path& data_dir );
using chain_context = std::variant<genesis_state, chain_id_type>;
static std::optional<chain_context> extract_chain_context(const std::filesystem::path& data_dir,
const std::filesystem::path& retained_dir);

static chain_id_type extract_chain_id( const std::filesystem::path& data_dir );
static std::optional<genesis_state>
extract_genesis_state(const std::filesystem::path& data_dir,
const std::filesystem::path& retained_dir = std::filesystem::path{});

static std::optional<chain_id_type>
extract_chain_id(const std::filesystem::path& data_dir,
const std::filesystem::path& retained_dir = std::filesystem::path{});

static void construct_index(const std::filesystem::path& block_file_name, const std::filesystem::path& index_file_name);

Expand Down
2 changes: 1 addition & 1 deletion libraries/chain/include/eosio/chain/log_index.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ class log_index {
bool is_open() const { return file_.is_open(); }

uint64_t back() { return nth_block_position(num_blocks()-1); }
unsigned num_blocks() const { return num_blocks_; }
uint32_t num_blocks() const { return num_blocks_; }
uint64_t nth_block_position(uint32_t n) {
file_.seek(n*sizeof(uint64_t));
uint64_t r;
Expand Down
11 changes: 7 additions & 4 deletions libraries/testing/tester.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,11 +301,14 @@ namespace eosio { namespace testing {
if( !expected_chain_id ) {
expected_chain_id = controller::extract_chain_id_from_db( cfg.state_dir );
if( !expected_chain_id ) {
if( std::filesystem::is_regular_file( cfg.blocks_dir / "blocks.log" ) ) {
expected_chain_id = block_log::extract_chain_id( cfg.blocks_dir );
} else {
expected_chain_id = genesis_state().compute_chain_id();
std::filesystem::path retained_dir;
auto partitioned_config = std::get_if<partitioned_blocklog_config>(&cfg.blog);
if (partitioned_config) {
retained_dir = partitioned_config->retained_dir;
if (retained_dir.is_relative())
retained_dir = cfg.blocks_dir/retained_dir;
}
expected_chain_id = block_log::extract_chain_id( cfg.blocks_dir, retained_dir );
}
}

Expand Down
133 changes: 64 additions & 69 deletions plugins/chain_plugin/chain_plugin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -629,45 +629,6 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
if( options.count( "terminate-at-block" ))
my->chain_config->terminate_at_block = options.at( "terminate-at-block" ).as<uint32_t>();

if( options.count( "extract-genesis-json" ) || options.at( "print-genesis-json" ).as<bool>()) {
std::optional<genesis_state> gs;

if( std::filesystem::exists( my->blocks_dir / "blocks.log" )) {
gs = block_log::extract_genesis_state( my->blocks_dir );
EOS_ASSERT( gs,
plugin_config_exception,
"Block log at '${path}' does not contain a genesis state, it only has the chain-id.",
("path", (my->blocks_dir / "blocks.log"))
);
} else {
wlog( "No blocks.log found at '${p}'. Using default genesis state.",
("p", (my->blocks_dir / "blocks.log")));
gs.emplace();
}

if( options.at( "print-genesis-json" ).as<bool>()) {
ilog( "Genesis JSON:\n${genesis}", ("genesis", json::to_pretty_string( *gs )));
}

if( options.count( "extract-genesis-json" )) {
auto p = options.at( "extract-genesis-json" ).as<std::filesystem::path>();

if( p.is_relative()) {
p = std::filesystem::current_path() / p;
}

EOS_ASSERT( fc::json::save_to_file( *gs, p, true ),
misc_exception,
"Error occurred while writing genesis JSON to '${path}'",
("path", p)
);

ilog( "Saved genesis JSON to '${path}'", ("path", p) );
}

EOS_THROW( extract_genesis_state_exception, "extracted genesis state from blocks.log" );
}

// move fork_db to new location
upgrade_from_reversible_to_fork_db( my.get() );

Expand All @@ -678,18 +639,22 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
EOS_ASSERT(!has_partitioned_block_log_options || !has_retain_blocks_option, plugin_config_exception,
"block-log-retain-blocks cannot be specified together with blocks-retained-dir, blocks-archive-dir or blocks-log-stride or max-retained-block-files.");


std::filesystem::path retained_dir;
if (has_partitioned_block_log_options) {
retained_dir = options.count("blocks-retained-dir") ? options.at("blocks-retained-dir").as<std::filesystem::path>()
: std::filesystem::path("");
if (retained_dir.is_relative())
retained_dir = std::filesystem::path{my->blocks_dir}/retained_dir;

my->chain_config->blog = eosio::chain::partitioned_blocklog_config{
.retained_dir = options.count("blocks-retained-dir") ? options.at("blocks-retained-dir").as<std::filesystem::path>()
: std::filesystem::path(""),
.retained_dir = retained_dir,
.archive_dir = options.count("blocks-archive-dir") ? options.at("blocks-archive-dir").as<std::filesystem::path>()
: std::filesystem::path("archive"),
: std::filesystem::path("archive"),
.stride = options.count("blocks-log-stride") ? options.at("blocks-log-stride").as<uint32_t>()
: UINT32_MAX,
.max_retained_files = options.count("max-retained-block-files")
? options.at("max-retained-block-files").as<uint32_t>()
: UINT32_MAX,
? options.at("max-retained-block-files").as<uint32_t>()
: UINT32_MAX,
};
} else if(has_retain_blocks_option) {
uint32_t block_log_retain_blocks = options.at("block-log-retain-blocks").as<uint32_t>();
Expand All @@ -703,6 +668,42 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
}
}



if( options.count( "extract-genesis-json" ) || options.at( "print-genesis-json" ).as<bool>()) {
std::optional<genesis_state> gs;

gs = block_log::extract_genesis_state( my->blocks_dir, retained_dir );
EOS_ASSERT( gs,
plugin_config_exception,
"Block log at '${path}' does not contain a genesis state, it only has the chain-id.",
("path", (my->blocks_dir / "blocks.log").generic_string())
);


if( options.at( "print-genesis-json" ).as<bool>()) {
ilog( "Genesis JSON:\n${genesis}", ("genesis", json::to_pretty_string( *gs )));
}

if( options.count( "extract-genesis-json" )) {
auto p = options.at( "extract-genesis-json" ).as<std::filesystem::path>();

if( p.is_relative()) {
p = std::filesystem::current_path() / p;
}

EOS_ASSERT( fc::json::save_to_file( *gs, p, true ),
misc_exception,
"Error occurred while writing genesis JSON to '${path}'",
("path", p.generic_string())
);

ilog( "Saved genesis JSON to '${path}'", ("path", p.generic_string()) );
}

EOS_THROW( extract_genesis_state_exception, "extracted genesis state from blocks.log" );
}

if( options.at( "delete-all-blocks" ).as<bool>()) {
ilog( "Deleting state database and blocks" );
if( options.at( "truncate-at-block" ).as<uint32_t>() > 0 )
Expand Down Expand Up @@ -746,41 +747,35 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
plugin_config_exception,
"Snapshot can only be used to initialize an empty database." );

if( std::filesystem::is_regular_file( my->blocks_dir / "blocks.log" )) {
auto block_log_genesis = block_log::extract_genesis_state(my->blocks_dir);
if( block_log_genesis ) {
const auto& block_log_chain_id = block_log_genesis->compute_chain_id();
EOS_ASSERT( *chain_id == block_log_chain_id,
plugin_config_exception,
"snapshot chain ID (${snapshot_chain_id}) does not match the chain ID from the genesis state in the block log (${block_log_chain_id})",
("snapshot_chain_id", *chain_id)
("block_log_chain_id", block_log_chain_id)
);
} else {
const auto& block_log_chain_id = block_log::extract_chain_id(my->blocks_dir);
EOS_ASSERT( *chain_id == block_log_chain_id,
auto block_log_chain_id = block_log::extract_chain_id(my->blocks_dir, retained_dir);

if (block_log_chain_id) {
EOS_ASSERT( *chain_id == *block_log_chain_id,
plugin_config_exception,
"snapshot chain ID (${snapshot_chain_id}) does not match the chain ID (${block_log_chain_id}) in the block log",
("snapshot_chain_id", *chain_id)
("block_log_chain_id", block_log_chain_id)
("block_log_chain_id", *block_log_chain_id)
);
}
}

} else {

chain_id = controller::extract_chain_id_from_db( my->chain_config->state_dir );

auto chain_context = block_log::extract_chain_context( my->blocks_dir, retained_dir );
std::optional<genesis_state> block_log_genesis;
std::optional<chain_id_type> block_log_chain_id;

if( std::filesystem::is_regular_file( my->blocks_dir / "blocks.log" ) ) {
block_log_genesis = block_log::extract_genesis_state( my->blocks_dir );
if( block_log_genesis ) {
block_log_chain_id = block_log_genesis->compute_chain_id();
} else {
block_log_chain_id = block_log::extract_chain_id( my->blocks_dir );
}
std::optional<chain_id_type> block_log_chain_id;

if (chain_context) {
std::visit(overloaded {
[&](const genesis_state& gs) {
block_log_genesis = gs;
block_log_chain_id = gs.compute_chain_id();
},
[&](const chain_id_type& id) {
block_log_chain_id = id;
}
}, *chain_context);

if( chain_id ) {
EOS_ASSERT( *block_log_chain_id == *chain_id, block_log_exception,
Expand Down
2 changes: 2 additions & 0 deletions plugins/http_plugin/http_plugin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ namespace eosio {
// It's a programming error when the control flow reaches this point,
// please make sure all the category names are returned from above statements.
assert(false && "No correspding category name for the category value");
return "";
}

std::string category_plugin_name(api_category category) {
Expand All @@ -94,6 +95,7 @@ namespace eosio {
// It's a programming error when the control flow reaches this point,
// please make sure all the plugin names are returned from above statements.
assert(false && "No correspding plugin for the category value");
return {};
}

std::string category_names(api_category_set set) {
Expand Down
27 changes: 14 additions & 13 deletions programs/leap-util/actions/blocklog.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,32 +166,33 @@ int blocklog_actions::extract_blocks() {
}

int blocklog_actions::do_genesis() {
std::optional<genesis_state> gs;
std::filesystem::path bld = opt->blocks_dir;
auto full_path = (bld / "blocks.log").generic_string();

if(std::filesystem::exists(bld / "blocks.log")) {
gs = block_log::extract_genesis_state(opt->blocks_dir);
if(!gs) {
std::cerr << "Block log at '" << full_path
<< "' does not contain a genesis state, it only has the chain-id." << std::endl;
return -1;
}
} else {
std::cerr << "No blocks.log found at '" << full_path << "'." << std::endl;
auto context = block_log::extract_chain_context(opt->blocks_dir,opt->blocks_dir);

if (!context) {
std::cerr << "No blocks log found at '" << opt->blocks_dir.c_str() << "'." << std::endl;
return -1;
}

if(!std::holds_alternative<genesis_state>(*context)) {
std::cerr << "Block log at '" << opt->blocks_dir.c_str()
<< "' does not contain a genesis state, it only has the chain-id." << std::endl;
return -1;
}

const genesis_state& gs = std::get<genesis_state>(*context);

// just print if output not set
if(opt->output_file.empty()) {
std::cout << json::to_pretty_string(*gs) << std::endl;
std::cout << json::to_pretty_string(gs) << std::endl;
} else {
std::filesystem::path p = opt->output_file;
if(p.is_relative()) {
p = std::filesystem::current_path() / p;
}

if(!fc::json::save_to_file(*gs, p, true)) {
if(!fc::json::save_to_file(gs, p, true)) {
std::cerr << "Error occurred while writing genesis JSON to '" << p.generic_string() << "'" << std::endl;
return -1;
}
Expand Down
4 changes: 1 addition & 3 deletions tests/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/launcher.py ${CMAKE_CURRENT_BINARY_DI
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/auto_bp_peering_test.py ${CMAKE_CURRENT_BINARY_DIR}/auto_bp_peering_test.py COPYONLY)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/auto_bp_peering_test_shape.json ${CMAKE_CURRENT_BINARY_DIR}/auto_bp_peering_test_shape.json COPYONLY)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/gelf_test.py ${CMAKE_CURRENT_BINARY_DIR}/gelf_test.py COPYONLY)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/split_blocklog_replay_test.py ${CMAKE_CURRENT_BINARY_DIR}/split_blocklog_replay_test.py COPYONLY)

if(DEFINED ENV{GITHUB_ACTIONS})
set(UNSHARE "--unshared")
Expand Down Expand Up @@ -262,9 +263,6 @@ set_property(TEST light_validation_sync_test PROPERTY LABELS nonparallelizable_t
add_test(NAME auto_bp_peering_test COMMAND tests/auto_bp_peering_test.py ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
set_property(TEST auto_bp_peering_test PROPERTY LABELS long_running_tests)

add_test(NAME gelf_test COMMAND tests/gelf_test.py ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
set_property(TEST gelf_test PROPERTY LABELS nonparallelizable_tests)

if(ENABLE_COVERAGE_TESTING)

set(Coverage_NAME ${PROJECT_NAME}_coverage)
Expand Down
Loading

0 comments on commit 92144f5

Please sign in to comment.