From 7aa17452f7a26562a6d82a47f773510c64398a25 Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Thu, 23 Nov 2023 14:56:01 +0100 Subject: [PATCH 01/24] feat: add database analyse-gas-usage command Add a command which will allow to analyse gas usage on the blockchain. The command can be executed by running: ```bash ./neard database analyse-gas-usage ``` The command will look at gas used in all of the blocks, shards and accounts, and print out an analysis report. This commit just adds the command, the implementation will be added in the following commits. --- Cargo.lock | 1 + tools/database/Cargo.toml | 1 + tools/database/src/analyse_gas_usage.rs | 39 +++++++++++++++++++++++++ tools/database/src/commands.rs | 5 ++++ tools/database/src/lib.rs | 1 + 5 files changed, 47 insertions(+) create mode 100644 tools/database/src/analyse_gas_usage.rs diff --git a/Cargo.lock b/Cargo.lock index 9316b653fb7..4a6bca7215d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3708,6 +3708,7 @@ dependencies = [ "borsh 1.0.0", "clap", "indicatif", + "near-chain", "near-chain-configs", "near-epoch-manager", "near-primitives", diff --git a/tools/database/Cargo.toml b/tools/database/Cargo.toml index 90dcd4b537d..ad3798b6cb9 100644 --- a/tools/database/Cargo.toml +++ b/tools/database/Cargo.toml @@ -24,6 +24,7 @@ tempfile.workspace = true nearcore.workspace = true near-epoch-manager.workspace = true +near-chain.workspace = true near-chain-configs.workspace = true near-store.workspace = true near-primitives.workspace = true diff --git a/tools/database/src/analyse_gas_usage.rs b/tools/database/src/analyse_gas_usage.rs new file mode 100644 index 00000000000..70c374fae31 --- /dev/null +++ b/tools/database/src/analyse_gas_usage.rs @@ -0,0 +1,39 @@ +use std::path::PathBuf; +use std::rc::Rc; + +use clap::Parser; +use near_chain::{ChainStore, ChainStoreAccess}; +use near_epoch_manager::EpochManager; +use near_store::{NodeStorage, Store}; +use nearcore::open_storage; + +#[derive(Parser)] +pub(crate) struct AnalyseGasUsageCommand; + +impl AnalyseGasUsageCommand { + pub(crate) fn run(&self, home: &PathBuf) -> anyhow::Result<()> { + // Create a ChainStore and EpochManager that will be used to read blockchain data. + let mut near_config = + nearcore::config::load_config(home, near_chain_configs::GenesisValidationMode::Full) + .unwrap(); + let node_storage: NodeStorage = open_storage(&home, &mut near_config).unwrap(); + let store: Store = + node_storage.get_split_store().unwrap_or_else(|| node_storage.get_hot_store()); + let chain_store = Rc::new(ChainStore::new( + store.clone(), + near_config.genesis.config.genesis_height, + false, + )); + let _epoch_manager = + EpochManager::new_from_genesis_config(store, &near_config.genesis.config).unwrap(); + + println!( + "Analysing gas usage, the tip of the blockchain is: {:#?}", + chain_store.head().unwrap() + ); + + // TODO: Implement the analysis + + Ok(()) + } +} diff --git a/tools/database/src/commands.rs b/tools/database/src/commands.rs index f3e656e68e7..ef9060b7118 100644 --- a/tools/database/src/commands.rs +++ b/tools/database/src/commands.rs @@ -1,5 +1,6 @@ use crate::adjust_database::ChangeDbKindCommand; use crate::analyse_data_size_distribution::AnalyseDataSizeDistributionCommand; +use crate::analyse_gas_usage::AnalyseGasUsageCommand; use crate::compact::RunCompactionCommand; use crate::corrupt::CorruptStateSnapshotCommand; use crate::make_snapshot::MakeSnapshotCommand; @@ -21,6 +22,9 @@ enum SubCommand { /// Analyse data size distribution in RocksDB AnalyseDataSizeDistribution(AnalyseDataSizeDistributionCommand), + /// Analyse gas usage in a chosen sequnce of blocks + AnalyseGasUsage(AnalyseGasUsageCommand), + /// Change DbKind of hot or cold db. ChangeDbKind(ChangeDbKindCommand), @@ -48,6 +52,7 @@ impl DatabaseCommand { pub fn run(&self, home: &PathBuf) -> anyhow::Result<()> { match &self.subcmd { SubCommand::AnalyseDataSizeDistribution(cmd) => cmd.run(home), + SubCommand::AnalyseGasUsage(cmd) => cmd.run(home), SubCommand::ChangeDbKind(cmd) => cmd.run(home), SubCommand::CompactDatabase(cmd) => cmd.run(home), SubCommand::CorruptStateSnapshot(cmd) => cmd.run(home), diff --git a/tools/database/src/lib.rs b/tools/database/src/lib.rs index 6d7e70f1691..b4db57fe664 100644 --- a/tools/database/src/lib.rs +++ b/tools/database/src/lib.rs @@ -1,5 +1,6 @@ mod adjust_database; mod analyse_data_size_distribution; +mod analyse_gas_usage; pub mod commands; mod compact; mod corrupt; From 4eb31c8b160d1f30c320bdad16f7899a93c68658 Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Thu, 23 Nov 2023 15:15:14 +0100 Subject: [PATCH 02/24] Make it possible to analyse the last N blocks Add a flag that can be used to specify how many latest blocks should be analysed using the analyse-gas-usage command. Now running: ```bash ./neard database analyse-gas-usage --last-blocks 1000 ``` Will go over the last 1000 blocks in the blockchain and analyse them. --- tools/database/src/analyse_gas_usage.rs | 33 ++++++++++---- .../src/block_iterators/last_blocks.rs | 43 +++++++++++++++++++ tools/database/src/block_iterators/mod.rs | 6 +++ tools/database/src/lib.rs | 1 + 4 files changed, 75 insertions(+), 8 deletions(-) create mode 100644 tools/database/src/block_iterators/last_blocks.rs create mode 100644 tools/database/src/block_iterators/mod.rs diff --git a/tools/database/src/analyse_gas_usage.rs b/tools/database/src/analyse_gas_usage.rs index 70c374fae31..a3b49ed3c79 100644 --- a/tools/database/src/analyse_gas_usage.rs +++ b/tools/database/src/analyse_gas_usage.rs @@ -2,13 +2,19 @@ use std::path::PathBuf; use std::rc::Rc; use clap::Parser; -use near_chain::{ChainStore, ChainStoreAccess}; +use near_chain::{Block, ChainStore}; use near_epoch_manager::EpochManager; use near_store::{NodeStorage, Store}; use nearcore::open_storage; +use crate::block_iterators::LastNBlocksIterator; + #[derive(Parser)] -pub(crate) struct AnalyseGasUsageCommand; +pub(crate) struct AnalyseGasUsageCommand { + /// Analyse the last N blocks in the blockchain + #[arg(long)] + last_blocks: Option, +} impl AnalyseGasUsageCommand { pub(crate) fn run(&self, home: &PathBuf) -> anyhow::Result<()> { @@ -24,16 +30,27 @@ impl AnalyseGasUsageCommand { near_config.genesis.config.genesis_height, false, )); - let _epoch_manager = + let epoch_manager = EpochManager::new_from_genesis_config(store, &near_config.genesis.config).unwrap(); - println!( - "Analysing gas usage, the tip of the blockchain is: {:#?}", - chain_store.head().unwrap() - ); + // Create an iterator over the blocks that should be analysed + let last_blocks_to_analyse: u64 = self.last_blocks.unwrap_or(100); + let blocks_iter = LastNBlocksIterator::new(last_blocks_to_analyse, chain_store.clone()); - // TODO: Implement the analysis + // Analyse + analyse_gas_usage(blocks_iter, &chain_store, &epoch_manager); Ok(()) } } + +fn analyse_gas_usage( + blocks_iter: impl Iterator, + _chain_store: &ChainStore, + _epoch_manager: &EpochManager, +) { + for block in blocks_iter { + println!("Analysing block with height: {}", block.header().height()); + // TODO: Analyse + } +} diff --git a/tools/database/src/block_iterators/last_blocks.rs b/tools/database/src/block_iterators/last_blocks.rs new file mode 100644 index 00000000000..bd6240a0da0 --- /dev/null +++ b/tools/database/src/block_iterators/last_blocks.rs @@ -0,0 +1,43 @@ +use std::rc::Rc; + +use near_chain::{Block, ChainStore, ChainStoreAccess}; +use near_primitives::hash::CryptoHash; + +/// Iterate over the last N blocks in the blockchain +pub struct LastNBlocksIterator { + chain_store: Rc, + blocks_left: u64, + /// Hash of the block that will be returned when next() is called + current_block_hash: Option, +} + +impl LastNBlocksIterator { + pub fn new(blocks_num: u64, chain_store: Rc) -> LastNBlocksIterator { + let current_block_hash = Some(chain_store.head().unwrap().last_block_hash); + LastNBlocksIterator { chain_store, blocks_left: blocks_num, current_block_hash } + } +} + +impl Iterator for LastNBlocksIterator { + type Item = Block; + + fn next(&mut self) -> Option { + // Decrease the amount of blocks left to produce + match self.blocks_left.checked_sub(1) { + Some(new_blocks_left) => self.blocks_left = new_blocks_left, + None => return None, + }; + + if let Some(current_block_hash) = self.current_block_hash.take() { + let current_block: Block = self.chain_store.get_block(¤t_block_hash).unwrap(); + + // Set the previous block as "current" one, as long as the current one isn't the genesis block + if current_block.header().height() != self.chain_store.get_genesis_height() { + self.current_block_hash = Some(*current_block.header().prev_hash()); + } + return Some(current_block); + } + + None + } +} diff --git a/tools/database/src/block_iterators/mod.rs b/tools/database/src/block_iterators/mod.rs new file mode 100644 index 00000000000..e293d1d8968 --- /dev/null +++ b/tools/database/src/block_iterators/mod.rs @@ -0,0 +1,6 @@ +//! This module contains iterators that can be used to iterate over blocks in the database + +pub mod last_blocks; + +/// Iterate over the last N blocks in the blockchain +pub use last_blocks::LastNBlocksIterator; diff --git a/tools/database/src/lib.rs b/tools/database/src/lib.rs index b4db57fe664..827b5da4a63 100644 --- a/tools/database/src/lib.rs +++ b/tools/database/src/lib.rs @@ -1,6 +1,7 @@ mod adjust_database; mod analyse_data_size_distribution; mod analyse_gas_usage; +mod block_iterators; pub mod commands; mod compact; mod corrupt; From 67967f50749d3433c917246e8ad533609470ac4a Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Thu, 23 Nov 2023 15:22:33 +0100 Subject: [PATCH 03/24] Implement analysing gas usage For each block we want to gather information about how much gas was used in every shard and account. Gas is used when transactions and receipts are executed, and every such execution produces an ExecutionOutcome, which is saved in the database. We can read all ExecutionOutcomes originating from a given shard from the database, and get all of the needed information from there. Every ExecutionOutcome contains the AccountId of the executor and the amount of gas that was burned during the execution. It's everything that is needed for analysing gas usage. Usage from all blocks is merged into a single instace of GasUsageStats and the interesting pieces of information are displayed for the user. --- tools/database/src/analyse_gas_usage.rs | 182 +++++++++++++++++++++++- 1 file changed, 177 insertions(+), 5 deletions(-) diff --git a/tools/database/src/analyse_gas_usage.rs b/tools/database/src/analyse_gas_usage.rs index a3b49ed3c79..7523df49131 100644 --- a/tools/database/src/analyse_gas_usage.rs +++ b/tools/database/src/analyse_gas_usage.rs @@ -1,14 +1,30 @@ +use std::collections::BTreeMap; use std::path::PathBuf; use std::rc::Rc; +use std::sync::Arc; use clap::Parser; use near_chain::{Block, ChainStore}; use near_epoch_manager::EpochManager; -use near_store::{NodeStorage, Store}; + +use near_primitives::epoch_manager::block_info::BlockInfo; +use near_primitives::hash::CryptoHash; +use near_primitives::shard_layout::{account_id_to_shard_id, ShardLayout}; +use near_primitives::transaction::ExecutionOutcome; +use near_primitives::types::{AccountId, BlockHeight, EpochId, ShardId}; + +use near_store::{NodeStorage, ShardUId, Store}; use nearcore::open_storage; use crate::block_iterators::LastNBlocksIterator; +/// `Gas` is an u64, but it stil might overflow when analysing a large amount of blocks. +/// 1ms of compute is about 1TGas = 10^12 gas. One epoch takes 43200 seconds (43200000ms). +/// This means that the amount of gas consumed during a single epoch can reach 43200000 * 10^12 = 4.32 * 10^19 +/// 10^19 doesn't fit in u64, so we need to use u128 +/// To avoid overflows, let's use `BigGas` for storing gas amounts in the code. +type BigGas = u128; + #[derive(Parser)] pub(crate) struct AnalyseGasUsageCommand { /// Analyse the last N blocks in the blockchain @@ -44,13 +60,169 @@ impl AnalyseGasUsageCommand { } } +#[derive(Clone, Debug, Default)] +struct GasUsageInShard { + pub used_gas_per_account: BTreeMap, + pub used_gas_total: BigGas, +} + +impl GasUsageInShard { + pub fn new() -> GasUsageInShard { + GasUsageInShard { used_gas_per_account: BTreeMap::new(), used_gas_total: 0 } + } + + pub fn add_used_gas(&mut self, account: AccountId, used_gas: BigGas) { + let old_used_gas: &BigGas = self.used_gas_per_account.get(&account).unwrap_or(&0); + let new_used_gas: BigGas = old_used_gas.checked_add(used_gas).unwrap(); + self.used_gas_per_account.insert(account, new_used_gas); + + self.used_gas_total = self.used_gas_total.checked_add(used_gas).unwrap(); + } + + pub fn merge(&mut self, other: &GasUsageInShard) { + for (account_id, used_gas) in &other.used_gas_per_account { + self.add_used_gas(account_id.clone(), *used_gas); + } + self.used_gas_total = self.used_gas_total.checked_add(other.used_gas_total).unwrap(); + } +} + +#[derive(Clone, Debug)] +struct GasUsageStats { + pub shards: BTreeMap, +} + +impl GasUsageStats { + pub fn new() -> GasUsageStats { + GasUsageStats { shards: BTreeMap::new() } + } + + pub fn add_gas_usage_in_shard(&mut self, shard_uid: ShardUId, shard_usage: GasUsageInShard) { + match self.shards.get_mut(&shard_uid) { + Some(existing_shard_usage) => existing_shard_usage.merge(&shard_usage), + None => { + let _ = self.shards.insert(shard_uid, shard_usage); + } + } + } + + pub fn used_gas_total(&self) -> BigGas { + let mut result: BigGas = 0; + for shard_usage in self.shards.values() { + result = result.checked_add(shard_usage.used_gas_total).unwrap(); + } + result + } + + pub fn merge(&mut self, other: GasUsageStats) { + for (shard_uid, shard_usage) in other.shards { + self.add_gas_usage_in_shard(shard_uid, shard_usage); + } + } +} + +fn get_gas_usage_in_block( + block: &Block, + chain_store: &ChainStore, + epoch_manager: &EpochManager, +) -> GasUsageStats { + let block_info: Arc = epoch_manager.get_block_info(block.hash()).unwrap(); + let epoch_id: &EpochId = block_info.epoch_id(); + let shard_layout: ShardLayout = epoch_manager.get_shard_layout(epoch_id).unwrap(); + + let mut result = GasUsageStats::new(); + + // Go over every chunk in this block and gather data + for chunk_header in block.chunks().iter() { + let shard_id: ShardId = chunk_header.shard_id(); + let shard_uid: ShardUId = ShardUId::from_shard_id_and_layout(shard_id, &shard_layout); + + let mut gas_usage_in_shard = GasUsageInShard::new(); + + // The outcome of each transaction and receipt executed in this chunk is saved in the database as an ExecutionOutcome. + // Go through all ExecutionOutcomes from this chunk and record the gas usage. + let outcome_ids: Vec = + chain_store.get_outcomes_by_block_hash_and_shard_id(block.hash(), shard_id).unwrap(); + for outcome_id in outcome_ids { + let outcome: ExecutionOutcome = chain_store + .get_outcome_by_id_and_block_hash(&outcome_id, block.hash()) + .unwrap() + .unwrap() + .outcome; + + // Sanity check - make sure that the executor of this outcome belongs to this shard + let account_shard: ShardId = + account_id_to_shard_id(&outcome.executor_id, &shard_layout); + assert_eq!(account_shard, shard_id); + + gas_usage_in_shard.add_used_gas(outcome.executor_id, outcome.gas_burnt.into()); + } + + result.add_gas_usage_in_shard(shard_uid, gas_usage_in_shard); + } + + result +} + fn analyse_gas_usage( blocks_iter: impl Iterator, - _chain_store: &ChainStore, - _epoch_manager: &EpochManager, + chain_store: &ChainStore, + epoch_manager: &EpochManager, ) { + // Gather statistics about gas usage in all of the blocks + let mut blocks_count: usize = 0; + let mut first_analysed_block: Option<(BlockHeight, CryptoHash)> = None; + let mut last_analysed_block: Option<(BlockHeight, CryptoHash)> = None; + + let mut gas_usage_stats = GasUsageStats::new(); + for block in blocks_iter { - println!("Analysing block with height: {}", block.header().height()); - // TODO: Analyse + blocks_count += 1; + if first_analysed_block.is_none() { + first_analysed_block = Some((block.header().height(), *block.hash())); + } + last_analysed_block = Some((block.header().height(), *block.hash())); + + let gas_usage_in_block: GasUsageStats = + get_gas_usage_in_block(&block, chain_store, epoch_manager); + gas_usage_stats.merge(gas_usage_in_block); + } + + // Calculates how much percent of `big` is `small` and returns it as a string. + // Example: as_percentage_of(10, 100) == "10.0%" + let as_percentage_of = |small: BigGas, big: BigGas| { + if big > 0 { + format!("{:.1}%", small as f64 / big as f64 * 100.0) + } else { + format!("-") + } + }; + + // Print out the analysis + if blocks_count == 0 { + println!("No blocks to analyse!"); + return; + } + println!(""); + println!("Analysed {} blocks between:", blocks_count); + if let Some((block_height, block_hash)) = first_analysed_block { + println!("Block: height = {block_height}, hash = {block_hash}"); + } + if let Some((block_height, block_hash)) = last_analysed_block { + println!("Block: height = {block_height}, hash = {block_hash}"); + } + let total_gas: BigGas = gas_usage_stats.used_gas_total(); + println!(""); + println!("Total gas used: {}", total_gas); + println!(""); + for (shard_uid, shard_usage) in &gas_usage_stats.shards { + println!("Shard: {}", shard_uid); + println!( + " Gas usage: {} ({} of total)", + shard_usage.used_gas_total, + as_percentage_of(shard_usage.used_gas_total, total_gas) + ); + println!(" Number of accounts: {}", shard_usage.used_gas_per_account.len()); + println!(""); } } From 202df41b9c9b21b5ab610d5f18cc58afd84faf87 Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Thu, 23 Nov 2023 15:45:49 +0100 Subject: [PATCH 04/24] Make it possible to analyse a range of block heights Add a flag that can be used to specify a range of blocks to be analysed. Now running: ```bash ./neard database analyse-gas-usage --from-block-height 120 --to-block-height 130 ``` Will analyse 11 blocks with heights in range of [120, 130] The logic of converting command line arguments to block iterators has been extracted to the block_iterators module. In the future it'll be possible to reuse it for another command that wants to analyse some subset of blocks. --- tools/database/src/analyse_gas_usage.rs | 28 ++++++- .../src/block_iterators/height_range.rs | 83 +++++++++++++++++++ tools/database/src/block_iterators/mod.rs | 57 ++++++++++++- 3 files changed, 164 insertions(+), 4 deletions(-) create mode 100644 tools/database/src/block_iterators/height_range.rs diff --git a/tools/database/src/analyse_gas_usage.rs b/tools/database/src/analyse_gas_usage.rs index 7523df49131..71b96302b84 100644 --- a/tools/database/src/analyse_gas_usage.rs +++ b/tools/database/src/analyse_gas_usage.rs @@ -16,7 +16,7 @@ use near_primitives::types::{AccountId, BlockHeight, EpochId, ShardId}; use near_store::{NodeStorage, ShardUId, Store}; use nearcore::open_storage; -use crate::block_iterators::LastNBlocksIterator; +use crate::block_iterators::{CommandArgs, LastNBlocksIterator}; /// `Gas` is an u64, but it stil might overflow when analysing a large amount of blocks. /// 1ms of compute is about 1TGas = 10^12 gas. One epoch takes 43200 seconds (43200000ms). @@ -30,6 +30,14 @@ pub(crate) struct AnalyseGasUsageCommand { /// Analyse the last N blocks in the blockchain #[arg(long)] last_blocks: Option, + + /// Analyse blocks from the given block height, inclusive + #[arg(long)] + from_block_height: Option, + + /// Analyse blocks up to the given block height, inclusive + #[arg(long)] + to_block_height: Option, } impl AnalyseGasUsageCommand { @@ -50,8 +58,22 @@ impl AnalyseGasUsageCommand { EpochManager::new_from_genesis_config(store, &near_config.genesis.config).unwrap(); // Create an iterator over the blocks that should be analysed - let last_blocks_to_analyse: u64 = self.last_blocks.unwrap_or(100); - let blocks_iter = LastNBlocksIterator::new(last_blocks_to_analyse, chain_store.clone()); + let blocks_iter_opt = crate::block_iterators::make_block_iterator_from_command_args( + CommandArgs { + last_blocks: self.last_blocks, + from_block_height: self.from_block_height, + to_block_height: self.to_block_height, + }, + chain_store.clone(), + ); + + let blocks_iter = match blocks_iter_opt { + Some(iter) => iter, + None => { + println!("No arguments, defaulting to last 100 blocks"); + Box::new(LastNBlocksIterator::new(100, chain_store.clone())) + } + }; // Analyse analyse_gas_usage(blocks_iter, &chain_store, &epoch_manager); diff --git a/tools/database/src/block_iterators/height_range.rs b/tools/database/src/block_iterators/height_range.rs new file mode 100644 index 00000000000..5be7cfa7073 --- /dev/null +++ b/tools/database/src/block_iterators/height_range.rs @@ -0,0 +1,83 @@ +use std::rc::Rc; + +use near_chain::{Block, ChainStore, ChainStoreAccess, Error}; +use near_primitives::{hash::CryptoHash, types::BlockHeight}; + +/// Iterate over blocks between two block heights. +/// `from_height` and `to_height` are inclusive +pub struct BlockHeightRangeIterator { + chain_store: Rc, + from_block_height: BlockHeight, + /// Hash of the block that will be returned when next() is called + current_block_hash: Option, +} + +impl BlockHeightRangeIterator { + /// Create an iterator which iterates over blocks between from_height and to_height. + /// `from_height` and `to_height` are inclusive. + /// Both arguments are optional, passing `None`` means that the limit is +- infinity. + pub fn new( + from_height_opt: Option, + to_height_opt: Option, + chain_store: Rc, + ) -> BlockHeightRangeIterator { + if let (Some(from), Some(to)) = (&from_height_opt, &to_height_opt) { + if *from > *to { + // Empty iterator + return BlockHeightRangeIterator { + chain_store, + from_block_height: 0, + current_block_hash: None, + }; + } + } + + let min_height: BlockHeight = chain_store.get_genesis_height(); + let max_height: BlockHeight = chain_store.head().unwrap().height; + + let from_height: BlockHeight = from_height_opt.unwrap_or(min_height); + let mut to_height: BlockHeight = to_height_opt.unwrap_or(max_height); + + // There is no point in going over nonexisting blocks past the highest height + if to_height > max_height { + to_height = max_height; + } + + // A block with height `to_height` might not exist. + // Go over the range in reverse and find the highest block that exists. + let mut current_block_hash: Option = None; + for height in (from_height..=to_height).rev() { + match chain_store.get_block_hash_by_height(height) { + Ok(hash) => { + current_block_hash = Some(hash); + break; + } + Err(Error::DBNotFoundErr(_)) => continue, + err => err.unwrap(), + }; + } + + BlockHeightRangeIterator { chain_store, from_block_height: from_height, current_block_hash } + } +} + +impl Iterator for BlockHeightRangeIterator { + type Item = Block; + + fn next(&mut self) -> Option { + if let Some(hash) = self.current_block_hash.take() { + let current_block = self.chain_store.get_block(&hash).unwrap(); + // Make sure that the block is within the from..=to range, stop iterating if it isn't + if current_block.header().height() >= self.from_block_height { + // Set the previous block as "current" one, as long as the current one isn't the genesis block + if current_block.header().height() != self.chain_store.get_genesis_height() { + self.current_block_hash = Some(*current_block.header().prev_hash()); + } + + return Some(current_block); + } + } + + None + } +} diff --git a/tools/database/src/block_iterators/mod.rs b/tools/database/src/block_iterators/mod.rs index e293d1d8968..f8c49c0995e 100644 --- a/tools/database/src/block_iterators/mod.rs +++ b/tools/database/src/block_iterators/mod.rs @@ -1,6 +1,61 @@ //! This module contains iterators that can be used to iterate over blocks in the database -pub mod last_blocks; +mod height_range; +mod last_blocks; + +use std::rc::Rc; + +use near_chain::{Block, ChainStore}; +use near_primitives::types::BlockHeight; + +/// Iterate over blocks between two block heights. +/// `from_height` and `to_height` are inclusive +pub use height_range::BlockHeightRangeIterator; /// Iterate over the last N blocks in the blockchain pub use last_blocks::LastNBlocksIterator; + +/// Arguments that user can pass to a command to choose some subset of blocks +pub struct CommandArgs { + /// Analyse the last N blocks + pub last_blocks: Option, + + /// Analyse blocks from the given block height, inclusive + pub from_block_height: Option, + + /// Analyse blocks up to the given block height, inclusive + pub to_block_height: Option, +} + +/// Produce to right iterator for a given set of command line arguments +pub fn make_block_iterator_from_command_args( + command_args: CommandArgs, + chain_store: Rc, +) -> Option>> { + // Make sure that only one type of argument is used (there is no mixing of last_blocks and from_block_height) + let mut arg_types_used: u64 = 0; + if command_args.last_blocks.is_some() { + arg_types_used += 1; + } + if command_args.from_block_height.is_some() || command_args.from_block_height.is_some() { + arg_types_used += 1; + } + + if arg_types_used > 1 { + panic!("It is illegal to mix multiple types of arguments specifying a subset of blocks"); + } + + if let Some(last_blocks) = command_args.last_blocks { + return Some(Box::new(LastNBlocksIterator::new(last_blocks, chain_store))); + } + + if command_args.from_block_height.is_some() || command_args.to_block_height.is_some() { + return Some(Box::new(BlockHeightRangeIterator::new( + command_args.from_block_height, + command_args.to_block_height, + chain_store, + ))); + } + + None +} From fe6875d8130ae417d02c353bd45fa8adaadd0dda Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Thu, 23 Nov 2023 15:59:04 +0100 Subject: [PATCH 05/24] Calculate the optimal split for each shard For each shard, calculate the optimal account that the shard could be split at, so that gas usage on both sides of the split is similar. An exact split isn't always possible, because it's possible for one account to consume 40% of all gas, but the function tries its best to make it fair. Information about optimal splits is displayed for the user. --- tools/database/src/analyse_gas_usage.rs | 56 +++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tools/database/src/analyse_gas_usage.rs b/tools/database/src/analyse_gas_usage.rs index 71b96302b84..7c416c7fb7e 100644 --- a/tools/database/src/analyse_gas_usage.rs +++ b/tools/database/src/analyse_gas_usage.rs @@ -88,6 +88,18 @@ struct GasUsageInShard { pub used_gas_total: BigGas, } +/// A shard can be split into two halves. +/// This struct represents the result of splitting a shard at `split_account`. +#[derive(Debug, Clone)] +struct ShardSplit { + /// Account on which the shard would be split + pub split_account: AccountId, + /// Gas used by accounts < split_account + pub gas_left: BigGas, + /// Gas used by accounts >= split_account + pub gas_right: BigGas, +} + impl GasUsageInShard { pub fn new() -> GasUsageInShard { GasUsageInShard { used_gas_per_account: BTreeMap::new(), used_gas_total: 0 } @@ -107,6 +119,33 @@ impl GasUsageInShard { } self.used_gas_total = self.used_gas_total.checked_add(other.used_gas_total).unwrap(); } + + /// Calculate the optimal point at which this shard could be split into two halves with similar gas usage + pub fn calculate_split(&self) -> Option { + let mut split_account = match self.used_gas_per_account.keys().next() { + Some(account_id) => account_id, + None => return None, + }; + + if self.used_gas_per_account.len() < 2 { + return None; + } + + let mut gas_left: BigGas = 0; + let mut gas_right: BigGas = self.used_gas_total; + + for (account, used_gas) in self.used_gas_per_account.iter() { + if gas_left >= gas_right { + break; + } + + split_account = &account; + gas_left = gas_left.checked_add(*used_gas).unwrap(); + gas_right = gas_right.checked_sub(*used_gas).unwrap(); + } + + Some(ShardSplit { split_account: split_account.clone(), gas_left, gas_right }) + } } #[derive(Clone, Debug)] @@ -245,6 +284,23 @@ fn analyse_gas_usage( as_percentage_of(shard_usage.used_gas_total, total_gas) ); println!(" Number of accounts: {}", shard_usage.used_gas_per_account.len()); + match shard_usage.calculate_split() { + Some(shard_split) => { + println!(" Optimal split:"); + println!(" split_account: {}", shard_split.split_account); + println!( + " gas(account < split_account): {} ({} of shard)", + shard_split.gas_left, + as_percentage_of(shard_split.gas_left, shard_usage.used_gas_total) + ); + println!( + " gas(account >= split_account): {} ({} of shard)", + shard_split.gas_right, + as_percentage_of(shard_split.gas_right, shard_usage.used_gas_total) + ); + } + None => println!(" No optimal split for this shard"), + } println!(""); } } From b7d4709249c3f63bddf6226168bdeb20ecd4caad Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Thu, 23 Nov 2023 16:00:51 +0100 Subject: [PATCH 06/24] Find 10 biggest accounts by gas usage Find the accounts that consume the most gas and display them in the analysis. This information is interesting, because it might turn out that one account consumes 50% of all gas. --- tools/database/src/analyse_gas_usage.rs | 44 ++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/tools/database/src/analyse_gas_usage.rs b/tools/database/src/analyse_gas_usage.rs index 7c416c7fb7e..e38ff30281b 100644 --- a/tools/database/src/analyse_gas_usage.rs +++ b/tools/database/src/analyse_gas_usage.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::path::PathBuf; use std::rc::Rc; use std::sync::Arc; @@ -225,6 +225,31 @@ fn get_gas_usage_in_block( result } +/// A struct that can be used to find N biggest accounts by gas usage in an efficient manner. +struct BiggestAccountsFinder { + accounts: BTreeSet<(BigGas, AccountId)>, + accounts_num: usize, +} + +impl BiggestAccountsFinder { + pub fn new(accounts_num: usize) -> BiggestAccountsFinder { + BiggestAccountsFinder { accounts: BTreeSet::new(), accounts_num } + } + + pub fn add_account_stats(&mut self, account: AccountId, used_gas: BigGas) { + self.accounts.insert((used_gas, account)); + + // If there are more accounts than desired, remove the one with the smallest gas usage + while self.accounts.len() > self.accounts_num { + self.accounts.pop_first(); + } + } + + pub fn get_biggest_accounts(&self) -> impl Iterator + '_ { + self.accounts.iter().rev().map(|(gas, account)| (account.clone(), *gas)) + } +} + fn analyse_gas_usage( blocks_iter: impl Iterator, chain_store: &ChainStore, @@ -303,4 +328,21 @@ fn analyse_gas_usage( } println!(""); } + + // Find 10 biggest accounts by gas usage + let mut biggest_accounts_finder = BiggestAccountsFinder::new(10); + for shard in gas_usage_stats.shards.values() { + for (account, used_gas) in &shard.used_gas_per_account { + biggest_accounts_finder.add_account_stats(account.clone(), *used_gas); + } + } + println!("10 biggest accounts by gas usage:"); + for (i, (account, gas_usage)) in biggest_accounts_finder.get_biggest_accounts().enumerate() { + println!("#{}: {}", i + 1, account); + println!( + " Used gas: {} ({} of total)", + gas_usage, + as_percentage_of(gas_usage, total_gas) + ) + } } From 555b94724e0a9b389cec420bdc1ea30764e75391 Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Thu, 23 Nov 2023 17:58:07 +0100 Subject: [PATCH 07/24] run scripts/fix_ngihtly_feature_flags.py --- tools/database/Cargo.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/database/Cargo.toml b/tools/database/Cargo.toml index ad3798b6cb9..d4526f18c6d 100644 --- a/tools/database/Cargo.toml +++ b/tools/database/Cargo.toml @@ -33,6 +33,7 @@ near-primitives.workspace = true nightly = [ "nightly_protocol", "near-chain-configs/nightly", + "near-chain/nightly", "near-epoch-manager/nightly", "near-primitives/nightly", "near-store/nightly", @@ -40,6 +41,7 @@ nightly = [ ] nightly_protocol = [ "near-chain-configs/nightly_protocol", + "near-chain/nightly_protocol", "near-epoch-manager/nightly_protocol", "near-primitives/nightly_protocol", "near-store/nightly_protocol", From 0e8c59a56614aa0309e61d0d5f8b0ef8fc34a204 Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Fri, 24 Nov 2023 13:55:30 +0100 Subject: [PATCH 08/24] Display gas amounts in TGas --- tools/database/src/analyse_gas_usage.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tools/database/src/analyse_gas_usage.rs b/tools/database/src/analyse_gas_usage.rs index e38ff30281b..537864a04ab 100644 --- a/tools/database/src/analyse_gas_usage.rs +++ b/tools/database/src/analyse_gas_usage.rs @@ -25,6 +25,12 @@ use crate::block_iterators::{CommandArgs, LastNBlocksIterator}; /// To avoid overflows, let's use `BigGas` for storing gas amounts in the code. type BigGas = u128; +/// Display gas amount in a human-friendly way +fn display_gas(gas: BigGas) -> String { + let tera_gas = gas as f64 / 1e12; + format!("{:.2} TGas", tera_gas) +} + #[derive(Parser)] pub(crate) struct AnalyseGasUsageCommand { /// Analyse the last N blocks in the blockchain @@ -299,13 +305,13 @@ fn analyse_gas_usage( } let total_gas: BigGas = gas_usage_stats.used_gas_total(); println!(""); - println!("Total gas used: {}", total_gas); + println!("Total gas used: {}", display_gas(total_gas)); println!(""); for (shard_uid, shard_usage) in &gas_usage_stats.shards { println!("Shard: {}", shard_uid); println!( " Gas usage: {} ({} of total)", - shard_usage.used_gas_total, + display_gas(shard_usage.used_gas_total), as_percentage_of(shard_usage.used_gas_total, total_gas) ); println!(" Number of accounts: {}", shard_usage.used_gas_per_account.len()); @@ -315,12 +321,12 @@ fn analyse_gas_usage( println!(" split_account: {}", shard_split.split_account); println!( " gas(account < split_account): {} ({} of shard)", - shard_split.gas_left, + display_gas(shard_split.gas_left), as_percentage_of(shard_split.gas_left, shard_usage.used_gas_total) ); println!( " gas(account >= split_account): {} ({} of shard)", - shard_split.gas_right, + display_gas(shard_split.gas_right), as_percentage_of(shard_split.gas_right, shard_usage.used_gas_total) ); } @@ -341,7 +347,7 @@ fn analyse_gas_usage( println!("#{}: {}", i + 1, account); println!( " Used gas: {} ({} of total)", - gas_usage, + display_gas(gas_usage), as_percentage_of(gas_usage, total_gas) ) } From 473747fbb30c5f03804bc85637312ad0e414088a Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Fri, 24 Nov 2023 13:59:31 +0100 Subject: [PATCH 09/24] Extract as_percentage_of to a separate function --- tools/database/src/analyse_gas_usage.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tools/database/src/analyse_gas_usage.rs b/tools/database/src/analyse_gas_usage.rs index 537864a04ab..47622a4dca2 100644 --- a/tools/database/src/analyse_gas_usage.rs +++ b/tools/database/src/analyse_gas_usage.rs @@ -256,6 +256,16 @@ impl BiggestAccountsFinder { } } +// Calculates how much percent of `big` is `small` and returns it as a string. +// Example: as_percentage_of(10, 100) == "10.0%" +fn as_percentage_of(small: BigGas, big: BigGas) -> String { + if big > 0 { + format!("{:.1}%", small as f64 / big as f64 * 100.0) + } else { + format!("-") + } +} + fn analyse_gas_usage( blocks_iter: impl Iterator, chain_store: &ChainStore, @@ -280,16 +290,6 @@ fn analyse_gas_usage( gas_usage_stats.merge(gas_usage_in_block); } - // Calculates how much percent of `big` is `small` and returns it as a string. - // Example: as_percentage_of(10, 100) == "10.0%" - let as_percentage_of = |small: BigGas, big: BigGas| { - if big > 0 { - format!("{:.1}%", small as f64 / big as f64 * 100.0) - } else { - format!("-") - } - }; - // Print out the analysis if blocks_count == 0 { println!("No blocks to analyse!"); From 29190112102f2234420cd548d446799f9b72aa74 Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Fri, 24 Nov 2023 16:53:48 +0100 Subject: [PATCH 10/24] fix: fix calculation of used_gas_total in GasUsageInShard::merge() GasUsageInShard::merge() merges results from two instances of GasUsageInShard into one. The merge is performed by adding information about each account from the other shard. Then used_gas_total is updated as well. However this is the wrong thing to do. Calling `add_used_gas` already increases `used_gas_total`, so doing it again doubles the total gas, which is incorrect. Fix the bug by removing the line that doubles the gas. --- tools/database/src/analyse_gas_usage.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/database/src/analyse_gas_usage.rs b/tools/database/src/analyse_gas_usage.rs index 47622a4dca2..ffa28d88691 100644 --- a/tools/database/src/analyse_gas_usage.rs +++ b/tools/database/src/analyse_gas_usage.rs @@ -123,7 +123,6 @@ impl GasUsageInShard { for (account_id, used_gas) in &other.used_gas_per_account { self.add_used_gas(account_id.clone(), *used_gas); } - self.used_gas_total = self.used_gas_total.checked_add(other.used_gas_total).unwrap(); } /// Calculate the optimal point at which this shard could be split into two halves with similar gas usage From f087966962212c0a01b77d0a094d9fd6d7898325 Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Fri, 24 Nov 2023 17:00:31 +0100 Subject: [PATCH 11/24] Remove GasUsageInShard::used_gas_total The field GasUsageInShard::used_gas_total keeps the total amount of gas used in a shard. The problem is that keeping it in sync with the rest of the struct requires special care and is prone to bugs. A mistake in this logic has already caused one bug in calculating shard splits. It's safer to get rid of the field and just calculate the total amount of used gas when needed. It doesn't take that long to calculate this value from scratch. --- tools/database/src/analyse_gas_usage.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tools/database/src/analyse_gas_usage.rs b/tools/database/src/analyse_gas_usage.rs index ffa28d88691..50a2a08ae50 100644 --- a/tools/database/src/analyse_gas_usage.rs +++ b/tools/database/src/analyse_gas_usage.rs @@ -91,7 +91,6 @@ impl AnalyseGasUsageCommand { #[derive(Clone, Debug, Default)] struct GasUsageInShard { pub used_gas_per_account: BTreeMap, - pub used_gas_total: BigGas, } /// A shard can be split into two halves. @@ -108,15 +107,21 @@ struct ShardSplit { impl GasUsageInShard { pub fn new() -> GasUsageInShard { - GasUsageInShard { used_gas_per_account: BTreeMap::new(), used_gas_total: 0 } + GasUsageInShard { used_gas_per_account: BTreeMap::new() } } pub fn add_used_gas(&mut self, account: AccountId, used_gas: BigGas) { let old_used_gas: &BigGas = self.used_gas_per_account.get(&account).unwrap_or(&0); let new_used_gas: BigGas = old_used_gas.checked_add(used_gas).unwrap(); self.used_gas_per_account.insert(account, new_used_gas); + } - self.used_gas_total = self.used_gas_total.checked_add(used_gas).unwrap(); + pub fn used_gas_total(&self) -> BigGas { + let mut result: BigGas = 0; + for used_gas in self.used_gas_per_account.values() { + result = result.checked_add(*used_gas).unwrap(); + } + result } pub fn merge(&mut self, other: &GasUsageInShard) { @@ -137,7 +142,7 @@ impl GasUsageInShard { } let mut gas_left: BigGas = 0; - let mut gas_right: BigGas = self.used_gas_total; + let mut gas_right: BigGas = self.used_gas_total(); for (account, used_gas) in self.used_gas_per_account.iter() { if gas_left >= gas_right { @@ -175,7 +180,7 @@ impl GasUsageStats { pub fn used_gas_total(&self) -> BigGas { let mut result: BigGas = 0; for shard_usage in self.shards.values() { - result = result.checked_add(shard_usage.used_gas_total).unwrap(); + result = result.checked_add(shard_usage.used_gas_total()).unwrap(); } result } @@ -310,8 +315,8 @@ fn analyse_gas_usage( println!("Shard: {}", shard_uid); println!( " Gas usage: {} ({} of total)", - display_gas(shard_usage.used_gas_total), - as_percentage_of(shard_usage.used_gas_total, total_gas) + display_gas(shard_usage.used_gas_total()), + as_percentage_of(shard_usage.used_gas_total(), total_gas) ); println!(" Number of accounts: {}", shard_usage.used_gas_per_account.len()); match shard_usage.calculate_split() { @@ -321,12 +326,12 @@ fn analyse_gas_usage( println!( " gas(account < split_account): {} ({} of shard)", display_gas(shard_split.gas_left), - as_percentage_of(shard_split.gas_left, shard_usage.used_gas_total) + as_percentage_of(shard_split.gas_left, shard_usage.used_gas_total()) ); println!( " gas(account >= split_account): {} ({} of shard)", display_gas(shard_split.gas_right), - as_percentage_of(shard_split.gas_right, shard_usage.used_gas_total) + as_percentage_of(shard_split.gas_right, shard_usage.used_gas_total()) ); } None => println!(" No optimal split for this shard"), From f01c80e4e2d5d2cabac15b1bb44ba0a4ab393242 Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Fri, 24 Nov 2023 17:57:21 +0100 Subject: [PATCH 12/24] Improve GasUsageInShard::calculate_split() The previous implementation of calculate_split() worked okayish, but it had some faults. The first fault was that it split the shard into two halves where the left one consistend of accounts smaller than the boundary account. This doesn't match the logic that NEAR uses for boundary accounts. In NEAR the left half includes the boundary account, i.e it's <= boundary_acc, not < boundary_acc. The new function uses the same division as NEAR: (left <= split_account), (right > split_account). The second fault was that the old function looked for a split where gas_left >= gas_right. It isn't easy to prove that this is the optimal one - why is the left half supposed to be bigger? Let's implement it in a way that is easier to understand. The new calculate_split() looks for a split that minimizes the difference between the two halves. This makes sense, we want the halves to be as similar as possible. The commit also adds unit tests to make sure that calculate_split() works correctly. --- tools/database/src/analyse_gas_usage.rs | 205 +++++++++++++++++++++--- 1 file changed, 185 insertions(+), 20 deletions(-) diff --git a/tools/database/src/analyse_gas_usage.rs b/tools/database/src/analyse_gas_usage.rs index 50a2a08ae50..6a5ab04b525 100644 --- a/tools/database/src/analyse_gas_usage.rs +++ b/tools/database/src/analyse_gas_usage.rs @@ -95,13 +95,13 @@ struct GasUsageInShard { /// A shard can be split into two halves. /// This struct represents the result of splitting a shard at `split_account`. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] struct ShardSplit { /// Account on which the shard would be split pub split_account: AccountId, - /// Gas used by accounts < split_account + /// Gas used by accounts <= split_account pub gas_left: BigGas, - /// Gas used by accounts >= split_account + /// Gas used by accounts > split_account pub gas_right: BigGas, } @@ -132,29 +132,32 @@ impl GasUsageInShard { /// Calculate the optimal point at which this shard could be split into two halves with similar gas usage pub fn calculate_split(&self) -> Option { - let mut split_account = match self.used_gas_per_account.keys().next() { - Some(account_id) => account_id, - None => return None, - }; - - if self.used_gas_per_account.len() < 2 { + let total_gas: BigGas = self.used_gas_total(); + if total_gas == 0 || self.used_gas_per_account.len() < 2 { return None; } - let mut gas_left: BigGas = 0; - let mut gas_right: BigGas = self.used_gas_total(); + // Find a split with the smallest difference between the two halves + let mut best_split: Option = None; + let mut best_difference: BigGas = total_gas; - for (account, used_gas) in self.used_gas_per_account.iter() { - if gas_left >= gas_right { - break; - } + let mut gas_left: BigGas = 0; + let mut gas_right: BigGas = total_gas; - split_account = &account; + for (account, used_gas) in &self.used_gas_per_account { + // We are now considering a split of (left <= account) and (right > account), + // so the current gas should be included in the left part of the split gas_left = gas_left.checked_add(*used_gas).unwrap(); gas_right = gas_right.checked_sub(*used_gas).unwrap(); - } - Some(ShardSplit { split_account: split_account.clone(), gas_left, gas_right }) + let difference: BigGas = gas_left.abs_diff(gas_right); + if difference < best_difference { + best_difference = difference; + best_split = + Some(ShardSplit { split_account: account.clone(), gas_left, gas_right }); + } + } + best_split } } @@ -324,12 +327,12 @@ fn analyse_gas_usage( println!(" Optimal split:"); println!(" split_account: {}", shard_split.split_account); println!( - " gas(account < split_account): {} ({} of shard)", + " gas(account <= split_account): {} ({} of shard)", display_gas(shard_split.gas_left), as_percentage_of(shard_split.gas_left, shard_usage.used_gas_total()) ); println!( - " gas(account >= split_account): {} ({} of shard)", + " gas(account > split_account): {} ({} of shard)", display_gas(shard_split.gas_right), as_percentage_of(shard_split.gas_right, shard_usage.used_gas_total()) ); @@ -356,3 +359,165 @@ fn analyse_gas_usage( ) } } + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use near_primitives::types::AccountId; + + use super::{GasUsageInShard, ShardSplit}; + + fn account(name: &str) -> AccountId { + AccountId::from_str(&format!("{name}.near")).unwrap() + } + + // There is no optimal split for a shard with no accounts + #[test] + fn empty_shard_no_split() { + let empty_shard = GasUsageInShard::new(); + assert_eq!(empty_shard.calculate_split(), None); + } + + // There is no optimal split for a shard with a single account + #[test] + fn one_account_no_split() { + let mut shard_usage = GasUsageInShard::new(); + + shard_usage.add_used_gas(account("a"), 12345); + + assert_eq!(shard_usage.calculate_split(), None); + } + + // A shard with two equally sized accounts should be split in half + #[test] + fn two_accounts_equal_split() { + let mut shard_usage = GasUsageInShard::new(); + + shard_usage.add_used_gas(account("a"), 12345); + shard_usage.add_used_gas(account("b"), 12345); + + let optimal_split = + ShardSplit { split_account: account("a"), gas_left: 12345, gas_right: 12345 }; + assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); + } + + // A shard with two accounts where the first is slightly smaller should be split in half + #[test] + fn two_accounts_first_smaller_split() { + let mut shard_usage = GasUsageInShard::new(); + + shard_usage.add_used_gas(account("a"), 123); + shard_usage.add_used_gas(account("b"), 12345); + + let optimal_split = + ShardSplit { split_account: account("a"), gas_left: 123, gas_right: 12345 }; + assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); + } + + // A shard with two accounts where the second one is slightly smaller should be split in half + #[test] + fn two_accounts_second_smaller_split() { + let mut shard_usage = GasUsageInShard::new(); + + shard_usage.add_used_gas(account("a"), 12345); + shard_usage.add_used_gas(account("b"), 123); + + let optimal_split = + ShardSplit { split_account: account("a"), gas_left: 12345, gas_right: 123 }; + assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); + } + + // A shard with multiple accounts where all of them use 0 gas has optimal split + #[test] + fn many_accounts_zero_no_split() { + let mut shard_usage = GasUsageInShard::new(); + + shard_usage.add_used_gas(account("a"), 0); + shard_usage.add_used_gas(account("b"), 0); + shard_usage.add_used_gas(account("c"), 0); + shard_usage.add_used_gas(account("d"), 0); + shard_usage.add_used_gas(account("e"), 0); + shard_usage.add_used_gas(account("f"), 0); + shard_usage.add_used_gas(account("g"), 0); + shard_usage.add_used_gas(account("h"), 0); + + assert_eq!(shard_usage.calculate_split(), None); + } + + // A shard with multiple accounts where only one is nonzero has no optimal split + #[test] + fn many_accounts_one_nonzero_no_split() { + let mut shard_usage = GasUsageInShard::new(); + + shard_usage.add_used_gas(account("a"), 0); + shard_usage.add_used_gas(account("b"), 0); + shard_usage.add_used_gas(account("c"), 0); + shard_usage.add_used_gas(account("d"), 0); + shard_usage.add_used_gas(account("e"), 12345); + shard_usage.add_used_gas(account("f"), 0); + shard_usage.add_used_gas(account("g"), 0); + shard_usage.add_used_gas(account("h"), 0); + + assert_eq!(shard_usage.calculate_split(), None); + } + + // An example set of accounts is split correctly + #[test] + fn many_accounts_split() { + let mut shard_usage = GasUsageInShard::new(); + + shard_usage.add_used_gas(account("a"), 1); + shard_usage.add_used_gas(account("b"), 3); + shard_usage.add_used_gas(account("c"), 5); + shard_usage.add_used_gas(account("d"), 2); + shard_usage.add_used_gas(account("e"), 8); + shard_usage.add_used_gas(account("f"), 1); + shard_usage.add_used_gas(account("g"), 2); + shard_usage.add_used_gas(account("h"), 8); + + // Optimal split: + // 1 + 3 + 5 + 2 = 11 + // 8 + 1 + 2 + 8 = 19 + let optimal_split = ShardSplit { split_account: account("d"), gas_left: 11, gas_right: 19 }; + assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); + } + + // The first account uses the most gas, it should be the only one in its half of the split + #[test] + fn first_heavy_split() { + let mut shard_usage = GasUsageInShard::new(); + + shard_usage.add_used_gas(account("a"), 10000); + shard_usage.add_used_gas(account("b"), 1); + shard_usage.add_used_gas(account("c"), 1); + shard_usage.add_used_gas(account("d"), 1); + shard_usage.add_used_gas(account("e"), 1); + shard_usage.add_used_gas(account("f"), 1); + shard_usage.add_used_gas(account("g"), 1); + shard_usage.add_used_gas(account("h"), 1); + + let optimal_split = + ShardSplit { split_account: account("a"), gas_left: 10000, gas_right: 7 }; + assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); + } + + // The last account uses the most gas, it should be the only one in its half of the split + #[test] + fn last_heavy_split() { + let mut shard_usage = GasUsageInShard::new(); + + shard_usage.add_used_gas(account("a"), 1); + shard_usage.add_used_gas(account("b"), 1); + shard_usage.add_used_gas(account("c"), 1); + shard_usage.add_used_gas(account("d"), 1); + shard_usage.add_used_gas(account("e"), 1); + shard_usage.add_used_gas(account("f"), 1); + shard_usage.add_used_gas(account("g"), 1); + shard_usage.add_used_gas(account("h"), 10000); + + let optimal_split = + ShardSplit { split_account: account("g"), gas_left: 7, gas_right: 10000 }; + assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); + } +} From d168c5d7c0f9e959afea54f3638ceea5b0bcc317 Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Fri, 24 Nov 2023 18:21:34 +0100 Subject: [PATCH 13/24] Import some things that weren't imported before --- tools/database/src/analyse_gas_usage.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tools/database/src/analyse_gas_usage.rs b/tools/database/src/analyse_gas_usage.rs index 6a5ab04b525..0bd2c46d158 100644 --- a/tools/database/src/analyse_gas_usage.rs +++ b/tools/database/src/analyse_gas_usage.rs @@ -5,7 +5,9 @@ use std::sync::Arc; use clap::Parser; use near_chain::{Block, ChainStore}; +use near_chain_configs::GenesisValidationMode; use near_epoch_manager::EpochManager; +use nearcore::config::load_config; use near_primitives::epoch_manager::block_info::BlockInfo; use near_primitives::hash::CryptoHash; @@ -16,7 +18,9 @@ use near_primitives::types::{AccountId, BlockHeight, EpochId, ShardId}; use near_store::{NodeStorage, ShardUId, Store}; use nearcore::open_storage; -use crate::block_iterators::{CommandArgs, LastNBlocksIterator}; +use crate::block_iterators::{ + make_block_iterator_from_command_args, CommandArgs, LastNBlocksIterator, +}; /// `Gas` is an u64, but it stil might overflow when analysing a large amount of blocks. /// 1ms of compute is about 1TGas = 10^12 gas. One epoch takes 43200 seconds (43200000ms). @@ -49,9 +53,7 @@ pub(crate) struct AnalyseGasUsageCommand { impl AnalyseGasUsageCommand { pub(crate) fn run(&self, home: &PathBuf) -> anyhow::Result<()> { // Create a ChainStore and EpochManager that will be used to read blockchain data. - let mut near_config = - nearcore::config::load_config(home, near_chain_configs::GenesisValidationMode::Full) - .unwrap(); + let mut near_config = load_config(home, GenesisValidationMode::Full).unwrap(); let node_storage: NodeStorage = open_storage(&home, &mut near_config).unwrap(); let store: Store = node_storage.get_split_store().unwrap_or_else(|| node_storage.get_hot_store()); @@ -64,7 +66,7 @@ impl AnalyseGasUsageCommand { EpochManager::new_from_genesis_config(store, &near_config.genesis.config).unwrap(); // Create an iterator over the blocks that should be analysed - let blocks_iter_opt = crate::block_iterators::make_block_iterator_from_command_args( + let blocks_iter_opt = make_block_iterator_from_command_args( CommandArgs { last_blocks: self.last_blocks, from_block_height: self.from_block_height, From b9469dd02d31448f77d207fb49f0de2e9765953c Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Fri, 24 Nov 2023 18:27:44 +0100 Subject: [PATCH 14/24] Use entry API in GasUsageInShard::add_used_gas() --- tools/database/src/analyse_gas_usage.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/database/src/analyse_gas_usage.rs b/tools/database/src/analyse_gas_usage.rs index 0bd2c46d158..8c6a5e269c7 100644 --- a/tools/database/src/analyse_gas_usage.rs +++ b/tools/database/src/analyse_gas_usage.rs @@ -113,9 +113,8 @@ impl GasUsageInShard { } pub fn add_used_gas(&mut self, account: AccountId, used_gas: BigGas) { - let old_used_gas: &BigGas = self.used_gas_per_account.get(&account).unwrap_or(&0); - let new_used_gas: BigGas = old_used_gas.checked_add(used_gas).unwrap(); - self.used_gas_per_account.insert(account, new_used_gas); + let account_gas = self.used_gas_per_account.entry(account).or_insert(0); + *account_gas = account_gas.checked_add(used_gas).unwrap(); } pub fn used_gas_total(&self) -> BigGas { From 367e8f39b1b005fb196f04e8d6894fb8b90b1b78 Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Fri, 24 Nov 2023 18:31:09 +0100 Subject: [PATCH 15/24] Reduce nesting depth in BlockHeightRangeIterator::next() --- .../src/block_iterators/height_range.rs | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tools/database/src/block_iterators/height_range.rs b/tools/database/src/block_iterators/height_range.rs index 5be7cfa7073..51a0511f822 100644 --- a/tools/database/src/block_iterators/height_range.rs +++ b/tools/database/src/block_iterators/height_range.rs @@ -65,17 +65,20 @@ impl Iterator for BlockHeightRangeIterator { type Item = Block; fn next(&mut self) -> Option { - if let Some(hash) = self.current_block_hash.take() { - let current_block = self.chain_store.get_block(&hash).unwrap(); - // Make sure that the block is within the from..=to range, stop iterating if it isn't - if current_block.header().height() >= self.from_block_height { - // Set the previous block as "current" one, as long as the current one isn't the genesis block - if current_block.header().height() != self.chain_store.get_genesis_height() { - self.current_block_hash = Some(*current_block.header().prev_hash()); - } + let current_block_hash: CryptoHash = match self.current_block_hash.take() { + Some(hash) => hash, + None => return None, + }; + let current_block = self.chain_store.get_block(¤t_block_hash).unwrap(); - return Some(current_block); + // Make sure that the block is within the from..=to range, stop iterating if it isn't + if current_block.header().height() >= self.from_block_height { + // Set the previous block as "current" one, as long as the current one isn't the genesis block + if current_block.header().height() != self.chain_store.get_genesis_height() { + self.current_block_hash = Some(*current_block.header().prev_hash()); } + + return Some(current_block); } None From c1641b208b123ef867e7630ab9bee19c66a2936c Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Fri, 24 Nov 2023 18:54:38 +0100 Subject: [PATCH 16/24] Display more information about each split shard --- tools/database/src/analyse_gas_usage.rs | 57 +++++++++++++++++++++---- 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/tools/database/src/analyse_gas_usage.rs b/tools/database/src/analyse_gas_usage.rs index 8c6a5e269c7..28d27f0799a 100644 --- a/tools/database/src/analyse_gas_usage.rs +++ b/tools/database/src/analyse_gas_usage.rs @@ -274,6 +274,40 @@ fn as_percentage_of(small: BigGas, big: BigGas) -> String { } } +fn display_shard_split_stats<'a>( + accounts: impl Iterator, + total_shard_gas: BigGas, +) { + let mut accounts_num: u64 = 0; + let mut total_gas: BigGas = 0; + let mut top_3_finder = BiggestAccountsFinder::new(3); + + for (account, used_gas) in accounts { + accounts_num += 1; + total_gas = total_gas.checked_add(*used_gas).unwrap(); + top_3_finder.add_account_stats(account.clone(), *used_gas); + } + + let indent = " "; + println!( + "{}Gas: {} ({} of shard total)", + indent, + display_gas(total_gas), + as_percentage_of(total_gas, total_shard_gas) + ); + println!("{}Accounts: {}", indent, accounts_num); + println!("{}Top 3 accounts:", indent); + for (i, (account, used_gas)) in top_3_finder.get_biggest_accounts().enumerate() { + println!("{} #{}: {}", indent, i + 1, account); + println!( + "{} Used gas: {} ({} of shard split half)", + indent, + display_gas(used_gas), + as_percentage_of(used_gas, total_gas) + ) + } +} + fn analyse_gas_usage( blocks_iter: impl Iterator, chain_store: &ChainStore, @@ -317,10 +351,11 @@ fn analyse_gas_usage( println!(""); for (shard_uid, shard_usage) in &gas_usage_stats.shards { println!("Shard: {}", shard_uid); + let shard_total_gas: BigGas = shard_usage.used_gas_total(); println!( " Gas usage: {} ({} of total)", display_gas(shard_usage.used_gas_total()), - as_percentage_of(shard_usage.used_gas_total(), total_gas) + as_percentage_of(shard_total_gas, total_gas) ); println!(" Number of accounts: {}", shard_usage.used_gas_per_account.len()); match shard_usage.calculate_split() { @@ -328,15 +363,19 @@ fn analyse_gas_usage( println!(" Optimal split:"); println!(" split_account: {}", shard_split.split_account); println!( - " gas(account <= split_account): {} ({} of shard)", - display_gas(shard_split.gas_left), - as_percentage_of(shard_split.gas_left, shard_usage.used_gas_total()) - ); - println!( - " gas(account > split_account): {} ({} of shard)", - display_gas(shard_split.gas_right), - as_percentage_of(shard_split.gas_right, shard_usage.used_gas_total()) + " gas(splitaccount): {}", + display_gas( + *shard_usage.used_gas_per_account.get(&shard_split.split_account).unwrap() + ) ); + println!(" Left (account <= split_account):"); + let left_accounts = + shard_usage.used_gas_per_account.range(..=shard_split.split_account.clone()); + display_shard_split_stats(left_accounts, shard_total_gas); + println!(" Right (account > split_account):"); + let right_accounts = + shard_usage.used_gas_per_account.range(shard_split.split_account..).skip(1); + display_shard_split_stats(right_accounts, shard_total_gas); } None => println!(" No optimal split for this shard"), } From 902728cb7feb82702b2c01461ccbfa465c335a0b Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Fri, 24 Nov 2023 19:08:29 +0100 Subject: [PATCH 17/24] Display the biggest account on each shard --- tools/database/src/analyse_gas_usage.rs | 28 ++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tools/database/src/analyse_gas_usage.rs b/tools/database/src/analyse_gas_usage.rs index 28d27f0799a..3cc3bfd665d 100644 --- a/tools/database/src/analyse_gas_usage.rs +++ b/tools/database/src/analyse_gas_usage.rs @@ -160,6 +160,24 @@ impl GasUsageInShard { } best_split } + + pub fn biggest_account(&self) -> Option<(AccountId, BigGas)> { + let mut result: Option<(AccountId, BigGas)> = None; + + for (account, used_gas) in &self.used_gas_per_account { + match &mut result { + None => result = Some((account.clone(), *used_gas)), + Some((best_account, best_gas)) => { + if *used_gas > *best_gas { + *best_account = account.clone(); + *best_gas = *used_gas + } + } + } + } + + result + } } #[derive(Clone, Debug)] @@ -358,12 +376,20 @@ fn analyse_gas_usage( as_percentage_of(shard_total_gas, total_gas) ); println!(" Number of accounts: {}", shard_usage.used_gas_per_account.len()); + if let Some((biggest_account, biggest_account_gas)) = shard_usage.biggest_account() { + println!(" Biggest account: {}", biggest_account); + println!( + " Biggest account gas: {} ({} of shard total)", + display_gas(biggest_account_gas), + as_percentage_of(biggest_account_gas, shard_total_gas) + ); + } match shard_usage.calculate_split() { Some(shard_split) => { println!(" Optimal split:"); println!(" split_account: {}", shard_split.split_account); println!( - " gas(splitaccount): {}", + " gas(split_account): {}", display_gas( *shard_usage.used_gas_per_account.get(&shard_split.split_account).unwrap() ) From 81b328a0f884f462b21d6bdacc5ca57549911829 Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Fri, 24 Nov 2023 19:27:58 +0100 Subject: [PATCH 18/24] Remove type annotations that aren't strictly needed --- tools/database/src/analyse_gas_usage.rs | 41 ++++++++----------- .../src/block_iterators/height_range.rs | 10 ++--- .../src/block_iterators/last_blocks.rs | 2 +- 3 files changed, 23 insertions(+), 30 deletions(-) diff --git a/tools/database/src/analyse_gas_usage.rs b/tools/database/src/analyse_gas_usage.rs index 3cc3bfd665d..4822ea8b30f 100644 --- a/tools/database/src/analyse_gas_usage.rs +++ b/tools/database/src/analyse_gas_usage.rs @@ -1,7 +1,6 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::PathBuf; use std::rc::Rc; -use std::sync::Arc; use clap::Parser; use near_chain::{Block, ChainStore}; @@ -9,13 +8,10 @@ use near_chain_configs::GenesisValidationMode; use near_epoch_manager::EpochManager; use nearcore::config::load_config; -use near_primitives::epoch_manager::block_info::BlockInfo; use near_primitives::hash::CryptoHash; -use near_primitives::shard_layout::{account_id_to_shard_id, ShardLayout}; -use near_primitives::transaction::ExecutionOutcome; -use near_primitives::types::{AccountId, BlockHeight, EpochId, ShardId}; +use near_primitives::shard_layout::{account_id_to_shard_id, ShardUId}; +use near_primitives::types::{AccountId, BlockHeight}; -use near_store::{NodeStorage, ShardUId, Store}; use nearcore::open_storage; use crate::block_iterators::{ @@ -54,9 +50,8 @@ impl AnalyseGasUsageCommand { pub(crate) fn run(&self, home: &PathBuf) -> anyhow::Result<()> { // Create a ChainStore and EpochManager that will be used to read blockchain data. let mut near_config = load_config(home, GenesisValidationMode::Full).unwrap(); - let node_storage: NodeStorage = open_storage(&home, &mut near_config).unwrap(); - let store: Store = - node_storage.get_split_store().unwrap_or_else(|| node_storage.get_hot_store()); + let node_storage = open_storage(&home, &mut near_config).unwrap(); + let store = node_storage.get_split_store().unwrap_or_else(|| node_storage.get_hot_store()); let chain_store = Rc::new(ChainStore::new( store.clone(), near_config.genesis.config.genesis_height, @@ -133,7 +128,7 @@ impl GasUsageInShard { /// Calculate the optimal point at which this shard could be split into two halves with similar gas usage pub fn calculate_split(&self) -> Option { - let total_gas: BigGas = self.used_gas_total(); + let total_gas = self.used_gas_total(); if total_gas == 0 || self.used_gas_per_account.len() < 2 { return None; } @@ -219,34 +214,33 @@ fn get_gas_usage_in_block( chain_store: &ChainStore, epoch_manager: &EpochManager, ) -> GasUsageStats { - let block_info: Arc = epoch_manager.get_block_info(block.hash()).unwrap(); - let epoch_id: &EpochId = block_info.epoch_id(); - let shard_layout: ShardLayout = epoch_manager.get_shard_layout(epoch_id).unwrap(); + let block_info = epoch_manager.get_block_info(block.hash()).unwrap(); + let epoch_id = block_info.epoch_id(); + let shard_layout = epoch_manager.get_shard_layout(epoch_id).unwrap(); let mut result = GasUsageStats::new(); // Go over every chunk in this block and gather data for chunk_header in block.chunks().iter() { - let shard_id: ShardId = chunk_header.shard_id(); - let shard_uid: ShardUId = ShardUId::from_shard_id_and_layout(shard_id, &shard_layout); + let shard_id = chunk_header.shard_id(); + let shard_uid = ShardUId::from_shard_id_and_layout(shard_id, &shard_layout); let mut gas_usage_in_shard = GasUsageInShard::new(); // The outcome of each transaction and receipt executed in this chunk is saved in the database as an ExecutionOutcome. // Go through all ExecutionOutcomes from this chunk and record the gas usage. - let outcome_ids: Vec = + let outcome_ids = chain_store.get_outcomes_by_block_hash_and_shard_id(block.hash(), shard_id).unwrap(); for outcome_id in outcome_ids { - let outcome: ExecutionOutcome = chain_store + let outcome = chain_store .get_outcome_by_id_and_block_hash(&outcome_id, block.hash()) .unwrap() .unwrap() .outcome; // Sanity check - make sure that the executor of this outcome belongs to this shard - let account_shard: ShardId = - account_id_to_shard_id(&outcome.executor_id, &shard_layout); - assert_eq!(account_shard, shard_id); + let account_shard_id = account_id_to_shard_id(&outcome.executor_id, &shard_layout); + assert_eq!(account_shard_id, shard_id); gas_usage_in_shard.add_used_gas(outcome.executor_id, outcome.gas_burnt.into()); } @@ -345,8 +339,7 @@ fn analyse_gas_usage( } last_analysed_block = Some((block.header().height(), *block.hash())); - let gas_usage_in_block: GasUsageStats = - get_gas_usage_in_block(&block, chain_store, epoch_manager); + let gas_usage_in_block = get_gas_usage_in_block(&block, chain_store, epoch_manager); gas_usage_stats.merge(gas_usage_in_block); } @@ -363,13 +356,13 @@ fn analyse_gas_usage( if let Some((block_height, block_hash)) = last_analysed_block { println!("Block: height = {block_height}, hash = {block_hash}"); } - let total_gas: BigGas = gas_usage_stats.used_gas_total(); + let total_gas = gas_usage_stats.used_gas_total(); println!(""); println!("Total gas used: {}", display_gas(total_gas)); println!(""); for (shard_uid, shard_usage) in &gas_usage_stats.shards { println!("Shard: {}", shard_uid); - let shard_total_gas: BigGas = shard_usage.used_gas_total(); + let shard_total_gas = shard_usage.used_gas_total(); println!( " Gas usage: {} ({} of total)", display_gas(shard_usage.used_gas_total()), diff --git a/tools/database/src/block_iterators/height_range.rs b/tools/database/src/block_iterators/height_range.rs index 51a0511f822..9b79675e4b1 100644 --- a/tools/database/src/block_iterators/height_range.rs +++ b/tools/database/src/block_iterators/height_range.rs @@ -32,11 +32,11 @@ impl BlockHeightRangeIterator { } } - let min_height: BlockHeight = chain_store.get_genesis_height(); - let max_height: BlockHeight = chain_store.head().unwrap().height; + let min_height = chain_store.get_genesis_height(); + let max_height = chain_store.head().unwrap().height; - let from_height: BlockHeight = from_height_opt.unwrap_or(min_height); - let mut to_height: BlockHeight = to_height_opt.unwrap_or(max_height); + let from_height = from_height_opt.unwrap_or(min_height); + let mut to_height = to_height_opt.unwrap_or(max_height); // There is no point in going over nonexisting blocks past the highest height if to_height > max_height { @@ -65,7 +65,7 @@ impl Iterator for BlockHeightRangeIterator { type Item = Block; fn next(&mut self) -> Option { - let current_block_hash: CryptoHash = match self.current_block_hash.take() { + let current_block_hash = match self.current_block_hash.take() { Some(hash) => hash, None => return None, }; diff --git a/tools/database/src/block_iterators/last_blocks.rs b/tools/database/src/block_iterators/last_blocks.rs index bd6240a0da0..8704391be56 100644 --- a/tools/database/src/block_iterators/last_blocks.rs +++ b/tools/database/src/block_iterators/last_blocks.rs @@ -29,7 +29,7 @@ impl Iterator for LastNBlocksIterator { }; if let Some(current_block_hash) = self.current_block_hash.take() { - let current_block: Block = self.chain_store.get_block(¤t_block_hash).unwrap(); + let current_block = self.chain_store.get_block(¤t_block_hash).unwrap(); // Set the previous block as "current" one, as long as the current one isn't the genesis block if current_block.header().height() != self.chain_store.get_genesis_height() { From 9abd60027233b1bd56143e365cd5b9d802a4c48e Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Wed, 29 Nov 2023 15:48:09 +0100 Subject: [PATCH 19/24] fix: left half of shard split should be less than split_account When splitting a shard into two halves the shard is divided into two on the split_account (boundary_account). The split should look like this: left < boundary_account right >= boundary_account Previously I misunderstood the semantics and implemented the split as if it was left <= boundary and right > boundary. Fix it. --- tools/database/src/analyse_gas_usage.rs | 28 ++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tools/database/src/analyse_gas_usage.rs b/tools/database/src/analyse_gas_usage.rs index 4822ea8b30f..715a3d599d1 100644 --- a/tools/database/src/analyse_gas_usage.rs +++ b/tools/database/src/analyse_gas_usage.rs @@ -96,9 +96,9 @@ struct GasUsageInShard { struct ShardSplit { /// Account on which the shard would be split pub split_account: AccountId, - /// Gas used by accounts <= split_account + /// Gas used by accounts < split_account pub gas_left: BigGas, - /// Gas used by accounts > split_account + /// Gas used by accounts => split_account pub gas_right: BigGas, } @@ -141,10 +141,7 @@ impl GasUsageInShard { let mut gas_right: BigGas = total_gas; for (account, used_gas) in &self.used_gas_per_account { - // We are now considering a split of (left <= account) and (right > account), - // so the current gas should be included in the left part of the split - gas_left = gas_left.checked_add(*used_gas).unwrap(); - gas_right = gas_right.checked_sub(*used_gas).unwrap(); + // We are now considering a split of (left < account) and (right >= account) let difference: BigGas = gas_left.abs_diff(gas_right); if difference < best_difference { @@ -152,6 +149,9 @@ impl GasUsageInShard { best_split = Some(ShardSplit { split_account: account.clone(), gas_left, gas_right }); } + + gas_left = gas_left.checked_add(*used_gas).unwrap(); + gas_right = gas_right.checked_sub(*used_gas).unwrap(); } best_split } @@ -387,11 +387,11 @@ fn analyse_gas_usage( *shard_usage.used_gas_per_account.get(&shard_split.split_account).unwrap() ) ); - println!(" Left (account <= split_account):"); + println!(" Left (account < split_account):"); let left_accounts = shard_usage.used_gas_per_account.range(..=shard_split.split_account.clone()); display_shard_split_stats(left_accounts, shard_total_gas); - println!(" Right (account > split_account):"); + println!(" Right (account >= split_account):"); let right_accounts = shard_usage.used_gas_per_account.range(shard_split.split_account..).skip(1); display_shard_split_stats(right_accounts, shard_total_gas); @@ -457,7 +457,7 @@ mod tests { shard_usage.add_used_gas(account("b"), 12345); let optimal_split = - ShardSplit { split_account: account("a"), gas_left: 12345, gas_right: 12345 }; + ShardSplit { split_account: account("b"), gas_left: 12345, gas_right: 12345 }; assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); } @@ -470,7 +470,7 @@ mod tests { shard_usage.add_used_gas(account("b"), 12345); let optimal_split = - ShardSplit { split_account: account("a"), gas_left: 123, gas_right: 12345 }; + ShardSplit { split_account: account("b"), gas_left: 123, gas_right: 12345 }; assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); } @@ -483,7 +483,7 @@ mod tests { shard_usage.add_used_gas(account("b"), 123); let optimal_split = - ShardSplit { split_account: account("a"), gas_left: 12345, gas_right: 123 }; + ShardSplit { split_account: account("b"), gas_left: 12345, gas_right: 123 }; assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); } @@ -538,7 +538,7 @@ mod tests { // Optimal split: // 1 + 3 + 5 + 2 = 11 // 8 + 1 + 2 + 8 = 19 - let optimal_split = ShardSplit { split_account: account("d"), gas_left: 11, gas_right: 19 }; + let optimal_split = ShardSplit { split_account: account("e"), gas_left: 11, gas_right: 19 }; assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); } @@ -557,7 +557,7 @@ mod tests { shard_usage.add_used_gas(account("h"), 1); let optimal_split = - ShardSplit { split_account: account("a"), gas_left: 10000, gas_right: 7 }; + ShardSplit { split_account: account("b"), gas_left: 10000, gas_right: 7 }; assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); } @@ -576,7 +576,7 @@ mod tests { shard_usage.add_used_gas(account("h"), 10000); let optimal_split = - ShardSplit { split_account: account("g"), gas_left: 7, gas_right: 10000 }; + ShardSplit { split_account: account("h"), gas_left: 7, gas_right: 10000 }; assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); } } From a36ea54e5e551c395fe946cb235d459a597b2f9d Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Wed, 29 Nov 2023 15:58:42 +0100 Subject: [PATCH 20/24] feat: display gas usage of all accounts as the percentage of shard total Previously gas usage of some accounts was displayed as percentage of shard split half, but this was hard to reason about. Let's change it to a percentage of total gas usage on a shard. --- tools/database/src/analyse_gas_usage.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/database/src/analyse_gas_usage.rs b/tools/database/src/analyse_gas_usage.rs index 715a3d599d1..750a9cbac8d 100644 --- a/tools/database/src/analyse_gas_usage.rs +++ b/tools/database/src/analyse_gas_usage.rs @@ -291,31 +291,31 @@ fn display_shard_split_stats<'a>( total_shard_gas: BigGas, ) { let mut accounts_num: u64 = 0; - let mut total_gas: BigGas = 0; + let mut total_split_half_gas: BigGas = 0; let mut top_3_finder = BiggestAccountsFinder::new(3); for (account, used_gas) in accounts { accounts_num += 1; - total_gas = total_gas.checked_add(*used_gas).unwrap(); + total_split_half_gas = total_split_half_gas.checked_add(*used_gas).unwrap(); top_3_finder.add_account_stats(account.clone(), *used_gas); } let indent = " "; println!( - "{}Gas: {} ({} of shard total)", + "{}Gas: {} ({} of shard)", indent, - display_gas(total_gas), - as_percentage_of(total_gas, total_shard_gas) + display_gas(total_split_half_gas), + as_percentage_of(total_split_half_gas, total_shard_gas) ); println!("{}Accounts: {}", indent, accounts_num); println!("{}Top 3 accounts:", indent); for (i, (account, used_gas)) in top_3_finder.get_biggest_accounts().enumerate() { println!("{} #{}: {}", indent, i + 1, account); println!( - "{} Used gas: {} ({} of shard split half)", + "{} Used gas: {} ({} of shard)", indent, display_gas(used_gas), - as_percentage_of(used_gas, total_gas) + as_percentage_of(used_gas, total_shard_gas) ) } } @@ -372,7 +372,7 @@ fn analyse_gas_usage( if let Some((biggest_account, biggest_account_gas)) = shard_usage.biggest_account() { println!(" Biggest account: {}", biggest_account); println!( - " Biggest account gas: {} ({} of shard total)", + " Biggest account gas: {} ({} of shard)", display_gas(biggest_account_gas), as_percentage_of(biggest_account_gas, shard_total_gas) ); From 8d1326489a4d1615ee485261c3b8dbaf46635962 Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Wed, 29 Nov 2023 18:48:05 +0100 Subject: [PATCH 21/24] feat: show gas distribution in a shard split as (left%, split_acc%, right%) --- tools/database/src/analyse_gas_usage.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/database/src/analyse_gas_usage.rs b/tools/database/src/analyse_gas_usage.rs index 750a9cbac8d..10f894864f0 100644 --- a/tools/database/src/analyse_gas_usage.rs +++ b/tools/database/src/analyse_gas_usage.rs @@ -381,11 +381,14 @@ fn analyse_gas_usage( Some(shard_split) => { println!(" Optimal split:"); println!(" split_account: {}", shard_split.split_account); + let split_account_gas = + *shard_usage.used_gas_per_account.get(&shard_split.split_account).unwrap(); + println!(" gas(split_account): {}", display_gas(split_account_gas)); println!( - " gas(split_account): {}", - display_gas( - *shard_usage.used_gas_per_account.get(&shard_split.split_account).unwrap() - ) + " Gas distribution (left, split_acc, right): ({}, {}, {})", + as_percentage_of(shard_split.gas_left, shard_total_gas), + as_percentage_of(split_account_gas, shard_total_gas), + as_percentage_of(shard_split.gas_right.saturating_sub(split_account_gas), shard_total_gas) ); println!(" Left (account < split_account):"); let left_accounts = From 913f1f849ab52bd9f86dfb200697b691ce46ce17 Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Wed, 29 Nov 2023 19:29:22 +0100 Subject: [PATCH 22/24] fix: adjust left_accounts and right_accounts to reflect the new definition of split --- tools/database/src/analyse_gas_usage.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/database/src/analyse_gas_usage.rs b/tools/database/src/analyse_gas_usage.rs index 10f894864f0..49d8704213f 100644 --- a/tools/database/src/analyse_gas_usage.rs +++ b/tools/database/src/analyse_gas_usage.rs @@ -392,11 +392,11 @@ fn analyse_gas_usage( ); println!(" Left (account < split_account):"); let left_accounts = - shard_usage.used_gas_per_account.range(..=shard_split.split_account.clone()); + shard_usage.used_gas_per_account.range(..shard_split.split_account.clone()); display_shard_split_stats(left_accounts, shard_total_gas); println!(" Right (account >= split_account):"); let right_accounts = - shard_usage.used_gas_per_account.range(shard_split.split_account..).skip(1); + shard_usage.used_gas_per_account.range(shard_split.split_account..); display_shard_split_stats(right_accounts, shard_total_gas); } None => println!(" No optimal split for this shard"), From 645a5a8eaffb53a25a71c2c34ccc72ca60209c54 Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Wed, 29 Nov 2023 19:30:33 +0100 Subject: [PATCH 23/24] feat: rename split_account to boundary_account --- tools/database/src/analyse_gas_usage.rs | 44 ++++++++++++------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tools/database/src/analyse_gas_usage.rs b/tools/database/src/analyse_gas_usage.rs index 49d8704213f..b07d4dc33d2 100644 --- a/tools/database/src/analyse_gas_usage.rs +++ b/tools/database/src/analyse_gas_usage.rs @@ -91,14 +91,14 @@ struct GasUsageInShard { } /// A shard can be split into two halves. -/// This struct represents the result of splitting a shard at `split_account`. +/// This struct represents the result of splitting a shard at `boundary_account`. #[derive(Debug, Clone, PartialEq, Eq)] struct ShardSplit { /// Account on which the shard would be split - pub split_account: AccountId, - /// Gas used by accounts < split_account + pub boundary_account: AccountId, + /// Gas used by accounts < boundary_account pub gas_left: BigGas, - /// Gas used by accounts => split_account + /// Gas used by accounts => boundary_account pub gas_right: BigGas, } @@ -147,7 +147,7 @@ impl GasUsageInShard { if difference < best_difference { best_difference = difference; best_split = - Some(ShardSplit { split_account: account.clone(), gas_left, gas_right }); + Some(ShardSplit { boundary_account: account.clone(), gas_left, gas_right }); } gas_left = gas_left.checked_add(*used_gas).unwrap(); @@ -380,23 +380,23 @@ fn analyse_gas_usage( match shard_usage.calculate_split() { Some(shard_split) => { println!(" Optimal split:"); - println!(" split_account: {}", shard_split.split_account); - let split_account_gas = - *shard_usage.used_gas_per_account.get(&shard_split.split_account).unwrap(); - println!(" gas(split_account): {}", display_gas(split_account_gas)); + println!(" boundary_account: {}", shard_split.boundary_account); + let boundary_account_gas = + *shard_usage.used_gas_per_account.get(&shard_split.boundary_account).unwrap(); + println!(" gas(boundary_account): {}", display_gas(boundary_account_gas)); println!( - " Gas distribution (left, split_acc, right): ({}, {}, {})", + " Gas distribution (left, boundary_acc, right): ({}, {}, {})", as_percentage_of(shard_split.gas_left, shard_total_gas), - as_percentage_of(split_account_gas, shard_total_gas), - as_percentage_of(shard_split.gas_right.saturating_sub(split_account_gas), shard_total_gas) + as_percentage_of(boundary_account_gas, shard_total_gas), + as_percentage_of(shard_split.gas_right.saturating_sub(boundary_account_gas), shard_total_gas) ); - println!(" Left (account < split_account):"); + println!(" Left (account < boundary_account):"); let left_accounts = - shard_usage.used_gas_per_account.range(..shard_split.split_account.clone()); + shard_usage.used_gas_per_account.range(..shard_split.boundary_account.clone()); display_shard_split_stats(left_accounts, shard_total_gas); - println!(" Right (account >= split_account):"); + println!(" Right (account >= boundary_account):"); let right_accounts = - shard_usage.used_gas_per_account.range(shard_split.split_account..); + shard_usage.used_gas_per_account.range(shard_split.boundary_account..); display_shard_split_stats(right_accounts, shard_total_gas); } None => println!(" No optimal split for this shard"), @@ -460,7 +460,7 @@ mod tests { shard_usage.add_used_gas(account("b"), 12345); let optimal_split = - ShardSplit { split_account: account("b"), gas_left: 12345, gas_right: 12345 }; + ShardSplit { boundary_account: account("b"), gas_left: 12345, gas_right: 12345 }; assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); } @@ -473,7 +473,7 @@ mod tests { shard_usage.add_used_gas(account("b"), 12345); let optimal_split = - ShardSplit { split_account: account("b"), gas_left: 123, gas_right: 12345 }; + ShardSplit { boundary_account: account("b"), gas_left: 123, gas_right: 12345 }; assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); } @@ -486,7 +486,7 @@ mod tests { shard_usage.add_used_gas(account("b"), 123); let optimal_split = - ShardSplit { split_account: account("b"), gas_left: 12345, gas_right: 123 }; + ShardSplit { boundary_account: account("b"), gas_left: 12345, gas_right: 123 }; assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); } @@ -541,7 +541,7 @@ mod tests { // Optimal split: // 1 + 3 + 5 + 2 = 11 // 8 + 1 + 2 + 8 = 19 - let optimal_split = ShardSplit { split_account: account("e"), gas_left: 11, gas_right: 19 }; + let optimal_split = ShardSplit { boundary_account: account("e"), gas_left: 11, gas_right: 19 }; assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); } @@ -560,7 +560,7 @@ mod tests { shard_usage.add_used_gas(account("h"), 1); let optimal_split = - ShardSplit { split_account: account("b"), gas_left: 10000, gas_right: 7 }; + ShardSplit { boundary_account: account("b"), gas_left: 10000, gas_right: 7 }; assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); } @@ -579,7 +579,7 @@ mod tests { shard_usage.add_used_gas(account("h"), 10000); let optimal_split = - ShardSplit { split_account: account("h"), gas_left: 7, gas_right: 10000 }; + ShardSplit { boundary_account: account("h"), gas_left: 7, gas_right: 10000 }; assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); } } From 95e43b9ee141cdf8c0a47a40b99cdb7e819774d4 Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Wed, 29 Nov 2023 23:44:55 +0100 Subject: [PATCH 24/24] fix: fix formatiing I guess VSCode froze and didn't run format on save :C --- tools/database/src/analyse_gas_usage.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/database/src/analyse_gas_usage.rs b/tools/database/src/analyse_gas_usage.rs index b07d4dc33d2..34c0a6b87c2 100644 --- a/tools/database/src/analyse_gas_usage.rs +++ b/tools/database/src/analyse_gas_usage.rs @@ -388,7 +388,10 @@ fn analyse_gas_usage( " Gas distribution (left, boundary_acc, right): ({}, {}, {})", as_percentage_of(shard_split.gas_left, shard_total_gas), as_percentage_of(boundary_account_gas, shard_total_gas), - as_percentage_of(shard_split.gas_right.saturating_sub(boundary_account_gas), shard_total_gas) + as_percentage_of( + shard_split.gas_right.saturating_sub(boundary_account_gas), + shard_total_gas + ) ); println!(" Left (account < boundary_account):"); let left_accounts = @@ -541,7 +544,8 @@ mod tests { // Optimal split: // 1 + 3 + 5 + 2 = 11 // 8 + 1 + 2 + 8 = 19 - let optimal_split = ShardSplit { boundary_account: account("e"), gas_left: 11, gas_right: 19 }; + let optimal_split = + ShardSplit { boundary_account: account("e"), gas_left: 11, gas_right: 19 }; assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); }