Skip to content

Commit

Permalink
Add storage monitor (#1530)
Browse files Browse the repository at this point in the history
  • Loading branch information
@aurexav
aurexav authored Jul 10, 2024
1 parent 192031c commit c1cc484
Show file tree
Hide file tree
Showing 6 changed files with 54 additions and 25 deletions.
1 change: 1 addition & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ frame-system-benchmarking = { git = "https://github.com/darwini
frame-system-rpc-runtime-api = { git = "https://github.com/darwinia-network/polkadot-sdk", branch = "release-polkadot-v1.7.2", default-features = false }
frame-try-runtime = { git = "https://github.com/darwinia-network/polkadot-sdk", branch = "release-polkadot-v1.7.2", default-features = false }
pallet-assets = { git = "https://github.com/darwinia-network/polkadot-sdk", branch = "release-polkadot-v1.7.2", default-features = false }
pallet-aura = { git = "https://github.com/darwinia-network/polkadot-sdk", branch = "release-polkadot-v1.7.2", default-features = false, features = ["experimental"]}
pallet-aura = { git = "https://github.com/darwinia-network/polkadot-sdk", branch = "release-polkadot-v1.7.2", default-features = false, features = ["experimental"] }
pallet-authorship = { git = "https://github.com/darwinia-network/polkadot-sdk", branch = "release-polkadot-v1.7.2", default-features = false }
pallet-balances = { git = "https://github.com/darwinia-network/polkadot-sdk", branch = "release-polkadot-v1.7.2", default-features = false, features = ["insecure_zero_ed"] }
pallet-collective = { git = "https://github.com/darwinia-network/polkadot-sdk", branch = "release-polkadot-v1.7.2", default-features = false }
Expand Down Expand Up @@ -156,6 +156,7 @@ sc-offchain = { git = "https://github.com/darwini
sc-rpc = { git = "https://github.com/darwinia-network/polkadot-sdk", branch = "release-polkadot-v1.7.2" }
sc-rpc-api = { git = "https://github.com/darwinia-network/polkadot-sdk", branch = "release-polkadot-v1.7.2" }
sc-service = { git = "https://github.com/darwinia-network/polkadot-sdk", branch = "release-polkadot-v1.7.2" }
sc-storage-monitor = { git = "https://github.com/darwinia-network/polkadot-sdk", branch = "release-polkadot-v1.7.2" }
sc-sysinfo = { git = "https://github.com/darwinia-network/polkadot-sdk", branch = "release-polkadot-v1.7.2" }
sc-telemetry = { git = "https://github.com/darwinia-network/polkadot-sdk", branch = "release-polkadot-v1.7.2" }
sc-tracing = { git = "https://github.com/darwinia-network/polkadot-sdk", branch = "release-polkadot-v1.7.2" }
Expand Down
1 change: 1 addition & 0 deletions node/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ sc-network-sync = { workspace = true }
sc-offchain = { workspace = true }
sc-rpc = { workspace = true }
sc-service = { workspace = true }
sc-storage-monitor = { workspace = true }
sc-sysinfo = { workspace = true }
sc-telemetry = { workspace = true }
sc-tracing = { workspace = true }
Expand Down
3 changes: 3 additions & 0 deletions node/src/cli.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,9 @@ pub struct Cli {
#[arg(long)]
pub no_hardware_benchmarks: bool,

#[clap(flatten)]
pub storage_monitor: sc_storage_monitor::StorageMonitorParams,

/// Relay chain arguments
#[arg(raw = true)]
pub relay_chain_args: Vec<String>,
Expand Down
44 changes: 24 additions & 20 deletions node/src/command.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
// along with Darwinia. If not, see <https://www.gnu.org/licenses/>.

// std
use std::{env, net::SocketAddr, path::PathBuf};
use std::{env, fs, io::ErrorKind, net::SocketAddr, path::PathBuf, result::Result as StdResult};
// darwinia
use crate::{
chain_spec::*,
Expand Down Expand Up @@ -70,7 +70,7 @@ impl SubstrateCli for Cli {
2018
}

fn load_spec(&self, id: &str) -> std::result::Result<Box<dyn ChainSpecT>, String> {
fn load_spec(&self, id: &str) -> StdResult<Box<dyn ChainSpecT>, String> {
load_spec(id)
}
}
Expand Down Expand Up @@ -106,7 +106,7 @@ impl SubstrateCli for RelayChainCli {
2018
}

fn load_spec(&self, id: &str) -> std::result::Result<Box<dyn ChainSpecT>, String> {
fn load_spec(&self, id: &str) -> StdResult<Box<dyn ChainSpecT>, String> {
polkadot_cli::Cli::from_iter([RelayChainCli::executable_name()].iter()).load_spec(id)
}
}
Expand Down Expand Up @@ -364,6 +364,7 @@ pub fn run() -> Result<()> {
runner.sync_run(|config| {
// Remove Frontier off-chain db
let db_config_dir = frontier::db_config_dir(&config);

match cli.eth_args.frontier_backend_type {
FrontierBackendType::KeyValue => {
let frontier_database_config = match config.database {
Expand All @@ -382,15 +383,17 @@ pub fn run() -> Result<()> {
.into())
}
};

cmd.base.run(frontier_database_config)?;
}
FrontierBackendType::Sql => {
let db_path = db_config_dir.join("sql");
match std::fs::remove_dir_all(&db_path) {

match fs::remove_dir_all(&db_path) {
Ok(_) => {
println!("{:?} removed.", &db_path);
}
Err(ref err) if err.kind() == std::io::ErrorKind::NotFound => {
Err(ref err) if err.kind() == ErrorKind::NotFound => {
eprintln!("{:?} did not exist.", &db_path);
}
Err(err) => {
Expand Down Expand Up @@ -429,6 +432,7 @@ pub fn run() -> Result<()> {
set_default_ss58_version(chain_spec);
runner.sync_run(|_config| {
let spec = cli.load_spec(&cmd.shared_params.chain.clone().unwrap_or_default())?;

cmd.run(&*spec)
})
},
Expand Down Expand Up @@ -473,29 +477,26 @@ pub fn run() -> Result<()> {

runner.run_node_until_exit(|config| async move {
let chain_spec = &config.chain_spec;
let hwbench = (!cli.no_hardware_benchmarks).then_some(
config.database.path().map(|database_path| {
let _ = std::fs::create_dir_all(database_path);
sc_sysinfo::gather_hwbench(Some(database_path))
})).flatten();

set_default_ss58_version(chain_spec);

let para_id = Extensions::try_get(&*config.chain_spec)
.map(|e| e.para_id)
.ok_or("Could not find parachain ID in chain-spec.")?;
let polkadot_cli = RelayChainCli::new(
&config,
[RelayChainCli::executable_name()].iter().chain(cli.relay_chain_args.iter()),
);
let tokio_handle = config.tokio_handle.clone();
let para_id = Extensions::try_get(&*config.chain_spec)
.map(|e| e.para_id)
.ok_or("Could not find parachain ID in chain-spec.")?;
let id = ParaId::from(para_id);
let parachain_account =
AccountIdConversion::<polkadot_primitives::AccountId>::into_account_truncating(&id);
let tokio_handle = config.tokio_handle.clone();
let no_hardware_benchmarks = cli.no_hardware_benchmarks;
let storage_monitor = cli.storage_monitor;
let eth_rpc_config = cli.eth_args.build_eth_rpc_config();

log::info!("Parachain id: {:?}", id);
log::info!("Parachain Account: {}", parachain_account);
log::info!("Parachain id: {id:?}");
log::info!("Parachain Account: {parachain_account}");
log::info!(
"Is collating: {}",
if config.role.is_authority() { "yes" } else { "no" }
Expand Down Expand Up @@ -544,7 +545,8 @@ pub fn run() -> Result<()> {
polkadot_config,
collator_options,
id,
hwbench,
no_hardware_benchmarks,
storage_monitor,
&eth_rpc_config,
)
.await
Expand All @@ -559,7 +561,8 @@ pub fn run() -> Result<()> {
polkadot_config,
collator_options,
id,
hwbench,
no_hardware_benchmarks,
storage_monitor,
&eth_rpc_config,
)
.await
Expand All @@ -574,7 +577,8 @@ pub fn run() -> Result<()> {
polkadot_config,
collator_options,
id,
hwbench,
no_hardware_benchmarks,
storage_monitor,
&eth_rpc_config,
)
.await
Expand All @@ -588,7 +592,7 @@ pub fn run() -> Result<()> {
}
}

fn load_spec(id: &str) -> std::result::Result<Box<dyn ChainSpecT>, String> {
fn load_spec(id: &str) -> StdResult<Box<dyn ChainSpecT>, String> {
let id = if id.is_empty() {
let n = get_exec_name().unwrap_or_default();
["darwinia", "crab", "koi"]
Expand Down
27 changes: 23 additions & 4 deletions node/src/service/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,6 @@ impl<Api> RuntimeApiCollection for Api where
///
/// Use this macro if you don't actually need the full service, but just the builder in order to
/// be able to perform chain operations.
#[allow(clippy::type_complexity)]
pub fn new_partial<RuntimeApi>(
config: &sc_service::Configuration,
eth_rpc_config: &crate::cli::EthRpcConfig,
Expand Down Expand Up @@ -245,7 +244,8 @@ async fn start_node_impl<RuntimeApi, SC>(
sybil_resistance_level: cumulus_client_service::CollatorSybilResistance,
para_id: cumulus_primitives_core::ParaId,
start_consensus: SC,
hwbench: Option<sc_sysinfo::HwBench>,
no_hardware_benchmarks: bool,
storage_monitor: sc_storage_monitor::StorageMonitorParams,
eth_rpc_config: &crate::cli::EthRpcConfig,
) -> sc_service::error::Result<(sc_service::TaskManager, Arc<FullClient<RuntimeApi>>)>
where
Expand Down Expand Up @@ -289,6 +289,14 @@ where
telemetry_worker_handle,
),
} = new_partial::<RuntimeApi>(&parachain_config, eth_rpc_config)?;
let database_path = parachain_config.database.path().map(|p| p.to_path_buf());
let hwbench = (!no_hardware_benchmarks)
.then_some(database_path.as_ref().map(|p| {
let _ = std::fs::create_dir_all(p);

sc_sysinfo::gather_hwbench(Some(p))
}))
.flatten();
let (relay_chain_interface, collator_key) =
cumulus_client_service::build_relay_chain_interface(
polkadot_config,
Expand Down Expand Up @@ -458,6 +466,7 @@ where

if let Some(hwbench) = hwbench {
sc_sysinfo::print_hwbench(&hwbench);

// Here you can check whether the hardware meets your chains' requirements. Putting a link
// in there and swapping out the requirements for your own are probably a good idea. The
// requirements for a para-chain are dictated by its relay-chain.
Expand All @@ -478,6 +487,14 @@ where
);
}
}
if let Some(database_path) = database_path {
sc_storage_monitor::StorageMonitorService::try_spawn(
storage_monitor,
database_path,
&task_manager.spawn_essential_handle(),
)
.map_err(|e| sc_service::Error::Application(e.into()))?;
}

let announce_block = {
let sync_service = sync_service.clone();
Expand Down Expand Up @@ -573,7 +590,8 @@ pub async fn start_parachain_node<RuntimeApi>(
polkadot_config: sc_service::Configuration,
collator_options: cumulus_client_cli::CollatorOptions,
para_id: cumulus_primitives_core::ParaId,
hwbench: Option<sc_sysinfo::HwBench>,
no_hardware_benchmarks: bool,
storage_monitor: sc_storage_monitor::StorageMonitorParams,
eth_rpc_config: &crate::cli::EthRpcConfig,
) -> sc_service::error::Result<(sc_service::TaskManager, Arc<FullClient<RuntimeApi>>)>
where
Expand Down Expand Up @@ -660,7 +678,8 @@ where

Ok(())
},
hwbench,
no_hardware_benchmarks,
storage_monitor,
eth_rpc_config,
)
.await
Expand Down

0 comments on commit c1cc484

Please sign in to comment.