Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add --halt-after-import flag to CLI #1835

Merged
merged 12 commits into from
Aug 23, 2022
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions forest/src/cli/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ pub struct Client {
pub snapshot: bool,
pub snapshot_height: Option<i64>,
pub snapshot_path: Option<String>,
pub halt_after_import: bool,
/// Skips loading import CAR file and assumes it's already been loaded.
/// Will use the CIDs in the header of the file to index the chain.
pub skip_load: bool,
Expand All @@ -44,6 +45,7 @@ impl Default for Client {
rpc_token: None,
snapshot_path: None,
snapshot: false,
halt_after_import: false,
snapshot_height: None,
skip_load: false,
encrypt_keystore: true,
Expand Down
2 changes: 2 additions & 0 deletions forest/src/cli/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ mod test {
rpc_port: u16::arbitrary(g),
rpc_token: Option::arbitrary(g),
snapshot: bool::arbitrary(g),
halt_after_import: bool::arbitrary(g),
snapshot_height: Option::arbitrary(g),
snapshot_path: Option::arbitrary(g),
skip_load: bool::arbitrary(g),
Expand All @@ -78,6 +79,7 @@ mod test {
compression_type: Option::arbitrary(g),
compaction_style: Option::arbitrary(g),
enable_statistics: bool::arbitrary(g),
log_level: String::arbitrary(g),
},
network: Libp2pConfig {
listening_multiaddr: Ipv4Addr::arbitrary(g).into(),
Expand Down
5 changes: 5 additions & 0 deletions forest/src/cli/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,9 @@ pub struct CliOpts {
pub height: Option<i64>,
#[structopt(long, help = "Import a snapshot from a local CAR file or url")]
pub import_snapshot: Option<String>,
/// Halt with exit code 0 after successfully importing a snapshot
#[structopt(long)]
pub halt_after_import: bool,
Copy link
Contributor

@elmattic elmattic Aug 19, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please add some doc comment help for this new flag. Consider refactoring to use doc comments for the other flags too (this will gives us documentation like IDE tooltip and spell checking for free).

Copy link
Contributor

@elmattic elmattic Aug 19, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@lemmih do you think such self-describing flags need help comments?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. The help message could even add more details such as "Halt with exit code 0 after successfully importing a snapshot".

#[structopt(long, help = "Import a chain from a local CAR file or url")]
pub import_chain: Option<String>,
#[structopt(
Expand Down Expand Up @@ -231,6 +234,8 @@ impl CliOpts {
cfg.client.skip_load = self.skip_load;
}

cfg.client.halt_after_import = self.halt_after_import;

cfg.network.kademlia = self.kademlia.unwrap_or(cfg.network.kademlia);
cfg.network.mdns = self.mdns.unwrap_or(cfg.network.mdns);
if let Some(target_peer_count) = self.target_peer_count {
Expand Down
9 changes: 8 additions & 1 deletion forest/src/daemon.rs
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,12 @@ pub(super) async fn start(config: Config) {
}
}

// Halt
if config.client.halt_after_import {
info!("Forest finish shutdown");
return;
}

// Fetch and ensure verification keys are downloaded
if cns::FETCH_PARAMS {
use forest_paramfetch::{
Expand Down Expand Up @@ -354,6 +360,7 @@ async fn sync_from_snapshot(config: &Config, state_manager: &Arc<StateManager<Ro
} else {
Some(0)
};

import_chain::<FullVerifier, _>(
state_manager,
path,
Expand All @@ -362,7 +369,7 @@ async fn sync_from_snapshot(config: &Config, state_manager: &Arc<StateManager<Ro
)
.await
.expect("Failed miserably while importing chain from snapshot");
debug!("Imported snapshot in: {}s", stopwatch.elapsed().as_secs());
info!("Imported snapshot in: {}s", stopwatch.elapsed().as_secs());
}
}

Expand Down
1 change: 1 addition & 0 deletions node/db/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ anyhow = "1"
cid = { version = "0.8", default-features = false, features = ["std"] }
forest_encoding = "0.2"
fvm_ipld_blockstore = "0.1.1"
log = "0.4.8"
num_cpus = "1.13"
parking_lot = "0.12"
rocksdb = "0.19"
Expand Down
12 changes: 10 additions & 2 deletions node/db/src/rocks.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@

use super::errors::Error;
use super::Store;
use crate::rocks_config::{compaction_style_from_str, compression_type_from_str, RocksDbConfig};
use crate::rocks_config::{
compaction_style_from_str, compression_type_from_str, log_level_from_str, RocksDbConfig,
};
use cid::Cid;
use fvm_ipld_blockstore::Blockstore;
pub use rocksdb::{Options, WriteBatch, DB};
pub use rocksdb::{LogLevel, Options, WriteBatch, DB};
use std::{path::Path, sync::Arc};

/// `RocksDB` instance this satisfies the [Store] interface.
Expand Down Expand Up @@ -40,13 +42,19 @@ impl RocksDb {
}
if let Some(compaction_style) = &config.compaction_style {
db_opts.set_compaction_style(compaction_style_from_str(compaction_style).unwrap());
db_opts.set_disable_auto_compactions(false);
} else {
db_opts.set_disable_auto_compactions(true);
Comment on lines +45 to +47
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This appears to be unrelated to the --halt-after-import flag. Could you explain the RocksDB changes in the PR description?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're right, this was part of testing to see if RocksDb was being dropped after import. I've left some of the configuration logic in as it was part of @elmattic 's work

}
if let Some(compression_type) = &config.compression_type {
db_opts.set_compression_type(compression_type_from_str(compression_type).unwrap());
}
if config.enable_statistics {
db_opts.enable_statistics();
};

db_opts.set_log_level(log_level_from_str(&config.log_level).unwrap());

Ok(Self {
db: Arc::new(DB::open(&db_opts, path)?),
})
Expand Down
18 changes: 16 additions & 2 deletions node/db/src/rocks_config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// SPDX-License-Identifier: Apache-2.0, MIT
use anyhow::anyhow;
use num_cpus;
use rocksdb::{DBCompactionStyle, DBCompressionType};
use rocksdb::{DBCompactionStyle, DBCompressionType, LogLevel};
use serde::{Deserialize, Serialize};

/// `RocksDB` configuration exposed in Forest.
Expand All @@ -20,6 +20,7 @@ pub struct RocksDbConfig {
pub compression_type: Option<String>,
pub compaction_style: Option<String>,
pub enable_statistics: bool,
pub log_level: String,
}

impl Default for RocksDbConfig {
Expand All @@ -30,9 +31,10 @@ impl Default for RocksDbConfig {
write_buffer_size: 256 * 1024 * 1024,
max_open_files: 1024,
max_background_jobs: None,
compaction_style: None,
compaction_style: Some("level".into()),
compression_type: Some("lz4".into()),
enable_statistics: false,
log_level: "warn".into(),
}
}
}
Expand Down Expand Up @@ -61,6 +63,18 @@ pub(crate) fn compression_type_from_str(s: &str) -> anyhow::Result<DBCompression
}
}

/// Converts string to a log level `RocksDB` variant.
pub(crate) fn log_level_from_str(s: &str) -> anyhow::Result<LogLevel> {
match s.to_lowercase().as_str() {
"debug" => Ok(LogLevel::Debug),
"warn" => Ok(LogLevel::Warn),
"error" => Ok(LogLevel::Error),
"fatal" => Ok(LogLevel::Fatal),
"header" => Ok(LogLevel::Header),
_ => Err(anyhow!("invalid log level option")),
}
}

#[cfg(test)]
mod test {
use super::*;
Expand Down