Skip to content

Commit

Permalink
resolve comments
Browse files Browse the repository at this point in the history
  • Loading branch information
@hanabi1224
hanabi1224 committed Aug 30, 2023
1 parent d4a1647 commit 8e5b137
Show file tree
Hide file tree
Showing 3 changed files with 54 additions and 56 deletions.
24 changes: 22 additions & 2 deletions src/cli_shared/snapshot.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,13 @@ use std::{
io,
path::{Path, PathBuf},
str::FromStr,
time::Duration,
};

use crate::networks::NetworkChain;
use crate::{
networks::NetworkChain,
utils::{retry, RetryArgs},
};
use anyhow::{anyhow, bail, Context as _};
use chrono::NaiveDate;
use tracing::{info, warn};
Expand Down Expand Up @@ -72,7 +76,23 @@ pub async fn fetch(
.date_and_height_and_forest();
let filename = filename(vendor, chain, date, height, forest_format);

download_file(url, directory, &filename).await
download_file_with_retry(url, directory, &filename).await
}

pub async fn download_file_with_retry(
url: Url,
directory: &Path,
filename: &str,
) -> anyhow::Result<PathBuf> {
Ok(retry(
RetryArgs {
timeout: None,
max_retries: Some(3),
delay: Some(Duration::from_secs(60)),
},
|| download_file(url.clone(), directory, filename),
)
.await?)
}

pub async fn download_file(url: Url, directory: &Path, filename: &str) -> anyhow::Result<PathBuf> {
Expand Down
24 changes: 12 additions & 12 deletions src/daemon/db_util.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,16 @@ use std::{
time,
};
use tokio::io::AsyncWriteExt;
use tracing::info;
use tracing::{debug, info};
use url::Url;
use walkdir::WalkDir;

pub fn load_all_forest_cars<T>(
store: &ManyCar<T>,
forest_car_db_dir: impl AsRef<Path>,
) -> anyhow::Result<()> {
let forest_car_db_dir = forest_car_db_dir.as_ref();
pub fn load_all_forest_cars<T>(store: &ManyCar<T>, forest_car_db_dir: &Path) -> anyhow::Result<()> {
if !forest_car_db_dir.is_dir() {
fs::create_dir_all(forest_car_db_dir)?;
}
for file in WalkDir::new(forest_car_db_dir)
.max_depth(1)
.into_iter()
.filter_map(|entry| {
if let Ok(entry) = entry {
Expand All @@ -45,7 +42,7 @@ pub fn load_all_forest_cars<T>(
let car = ForestCar::try_from(file.as_path())
.with_context(|| format!("Error loading car DB at {}", file.display()))?;
store.read_only(car.into());
info!("Loaded car DB at {}", file.display());
debug!("Loaded car DB at {}", file.display());
}

Ok(())
Expand Down Expand Up @@ -98,15 +95,18 @@ pub async fn import_chain_as_forest_car(
}

async fn download_to(url: Url, destination: &Path) -> anyhow::Result<()> {
snapshot::download_file(
snapshot::download_file_with_retry(
url,
destination
.parent()
.context("Infallible getting the directory")?,
destination.parent().with_context(|| {
format!(
"Error getting the parent directory of {}",
destination.display()
)
})?,
destination
.file_name()
.and_then(OsStr::to_str)
.context("Infallible getting the file name")?,
.with_context(|| format!("Error getting the file name of {}", destination.display()))?,
)
.await?;

Expand Down
62 changes: 20 additions & 42 deletions src/daemon/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ use crate::shim::clock::ChainEpoch;
use crate::shim::version::NetworkVersion;
use crate::state_manager::StateManager;
use crate::utils::{
monitoring::MemStatsTracker, proofs_api::paramfetch::ensure_params_downloaded, retry,
version::FOREST_VERSION_STRING, RetryArgs,
monitoring::MemStatsTracker, proofs_api::paramfetch::ensure_params_downloaded,
version::FOREST_VERSION_STRING,
};
use anyhow::{bail, Context};
use bundle::load_actor_bundles;
Expand All @@ -44,7 +44,6 @@ use lazy_static::lazy_static;
use raw_sync::events::{Event, EventInit as _, EventState};
use shared_memory::ShmemConf;
use std::path::Path;
use std::time::Duration;
use std::{cell::RefCell, net::TcpListener, path::PathBuf, sync::Arc};
use tempfile::{Builder, TempPath};
use tokio::{
Expand Down Expand Up @@ -394,10 +393,10 @@ pub(super) async fn start(

// TODO: respect `--consume-snapshot` CLI option once it's implemented
let mut consume_snapshot_file = false;
// Fetch the latest snapshot if needed
// Sets the latest snapshot if needed for downloading later
let mut config = config;
if config.client.snapshot_path.is_none() {
fetch_snapshot_if_required(
set_snapshot_path_if_needed(
&mut config,
epoch,
opts.auto_download_snapshot,
Expand All @@ -413,7 +412,7 @@ pub(super) async fn start(
let (car_db_path, ts) =
import_chain_as_forest_car(path, &forest_car_db_dir, consume_snapshot_file).await?;
db.read_only_files(std::iter::once(car_db_path.clone()))?;
info!("Loaded car DB at {}", car_db_path.display());
debug!("Loaded car DB at {}", car_db_path.display());
state_manager
.chain_store()
.set_heaviest_tipset(Arc::new(ts))?;
Expand Down Expand Up @@ -459,7 +458,7 @@ pub(super) async fn start(
/// to a supported height. If we've not been given a snapshot by the user, get one.
///
/// An [`Err`] should be considered fatal.
async fn fetch_snapshot_if_required(
async fn set_snapshot_path_if_needed(
config: &mut Config,
epoch: ChainEpoch,
auto_download_snapshot: bool,
Expand All @@ -485,35 +484,18 @@ async fn fetch_snapshot_if_required(
let have_a_snapshot = config.client.snapshot_path.is_some();

match (require_a_snapshot, have_a_snapshot, auto_download_snapshot) {
(false, _, _) => Ok(()), // noop - don't need a snapshot
(true, true, _) => Ok(()), // noop - we need a snapshot, and we have one
(false, _, _) => {} // noop - don't need a snapshot
(true, true, _) => {} // noop - we need a snapshot, and we have one
(true, false, true) => {
// we need a snapshot, don't have one, and have permission to download one, so do that
let max_retries = 3;
match retry(
RetryArgs {
timeout: None,
max_retries: Some(max_retries),
delay: Some(Duration::from_secs(60)),
},
|| crate::cli_shared::snapshot::fetch(download_directory, chain, vendor),
)
.await
{
Ok(path) => {
config.client.snapshot_path = Some(path);
config.client.snapshot = true;
Ok(())
}
Err(_) => bail!("failed to fetch snapshot after {max_retries} attempts"),
}
let (_len, url) = crate::cli_shared::snapshot::peek(vendor, chain).await?;
config.client.snapshot_path = Some(url.to_string().into());
config.client.snapshot = true;
}
(true, false, false) => {
// we need a snapshot, don't have one, and don't have permission to download one, so ask the user
let (num_bytes, _url) =
crate::cli_shared::snapshot::peek(vendor, &config.chain.network)
.await
.context("couldn't get snapshot size")?;
let (num_bytes, url) = crate::cli_shared::snapshot::peek(vendor, &config.chain.network)
.await
.context("couldn't get snapshot size")?;
// dialoguer will double-print long lines, so manually print the first clause ourselves,
// then let `Confirm` handle the second.
println!("Forest requires a snapshot to sync with the network, but automatic fetching is disabled.");
Expand All @@ -531,18 +513,14 @@ async fn fetch_snapshot_if_required(
})
.await;
if !have_permission {
bail!("Forest requires a snapshot to sync with the network, but automatic fetching is disabled.");
}
match crate::cli_shared::snapshot::fetch(download_directory, chain, vendor).await {
Ok(path) => {
config.client.snapshot_path = Some(path);
config.client.snapshot = true;
Ok(())
}
Err(e) => Err(e).context("downloading required snapshot failed"),
bail!("Forest requires a snapshot to sync with the network, but automatic fetching is disabled.")
}
config.client.snapshot_path = Some(url.to_string().into());
config.client.snapshot = true;
}
}
};

Ok(())
}

/// Generates, prints and optionally writes to a file the administrator JWT
Expand Down

0 comments on commit 8e5b137

Please sign in to comment.