feat: serve forest CAR file as Blockstore (#3365)

Co-authored-by: David Himmelstrup <david.himmelstrup@chainsafe.io>

Cargo.toml

@@ -183,7 +183,6 @@ url = { version = "2.3", features = ["serde"] }
 uuid = { version = "1.3", features = ['v4'] }
 walkdir = "2"
 zstd = "0.12.3"
-zstd-safe = "6.0.5"
 
 [target.'cfg(unix)'.dependencies]
 termios = "0.3"

scripts/tests/calibnet_other_check.sh

@@ -7,8 +7,12 @@ set -e
 
 source "$(dirname "$0")/harness.sh"
 
+forest_import_non_calibnet_snapshot
 forest_init
 
+echo "Verifying the non calibnet snapshot (./test-snapshots/chain4.car) is being served properly."
+$FOREST_CLI_PATH chain read-obj -c bafy2bzacedjrqan2fwfvhfopi64yickki7miiksecglpeiavf7xueytnzevlu
+
 echo "Running database garbage collection"
 forest_check_db_stats
 $FOREST_CLI_PATH db gc

scripts/tests/harness.sh

@@ -15,6 +15,11 @@ export TMP_DIR
 export LOG_DIRECTORY
 export FOREST_WALLET_PATH
 
+function forest_import_non_calibnet_snapshot {
+  echo "Importing a non calibnet snapshot"
+  $FOREST_PATH --chain calibnet --encrypt-keystore false --halt-after-import --import-snapshot ./test-snapshots/chain4.car
+}
+
 function forest_download_and_import_snapshot {
   echo "Downloading and importing snapshot"
   $FOREST_PATH --chain calibnet --encrypt-keystore false --halt-after-import --height=-200 --auto-download-snapshot

src/blocks/tipset.rs

@@ -3,6 +3,7 @@
 
 use std::{fmt, sync::OnceLock};
 
+use crate::db::{SettingsStore, SettingsStoreExt};
 use crate::ipld::FrozenCids;
 use crate::networks::{calibnet, mainnet};
 use crate::shim::{address::Address, clock::ChainEpoch};
@@ -151,6 +152,25 @@ impl Tipset {
             .transpose()?)
     }
 
+    /// Load the heaviest tipset from the blockstore
+    pub fn load_heaviest(
+        store: &impl Blockstore,
+        settings: &impl SettingsStore,
+    ) -> anyhow::Result<Option<Tipset>> {
+        Ok(
+            match settings.read_obj::<TipsetKeys>(crate::db::setting_keys::HEAD_KEY)? {
+                Some(tsk) => tsk
+                    .cids
+                    .into_iter()
+                    .map(|key| BlockHeader::load(store, key))
+                    .collect::<anyhow::Result<Option<_>>>()?
+                    .map(Tipset::new)
+                    .transpose()?,
+                None => None,
+            },
+        )
+    }
+
     /// Fetch a tipset from the blockstore. This calls fails if the tipset is
     /// missing or invalid.
     pub fn load_required(store: impl Blockstore, tsk: &TipsetKeys) -> anyhow::Result<Tipset> {

src/chain/store/chain_store.rs

@@ -32,7 +32,7 @@ use super::{
     tipset_tracker::TipsetTracker,
     Error,
 };
-use crate::db::setting_keys::{ESTIMATED_RECORDS_KEY, HEAD_KEY};
+use crate::db::setting_keys::HEAD_KEY;
 use crate::db::{SettingsStore, SettingsStoreExt};
 
 // A cap on the size of the future_sink
@@ -150,11 +150,6 @@ where
         self.tipset_tracker.add(header);
     }
 
-    pub fn set_estimated_records(&self, records: u64) -> anyhow::Result<()> {
-        self.settings.write_obj(ESTIMATED_RECORDS_KEY, &records)?;
-        Ok(())
-    }
-
     /// Writes tipset block headers to data store and updates heaviest tipset
     /// with other compatible tracked headers.
     pub fn put_tipset(&self, ts: &Tipset) -> Result<(), Error> {

src/cli_shared/snapshot.rs

@@ -8,7 +8,10 @@ use std::{
     str::FromStr,
 };
 
-use crate::networks::NetworkChain;
+use crate::{
+    networks::NetworkChain,
+    utils::{retry, RetryArgs},
+};
 use anyhow::{anyhow, bail, Context as _};
 use chrono::NaiveDate;
 use tracing::{info, warn};
@@ -72,11 +75,30 @@ pub async fn fetch(
         .date_and_height_and_forest();
     let filename = filename(vendor, chain, date, height, forest_format);
 
-    match download_aria2c(&url, directory, &filename).await {
+    download_file_with_retry(&url, directory, &filename).await
+}
+
+pub async fn download_file_with_retry(
+    url: &Url,
+    directory: &Path,
+    filename: &str,
+) -> anyhow::Result<PathBuf> {
+    Ok(retry(
+        RetryArgs {
+            timeout: None,
+            ..Default::default()
+        },
+        || download_file(url.clone(), directory, filename),
+    )
+    .await?)
+}
+
+pub async fn download_file(url: Url, directory: &Path, filename: &str) -> anyhow::Result<PathBuf> {
+    match download_aria2c(&url, directory, filename).await {
         Ok(path) => Ok(path),
         Err(AriaErr::CouldNotExec(reason)) => {
             warn!(%reason, "couldn't run aria2c. Falling back to conventional download, which will be much slower - consider installing aria2c.");
-            download_http(url, directory, &filename).await
+            download_http(url, directory, filename).await
         }
         Err(AriaErr::Other(o)) => Err(o),
     }