feat: symlink snapshots (#4620)

CHANGELOG.md

@@ -27,6 +27,10 @@
 
 ### Breaking
 
+- [#4620](https://github.com/ChainSafe/forest/pull/4620) Removed the
+  `--consume-snapshot` parameter from the `forest` binary. To consume a
+  snapshot, use `--import-snapshot <path> --import-mode=move`.
+
 ### Added
 
 - [#3959](https://github.com/ChainSafe/forest/issues/3959) Added support for the
@@ -39,6 +43,10 @@
   propagation delays are now configurable via
   [environment variables](https://github.com/ChainSafe/forest/blob/main/documentation/src/environment_variables.md).
 
+- [#4620](https://github.com/ChainSafe/forest/pull/4620) Added an option to link
+  snapshots instead of moving or copying them. This can be invoked with
+  `--import-snapshot <path> --import-mode=symlink`.
+
 ### Changed
 
 - [#4583](https://github.com/ChainSafe/forest/pull/4583) Removed the expiration

documentation/src/configuration.md

@@ -21,7 +21,7 @@ use of the following flags:
 | --kademlia           | Boolean      | Determines whether Kademilia is allowed                                                             |
 | --mdns               | Boolean      | Determines whether MDNS is allowed                                                                  |
 | --import-snapshot    | OS File Path | Path to snapshot CAR file                                                                           |
-| --consume-snapshot   | OS File Path | Path to snapshot CAR file (delete after importing)                                                  |
+| --import-mode        | String       | Snapshot import mode: Copy, Move, Symlink                                                           |
 | --skip-load          | Boolean      | Skips loading CAR File and uses header to index chain                                               |
 | --req-window         | Integer      | Sets the number of tipsets requested over chain exchange                                            |
 | --tipset-sample-size | Integer      | Number of tipsets to include in the sample which determines the network head during synchronization |

scripts/tests/api_compare/docker-compose.yml

@@ -54,7 +54,8 @@ services:
           --rpc-address 0.0.0.0:${FOREST_RPC_PORT} \
           --healthcheck-address 0.0.0.0:${FOREST_HEALTHZ_RPC_PORT} \
           --height=-50 \
-          --import-snapshot $(ls /data/*.car.zst | tail -n 1)
+          --import-snapshot $(ls /data/*.car.zst | tail -n 1) \
+          --import-mode=symlink
     healthcheck:
       test: [ "CMD", "forest-cli", "sync", "wait" ]
       interval: 15s

scripts/tests/bootstrapper/docker-compose-forest.yml

@@ -83,7 +83,8 @@ services:
         forest --chain ${CHAIN} --encrypt-keystore false --no-gc \
           --config config.toml \
           --rpc-address 0.0.0.0:${FOREST_RPC_PORT} \
-          --consume-snapshot $(ls /data/*.car.zst | tail -n 1)
+          --import-snapshot $(ls /data/*.car.zst | tail -n 1) \
+          --import-mode=move
     healthcheck:
       test: [ "CMD", "forest-cli", "sync", "wait" ]
       interval: 15s

scripts/tests/snapshot_parity/docker-compose.yml

@@ -47,7 +47,8 @@ services:
         set -euxo pipefail
         forest --chain ${CHAIN} --encrypt-keystore false --no-gc \
           --rpc-address 0.0.0.0:${FOREST_RPC_PORT} \
-          --import-snapshot $(ls /data/*.car.zst | tail -n 1) &
+          --import-snapshot $(ls /data/*.car.zst | tail -n 1) \
+          --import-mode=symlink &
         sleep 15
         forest-cli sync wait
         forest-cli snapshot export -t=$(ls /data/*.car.zst | tail -n 1 | grep -Eo '[0-9]+' | tail -n 1) -d=${EXPORT_EPOCHS} -o /data/exported/forest.car.zst

src/cli_shared/cli/client.rs

@@ -11,6 +11,8 @@ use directories::ProjectDirs;
 use serde::{Deserialize, Serialize};
 use serde_with::serde_as;
 
+use crate::daemon::db_util::ImportMode;
+
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(transparent)]
 #[cfg_attr(test, derive(derive_quickcheck_arbitrary::Arbitrary))]
@@ -41,11 +43,10 @@ pub struct Client {
     pub enable_rpc: bool,
     pub enable_metrics_endpoint: bool,
     pub enable_health_check: bool,
-    /// If this is true, delete the snapshot at `snapshot_path` if it's a local file.
-    pub consume_snapshot: bool,
     pub snapshot_height: Option<i64>,
     pub snapshot_head: Option<i64>,
     pub snapshot_path: Option<PathBuf>,
+    pub import_mode: ImportMode,
     /// Skips loading import CAR file and assumes it's already been loaded.
     /// Will use the CIDs in the header of the file to index the chain.
     pub skip_load: bool,
@@ -77,7 +78,7 @@ impl Default for Client {
             enable_metrics_endpoint: true,
             enable_health_check: true,
             snapshot_path: None,
-            consume_snapshot: false,
+            import_mode: ImportMode::default(),
             snapshot_height: None,
             snapshot_head: None,
             skip_load: false,

src/cli_shared/cli/mod.rs

@@ -9,9 +9,9 @@ use std::{
     path::{Path, PathBuf},
 };
 
-use crate::cli_shared::read_config;
 use crate::networks::NetworkChain;
 use crate::utils::misc::LoggingColor;
+use crate::{cli_shared::read_config, daemon::db_util::ImportMode};
 use ahash::HashSet;
 use clap::Parser;
 use directories::ProjectDirs;
@@ -84,9 +84,9 @@ pub struct CliOpts {
     /// Import a snapshot from a local CAR file or URL
     #[arg(long)]
     pub import_snapshot: Option<String>,
-    /// Import a snapshot from a local CAR file and delete it, or from a URL
-    #[arg(long)]
-    pub consume_snapshot: Option<String>,
+    /// Snapshot import mode. Available modes are `copy`, `move` and `symlink`.
+    #[arg(long, default_value = "copy")]
+    pub import_mode: ImportMode,
     /// Halt with exit code 0 after successfully importing a snapshot
     #[arg(long)]
     pub halt_after_import: bool,
@@ -194,17 +194,11 @@ impl CliOpts {
             cfg.network.listening_multiaddrs.clone_from(addresses);
         }
 
-        if self.import_snapshot.is_some() && self.consume_snapshot.is_some() {
-            anyhow::bail!("Can't set import_snapshot and consume_snapshot at the same time!")
-        }
-
         if let Some(snapshot_path) = &self.import_snapshot {
             cfg.client.snapshot_path = Some(snapshot_path.into());
+            cfg.client.import_mode = self.import_mode;
         }
-        if let Some(snapshot_path) = &self.consume_snapshot {
-            cfg.client.snapshot_path = Some(snapshot_path.into());
-            cfg.client.consume_snapshot = true;
-        }
+
         cfg.client.snapshot_height = self.height;
         cfg.client.snapshot_head = self.head.map(|head| head as i64);
         if let Some(skip_load) = self.skip_load {

src/daemon/db_util.rs

@@ -9,11 +9,11 @@ use crate::networks::Height;
 use crate::state_manager::StateManager;
 use crate::utils::db::car_stream::CarStream;
 use crate::utils::io::EitherMmapOrRandomAccessFile;
-use anyhow::Context as _;
+use anyhow::{bail, Context};
 use futures::TryStreamExt;
+use serde::{Deserialize, Serialize};
 use std::ffi::OsStr;
 use std::fs;
-use std::io;
 use std::{
     path::{Path, PathBuf},
     time,
@@ -59,41 +59,79 @@ pub fn load_all_forest_cars<T>(store: &ManyCar<T>, forest_car_db_dir: &Path) ->
     Ok(())
 }
 
+#[derive(
+    Default,
+    PartialEq,
+    Eq,
+    Debug,
+    Clone,
+    Copy,
+    strum::Display,
+    strum::EnumString,
+    Serialize,
+    Deserialize,
+)]
+#[strum(serialize_all = "lowercase")]
+#[cfg_attr(test, derive(derive_quickcheck_arbitrary::Arbitrary))]
+pub enum ImportMode {
+    #[default]
+    /// Copies the snapshot to the database directory.
+    Copy,
+    /// Moves the snapshot to the database directory (or copies and deletes the original).
+    Move,
+    /// Creates a symbolic link to the snapshot in the database directory.
+    Symlink,
+}
+
 /// This function validates and stores the CAR binary from `from_path`(either local path or URL) into the `{DB_ROOT}/car_db/`
 /// (automatically trans-code into `.forest.car.zst` format when needed), and returns its final file path and the heaviest tipset.
 pub async fn import_chain_as_forest_car(
     from_path: &Path,
     forest_car_db_dir: &Path,
-    consume_snapshot_file: bool,
+    import_mode: ImportMode,
 ) -> anyhow::Result<(PathBuf, Tipset)> {
     info!("Importing chain from snapshot at: {}", from_path.display());
 
     let stopwatch = time::Instant::now();
 
-    let downloaded_car_temp_path =
-        tempfile::NamedTempFile::new_in(forest_car_db_dir)?.into_temp_path();
-    if let Ok(url) = Url::parse(&from_path.display().to_string()) {
-        download_to(&url, &downloaded_car_temp_path).await?;
-    } else {
-        move_or_copy_file(from_path, &downloaded_car_temp_path, consume_snapshot_file)?;
-    }
-
     let forest_car_db_path = forest_car_db_dir.join(format!(
         "{}{FOREST_CAR_FILE_EXTENSION}",
         chrono::Utc::now().timestamp_millis()
     ));
 
-    if ForestCar::is_valid(&EitherMmapOrRandomAccessFile::open(
-        &downloaded_car_temp_path,
-    )?) {
-        downloaded_car_temp_path.persist(&forest_car_db_path)?;
-    } else {
-        // Use another temp file to make sure all final `.forest.car.zst` files are complete and valid.
-        let forest_car_db_temp_path =
-            tempfile::NamedTempFile::new_in(forest_car_db_dir)?.into_temp_path();
-        transcode_into_forest_car(&downloaded_car_temp_path, &forest_car_db_temp_path).await?;
-        forest_car_db_temp_path.persist(&forest_car_db_path)?;
-    }
+    match import_mode {
+        ImportMode::Copy | ImportMode::Move => {
+            let downloaded_car_temp_path =
+                tempfile::NamedTempFile::new_in(forest_car_db_dir)?.into_temp_path();
+            if let Ok(url) = Url::parse(&from_path.display().to_string()) {
+                download_to(&url, &downloaded_car_temp_path).await?;
+            } else {
+                move_or_copy_file(from_path, &downloaded_car_temp_path, import_mode)?;
+            }
+
+            if ForestCar::is_valid(&EitherMmapOrRandomAccessFile::open(
+                &downloaded_car_temp_path,
+            )?) {
+                downloaded_car_temp_path.persist(&forest_car_db_path)?;
+            } else {
+                // Use another temp file to make sure all final `.forest.car.zst` files are complete and valid.
+                let forest_car_db_temp_path =
+                    tempfile::NamedTempFile::new_in(forest_car_db_dir)?.into_temp_path();
+                transcode_into_forest_car(&downloaded_car_temp_path, &forest_car_db_temp_path)
+                    .await?;
+                forest_car_db_temp_path.persist(&forest_car_db_path)?;
+            }
+        }
+        ImportMode::Symlink => {
+            let from_path = std::path::absolute(from_path)?;
+            if ForestCar::is_valid(&EitherMmapOrRandomAccessFile::open(&from_path)?) {
+                std::os::unix::fs::symlink(from_path, &forest_car_db_path)
+                    .context("Error creating symlink")?;
+            } else {
+                bail!("Snapshot file must be a valid forest.car.zst file");
+            }
+        }
+    };
 
     let ts = ForestCar::try_from(forest_car_db_path.as_path())?.heaviest_tipset()?;
     info!(
@@ -124,15 +162,21 @@ pub async fn download_to(url: &Url, destination: &Path) -> anyhow::Result<()> {
     Ok(())
 }
 
-fn move_or_copy_file(from: &Path, to: &Path, consume: bool) -> io::Result<()> {
-    if consume && fs::rename(from, to).is_ok() {
-        Ok(())
-    } else {
-        fs::copy(from, to)?;
-        if consume {
-            fs::remove_file(from)?;
+fn move_or_copy_file(from: &Path, to: &Path, import_mode: ImportMode) -> anyhow::Result<()> {
+    match import_mode {
+        ImportMode::Move => {
+            if fs::rename(from, to).is_ok() {
+                Ok(())
+            } else {
+                fs::copy(from, to).context("Error copying file")?;
+                fs::remove_file(from).context("Error removing original file")?;
+                Ok(())
+            }
+        }
+        ImportMode::Copy => fs::copy(from, to).map(|_| ()).context("Error copying file"),
+        ImportMode::Symlink => {
+            bail!("Symlinking must be handled elsewhere");
         }
-        Ok(())
     }
 }
 
@@ -204,40 +248,91 @@ mod test {
 
     #[tokio::test]
     async fn import_snapshot_from_file_valid() {
-        import_snapshot_from_file("test-snapshots/chain4.car")
+        for import_mode in &[ImportMode::Copy, ImportMode::Move] {
+            import_snapshot_from_file("test-snapshots/chain4.car", *import_mode)
+                .await
+                .unwrap();
+        }
+
+        // Linking is not supported for raw CAR files.
+        import_snapshot_from_file("test-snapshots/chain4.car", ImportMode::Symlink)
             .await
-            .unwrap();
+            .unwrap_err();
     }
 
     #[tokio::test]
     async fn import_snapshot_from_compressed_file_valid() {
-        import_snapshot_from_file("test-snapshots/chain4.car.zst")
+        for import_mode in &[ImportMode::Copy, ImportMode::Move] {
+            import_snapshot_from_file("test-snapshots/chain4.car.zst", *import_mode)
+                .await
+                .unwrap();
+        }
+
+        // Linking is supported only for `forest.car.zst` files.
+        import_snapshot_from_file("test-snapshots/chain4.car.zst", ImportMode::Symlink)
             .await
-            .unwrap()
+            .unwrap_err();
+    }
+
+    #[tokio::test]
+    async fn import_snapshot_from_forest_car_valid() {
+        for import_mode in &[ImportMode::Copy, ImportMode::Move, ImportMode::Symlink] {
+            import_snapshot_from_file("test-snapshots/chain4.forest.car.zst", *import_mode)
+                .await
+                .unwrap();
+        }
     }
 
     #[tokio::test]
     async fn import_snapshot_from_file_invalid() {
-        import_snapshot_from_file("Cargo.toml").await.unwrap_err();
+        for import_mode in &[ImportMode::Copy, ImportMode::Move, ImportMode::Symlink] {
+            import_snapshot_from_file("Cargo.toml", *import_mode)
+                .await
+                .unwrap_err();
+        }
     }
 
     #[tokio::test]
     async fn import_snapshot_from_file_not_found() {
-        import_snapshot_from_file("dummy.car").await.unwrap_err();
+        for import_mode in &[ImportMode::Copy, ImportMode::Move, ImportMode::Symlink] {
+            import_snapshot_from_file("dummy.car", *import_mode)
+                .await
+                .unwrap_err();
+        }
     }
 
     #[tokio::test]
     async fn import_snapshot_from_url_not_found() {
-        import_snapshot_from_file("https://forest.chainsafe.io/dummy.car")
-            .await
-            .unwrap_err();
+        for import_mode in &[ImportMode::Copy, ImportMode::Move, ImportMode::Symlink] {
+            import_snapshot_from_file("https://forest.chainsafe.io/dummy.car", *import_mode)
+                .await
+                .unwrap_err();
+        }
     }
 
-    async fn import_snapshot_from_file(file_path: &str) -> anyhow::Result<()> {
-        let temp = tempfile::Builder::new().tempdir()?;
+    async fn import_snapshot_from_file(
+        file_path: &str,
+        import_mode: ImportMode,
+    ) -> anyhow::Result<()> {
+        // Prevent modifications on the original file, e.g., deletion via `ImportMode::Move`.
+        let temp_file = tempfile::Builder::new().tempfile()?;
+        fs::copy(Path::new(file_path), temp_file.path())?;
+        let file_path = temp_file.path();
+
+        let temp_db_dir = tempfile::Builder::new().tempdir()?;
         let (path, ts) =
-            import_chain_as_forest_car(Path::new(file_path), temp.path(), false).await?;
-        assert!(path.is_file());
+            import_chain_as_forest_car(file_path, temp_db_dir.path(), import_mode).await?;
+        match import_mode {
+            ImportMode::Symlink => {
+                assert_eq!(
+                    std::path::absolute(path.read_link()?)?,
+                    std::path::absolute(file_path)?
+                );
+            }
+            _ => {
+                assert!(path.is_file());
+            }
+        }
         assert!(ts.epoch() > 0);
         Ok(())
     }