Forest tool migration of forest-cli subcommands

.github/CHECKPOINT_ISSUE_TEMPLATE.md

@@ -10,7 +10,7 @@ Checkpoints have to be regularly updated, though, and this issue is automaticall
 How to compute a new checkpoint for calibnet:
 
 1. Install `forest-cli`
-2. Download calibnet snapshot: `forest-cli --chain calibnet snapshot fetch`
+2. Download calibnet snapshot: `forest-tool snapshot fetch --chain calibnet`
 3. Decompress snapshot: `zstd -d forest_snapshot_calibnet_*.car.zst`
 4. Extract checkpoints: `forest-cli archive checkpoints forest_snapshot_calibnet_*.car`
 5. Put checkpoints in `build/known_blocks.yaml`

.github/workflows/release.yml

@@ -36,14 +36,15 @@ jobs:
         timeout-minutes: 5
         continue-on-error: true
       - name: Cargo Build
-        run: cargo build --release --bin forest --bin forest-cli
+        run: cargo build --release --bin forest --bin forest-cli --bin forest-tool
       - name: Compress Binary
         run: |
           mkdir -p forest-${{ github.ref_name }}
-          cp -v target/release/forest target/release/forest-cli forest-${{ github.ref_name }}
+          cp -v target/release/forest target/release/forest-cli target/release/forest-tool forest-${{ github.ref_name }}
           cp -rv CHANGELOG.md LICENSE-APACHE LICENSE-MIT README.md documentation forest-${{ github.ref_name }}
           sha256sum forest-${{ github.ref_name }}/forest > forest-${{ github.ref_name }}/forest.sha256
           sha256sum forest-${{ github.ref_name }}/forest-cli > forest-${{ github.ref_name }}/forest-cli.sha256
+          sha256sum forest-${{ github.ref_name }}/forest-tool > forest-${{ github.ref_name }}/forest-tool.sha256
           zip -r ${{ matrix.file }} forest-${{ github.ref_name }}
       - name: Upload Binary
         uses: svenstaro/upload-release-action@v2

CHANGELOG.md

@@ -31,6 +31,12 @@
   internal settings from files to the database.
 - [#3333](https://github.com/ChainSafe/forest/pull/3333) Changed default rpc
   port from 1234 to 2345.
+- [#3336](https://github.com/ChainSafe/forest/pull/3336) Moved following
+  `forest-cli` subcommands to `forest-tool`
+  - `archive info`
+  - `fetch-params`
+  - `snapshot fetch`
+  - `snapshot validate`
 
 ### Added
 

Dockerfile

@@ -87,6 +87,6 @@ USER ${SERVICE_USER}
 WORKDIR /home/${SERVICE_USER}
 
 # Basic verification of dynamically linked dependencies
-RUN forest -V && forest-cli -V
+RUN forest -V && forest-cli -V && forest-tool -V
 
 ENTRYPOINT ["forest"]

README.md

@@ -268,6 +268,17 @@ The admin token can also be set using `--token` flag.
 forest-cli --token <ADMIN_TOKEN>
 ```
 
+### Forest executable organization
+
+The binaries in the Forest repository are organized into the following
+categories:
+
+| Binary        | Role                                                     | Command example                                    |
+| ------------- | -------------------------------------------------------- | -------------------------------------------------- |
+| `forest`      | Forest daemon, used to connect to the Filecoin network   | `forest --chain calibnet --encrypt-keystore false` |
+| `forest-cli`  | Human-friendly wrappers around the Filecoin JSON-RPC API | `forest-cli info show`                             |
+| `forest-tool` | Handle tasks not involving the Forest daemon             | `forest-tool snapshot fetch`                       |
+
 ### Detaching Forest process
 
 You can detach Forest process via the `--detach` flag so that it runs in the

scripts/gen_coverage_report.sh

@@ -31,11 +31,11 @@ function cov {
 cargo llvm-cov --workspace clean
 cargo llvm-cov --workspace --no-report
 cov forest-cli --chain calibnet db clean --force
-cov forest-cli --chain calibnet snapshot fetch --aria2 --provider filecoin -s "$TMP_DIR"
+cov forest-tool snapshot fetch --chain calibnet --vendor filops -s "$TMP_DIR"
 SNAPSHOT_PATH=$(find "$TMP_DIR" -name \*.zst | head -n 1)
 cov forest --chain calibnet --encrypt-keystore false --import-snapshot "$SNAPSHOT_PATH" --halt-after-import --height=-200 --track-peak-rss
 cov forest-cli --chain calibnet db clean --force
-cov forest-cli --chain calibnet snapshot fetch --aria2 -s "$TMP_DIR"
+cov forest-tool snapshot fetch --chain calibnet -s "$TMP_DIR"
 SNAPSHOT_PATH=$(find "$TMP_DIR" -name \*.car | head -n 1)
 cov forest --chain calibnet --encrypt-keystore false --import-snapshot "$SNAPSHOT_PATH" --height=-200 --detach --track-peak-rss --save-token "$TOKEN_PATH"
 cov forest-cli sync wait

scripts/tests/calibnet_export_check.sh

@@ -25,5 +25,5 @@ echo "Validating CAR files"
 zstd --decompress ./*.car.zst
 for f in *.car; do
   echo "Validating CAR file $f"
-  $FOREST_CLI_PATH --chain calibnet snapshot validate "$f"
+  $FOREST_TOOL_PATH snapshot validate "$f"
 done

scripts/tests/forest_cli_check.sh

@@ -19,7 +19,7 @@ function num-files-here() {
         | wc --lines
 }
 
-"$FOREST_CLI_PATH" fetch-params --keys
+"$FOREST_TOOL_PATH" fetch-params --keys
 
 : "cleaning an empty database doesn't fail (see #2811)"
 "$FOREST_CLI_PATH" --chain calibnet db clean --force
@@ -28,8 +28,8 @@ function num-files-here() {
 
 : fetch snapshot
 pushd "$(mktemp --directory)"
-    "$FOREST_CLI_PATH" --chain calibnet snapshot fetch --vendor forest
-    "$FOREST_CLI_PATH" --chain calibnet snapshot fetch --vendor filops
+    "$FOREST_TOOL_PATH" snapshot fetch --chain calibnet --vendor forest
+    "$FOREST_TOOL_PATH" snapshot fetch --chain calibnet --vendor filops
     # this will fail if they happen to have the same height - we should change the format of our filenames
     test "$(num-files-here)" -eq 2
 
@@ -59,7 +59,7 @@ pushd "$(mktemp --directory)"
     #assert_eq "$DIFF_STATE_ROOTS" 1100
 
     : Validate the union of a snapshot and a diff
-    "$FOREST_CLI_PATH" snapshot validate --check-network calibnet base_snapshot.forest.car.zst diff_snapshot.forest.car.zst
+    "$FOREST_TOOL_PATH" snapshot validate --check-network calibnet base_snapshot.forest.car.zst diff_snapshot.forest.car.zst
 rm -- *
 popd
 
@@ -68,7 +68,7 @@ popd
 : validate latest calibnet snapshot
 pushd "$(mktemp --directory)"
     : : fetch a compressed calibnet snapshot
-    "$FOREST_CLI_PATH" --chain calibnet snapshot fetch
+    "$FOREST_TOOL_PATH" snapshot fetch --chain calibnet
     test "$(num-files-here)" -eq 1
     uncompress_me=$(find . -type f | head -1)
 
@@ -77,10 +77,10 @@ pushd "$(mktemp --directory)"
 
     validate_me=$(find . -type f | head -1)
     : : validating under calibnet chain should succeed
-    "$FOREST_CLI_PATH" snapshot validate --check-network calibnet "$validate_me"
+    "$FOREST_TOOL_PATH" snapshot validate --check-network calibnet "$validate_me"
 
     : : validating under mainnet chain should fail
-    if "$FOREST_CLI_PATH" snapshot validate --check-network mainnet "$validate_me"; then
+    if "$FOREST_TOOL_PATH" snapshot validate --check-network mainnet "$validate_me"; then
         exit 1
     fi
 

scripts/tests/harness.sh

@@ -5,6 +5,7 @@
 
 FOREST_PATH="forest"
 FOREST_CLI_PATH="forest-cli"
+FOREST_TOOL_PATH="forest-tool"
 
 TMP_DIR=$(mktemp --directory)
 LOG_DIRECTORY=$TMP_DIR/logs
@@ -23,15 +24,15 @@ function forest_check_db_stats {
 }
 
 function forest_query_epoch {
-  $FOREST_CLI_PATH archive info "$1" | grep Epoch | awk '{print $2}'
+  $FOREST_TOOL_PATH archive info "$1" | grep Epoch | awk '{print $2}'
 }
 
 function forest_query_state_roots {
-  $FOREST_CLI_PATH archive info "$1" | grep State-roots | awk '{print $2}'
+  $FOREST_TOOL_PATH archive info "$1" | grep State-roots | awk '{print $2}'
 }
 
 function forest_query_format {
-  $FOREST_CLI_PATH archive info "$1" | grep "CAR format" | awk '{print $3}'
+  $FOREST_TOOL_PATH archive info "$1" | grep "CAR format" | awk '{print $3}'
 }
 
 function forest_run_node_detached {

src/cli/main.rs

@@ -16,6 +16,16 @@ use clap::Parser;
 
 use super::subcommands::Subcommand;
 
+#[macro_export]
+macro_rules! bail_moved_cmd {
+    ($src:literal) => {
+        anyhow::bail!(
+            "Invalid subcommand: {}. It has been moved to forest-tool binary.",
+            $src
+        )
+    };
+}
+
 pub fn main<ArgT>(args: impl IntoIterator<Item = ArgT>) -> anyhow::Result<()>
 where
     ArgT: Into<OsString> + Clone,
@@ -50,7 +60,7 @@ where
                     }
                     // Run command
                     match cmd {
-                        Subcommand::Fetch(cmd) => cmd.run(config).await,
+                        Subcommand::Fetch(_cmd) => bail_moved_cmd!("fetch-params"),
                         Subcommand::Chain(cmd) => cmd.run(config).await,
                         Subcommand::Auth(cmd) => cmd.run(config).await,
                         Subcommand::Net(cmd) => cmd.run(config).await,

src/cli/subcommands/archive_cmd.rs

@@ -32,7 +32,7 @@ use crate::chain::{
     ChainEpochDelta,
 };
 use crate::cli_shared::{snapshot, snapshot::TrustedVendor};
-use crate::db::car::{AnyCar, ManyCar, RandomAccessFileReader};
+use crate::db::car::ManyCar;
 use crate::ipld::{stream_graph, CidHashSet};
 use crate::networks::{calibnet, mainnet, ChainConfig, NetworkChain};
 use crate::shim::clock::{ChainEpoch, EPOCHS_IN_DAY, EPOCH_DURATION_SECONDS};
@@ -41,20 +41,16 @@ use chrono::NaiveDateTime;
 use clap::Subcommand;
 use futures::TryStreamExt;
 use fvm_ipld_blockstore::Blockstore;
-use indicatif::ProgressIterator;
-use itertools::Itertools;
 use sha2::Sha256;
 use std::path::PathBuf;
 use std::sync::Arc;
 use tracing::info;
 
 #[derive(Debug, Subcommand)]
 pub enum ArchiveCommands {
-    /// Show basic information about an archive.
-    Info {
-        /// Path to an uncompressed archive (CAR)
-        snapshot: PathBuf,
-    },
+    // This subcommand is hidden and only here to help users migrating to forest-tool
+    #[command(hide = true)]
+    Info { snapshot: PathBuf },
     /// Trim a snapshot of the chain and write it to `<output_path>`
     Export {
         /// Snapshot input path. Currently supports only `.car` file format.
@@ -90,10 +86,7 @@ pub enum ArchiveCommands {
 impl ArchiveCommands {
     pub async fn run(self) -> anyhow::Result<()> {
         match self {
-            Self::Info { snapshot } => {
-                println!("{}", ArchiveInfo::from_store(AnyCar::try_from(snapshot)?)?);
-                Ok(())
-            }
+            Self::Info { .. } => crate::bail_moved_cmd!("archive info"),
             Self::Export {
                 snapshot_files,
                 output_path,
@@ -233,112 +226,6 @@ async fn do_export(
     Ok(())
 }
 
-#[derive(Debug)]
-struct ArchiveInfo {
-    variant: String,
-    network: String,
-    epoch: ChainEpoch,
-    tipsets: ChainEpoch,
-    messages: ChainEpoch,
-}
-
-impl std::fmt::Display for ArchiveInfo {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        writeln!(f, "CAR format:    {}", self.variant)?;
-        writeln!(f, "Network:       {}", self.network)?;
-        writeln!(f, "Epoch:         {}", self.epoch)?;
-        writeln!(f, "State-roots:   {}", self.epoch - self.tipsets + 1)?;
-        write!(f, "Messages sets: {}", self.epoch - self.messages + 1)?;
-        Ok(())
-    }
-}
-
-impl ArchiveInfo {
-    // Scan a CAR archive to identify which network it belongs to and how many
-    // tipsets/messages are available. Progress is rendered to stdout.
-    fn from_store(store: AnyCar<impl RandomAccessFileReader>) -> anyhow::Result<Self> {
-        Self::from_store_with(store, true)
-    }
-
-    // Scan a CAR archive to identify which network it belongs to and how many
-    // tipsets/messages are available. Progress is optionally rendered to
-    // stdout.
-    fn from_store_with(
-        store: AnyCar<impl RandomAccessFileReader>,
-        progress: bool,
-    ) -> anyhow::Result<Self> {
-        let root = store.heaviest_tipset()?;
-        let root_epoch = root.epoch();
-
-        let tipsets = root.clone().chain(&store);
-
-        let windowed = (std::iter::once(root).chain(tipsets)).tuple_windows();
-
-        let mut network: String = "unknown".into();
-        let mut lowest_stateroot_epoch = root_epoch;
-        let mut lowest_message_epoch = root_epoch;
-
-        let iter = if progress {
-            itertools::Either::Left(windowed.progress_count(root_epoch as u64))
-        } else {
-            itertools::Either::Right(windowed)
-        };
-
-        for (parent, tipset) in iter {
-            if tipset.epoch() >= parent.epoch() && parent.epoch() != root_epoch {
-                bail!("Broken invariant: non-sequential epochs");
-            }
-
-            if tipset.epoch() < 0 {
-                bail!("Broken invariant: tipset with negative epoch");
-            }
-
-            // Update the lowest-stateroot-epoch only if our parent also has a
-            // state-root. The genesis state-root is usually available but we're
-            // not interested in that.
-            if lowest_stateroot_epoch == parent.epoch() && store.has(tipset.parent_state())? {
-                lowest_stateroot_epoch = tipset.epoch();
-            }
-            if lowest_message_epoch == parent.epoch()
-                && store.has(tipset.min_ticket_block().messages())?
-            {
-                lowest_message_epoch = tipset.epoch();
-            }
-
-            if tipset.epoch() == 0 {
-                if tipset.min_ticket_block().cid() == &*calibnet::GENESIS_CID {
-                    network = "calibnet".into();
-                } else if tipset.min_ticket_block().cid() == &*mainnet::GENESIS_CID {
-                    network = "mainnet".into();
-                }
-            }
-
-            // If we've already found the lowest-stateroot-epoch and
-            // lowest-message-epoch then we can skip scanning the rest of the
-            // archive when we find a checkpoint.
-            let may_skip =
-                lowest_stateroot_epoch != tipset.epoch() && lowest_message_epoch != tipset.epoch();
-            if may_skip {
-                let genesis_block = tipset.genesis(&store)?;
-                if genesis_block.cid() == &*calibnet::GENESIS_CID {
-                    network = "calibnet".into();
-                } else if genesis_block.cid() == &*mainnet::GENESIS_CID {
-                    network = "mainnet".into();
-                }
-                break;
-            }
-        }
-
-        Ok(ArchiveInfo {
-            variant: store.variant().to_string(),
-            network,
-            epoch: root_epoch,
-            tipsets: lowest_stateroot_epoch,
-            messages: lowest_message_epoch,
-        })
-    }
-}
-
 // Print a mapping of epochs to block headers in yaml format. This mapping can
 // be used by Forest to quickly identify tipsets.
 fn print_checkpoints(snapshot_files: Vec<PathBuf>) -> anyhow::Result<()> {
@@ -380,34 +267,13 @@ fn list_checkpoints(
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::db::car::AnyCar;
     use async_compression::tokio::bufread::ZstdDecoder;
     use fvm_ipld_car::CarReader;
     use tempfile::TempDir;
     use tokio::io::BufReader;
     use tokio_util::compat::TokioAsyncReadCompatExt;
 
-    #[test]
-    fn archive_info_calibnet() {
-        let info = ArchiveInfo::from_store_with(
-            AnyCar::try_from(calibnet::DEFAULT_GENESIS).unwrap(),
-            false,
-        )
-        .unwrap();
-        assert_eq!(info.network, "calibnet");
-        assert_eq!(info.epoch, 0);
-    }
-
-    #[test]
-    fn archive_info_mainnet() {
-        let info = ArchiveInfo::from_store_with(
-            AnyCar::try_from(mainnet::DEFAULT_GENESIS).unwrap(),
-            false,
-        )
-        .unwrap();
-        assert_eq!(info.network, "mainnet");
-        assert_eq!(info.epoch, 0);
-    }
-
     fn genesis_timestamp(genesis_car: &'static [u8]) -> u64 {
         let db = crate::db::car::PlainCar::try_from(genesis_car).unwrap();
         let ts = db.heaviest_tipset().unwrap();