Forest tool migration of forest-cli subcommands (#3336)

Co-authored-by: David Himmelstrup <david.himmelstrup@chainsafe.io>

.github/CHECKPOINT_ISSUE_TEMPLATE.md

@@ -10,7 +10,7 @@ Checkpoints have to be regularly updated, though, and this issue is automaticall
 How to compute a new checkpoint for calibnet:
 
 1. Install `forest-cli`
-2. Download calibnet snapshot: `forest-cli --chain calibnet snapshot fetch`
+2. Download calibnet snapshot: `forest-tool snapshot fetch --chain calibnet`
 3. Decompress snapshot: `zstd -d forest_snapshot_calibnet_*.car.zst`
 4. Extract checkpoints: `forest-cli archive checkpoints forest_snapshot_calibnet_*.car`
 5. Put checkpoints in `build/known_blocks.yaml`

.github/workflows/release.yml

@@ -37,14 +37,15 @@ jobs:
         timeout-minutes: 5
         continue-on-error: true
       - name: Cargo Build
-        run: cargo build --release --bin forest --bin forest-cli
+        run: cargo build --release --bin forest --bin forest-cli --bin forest-tool
       - name: Compress Binary
         run: |
           mkdir -p forest-${{ github.ref_name }}
-          cp -v target/release/forest target/release/forest-cli forest-${{ github.ref_name }}
+          cp -v target/release/forest target/release/forest-cli target/release/forest-tool forest-${{ github.ref_name }}
           cp -rv CHANGELOG.md LICENSE-APACHE LICENSE-MIT README.md documentation forest-${{ github.ref_name }}
           sha256sum forest-${{ github.ref_name }}/forest > forest-${{ github.ref_name }}/forest.sha256
           sha256sum forest-${{ github.ref_name }}/forest-cli > forest-${{ github.ref_name }}/forest-cli.sha256
+          sha256sum forest-${{ github.ref_name }}/forest-tool > forest-${{ github.ref_name }}/forest-tool.sha256
           zip -r ${{ matrix.file }} forest-${{ github.ref_name }}
       - name: Upload Binary
         uses: svenstaro/upload-release-action@v2

CHANGELOG.md

@@ -31,6 +31,15 @@
   internal settings from files to the database.
 - [#3333](https://github.com/ChainSafe/forest/pull/3333) Changed default rpc
   port from 1234 to 2345.
+- [#3336](https://github.com/ChainSafe/forest/pull/3336) Moved following
+  `forest-cli` subcommands to `forest-tool`
+  - `archive info`
+  - `fetch-params`
+  - `snapshot fetch`
+  - `snapshot validate`
+- [#3355](https://github.com/ChainSafe/forest/pull/3355) Moved commands
+  - `forest-cli db stats` to `forest-tool db stats`
+  - `forest-cli db clean` to `forest-tool db destroy`
 
 ### Added
 

Dockerfile

@@ -87,6 +87,6 @@ USER ${SERVICE_USER}
 WORKDIR /home/${SERVICE_USER}
 
 # Basic verification of dynamically linked dependencies
-RUN forest -V && forest-cli -V
+RUN forest -V && forest-cli -V && forest-tool -V
 
 ENTRYPOINT ["forest"]

Dockerfile-alpine

@@ -64,6 +64,6 @@ USER ${SERVICE_USER}
 WORKDIR /home/${SERVICE_USER}
 
 # Basic verification of dynamically linked dependencies
-RUN forest -V && forest-cli -V
+RUN forest -V && forest-cli -V && forest-tool -V
 
 ENTRYPOINT ["forest"]

README.md

@@ -269,6 +269,17 @@ The admin token can also be set using `--token` flag.
 forest-cli --token <ADMIN_TOKEN>
 ```
 
+### Forest executable organization
+
+The binaries in the Forest repository are organized into the following
+categories:
+
+| Binary        | Role                                                     | Command example                                    |
+| ------------- | -------------------------------------------------------- | -------------------------------------------------- |
+| `forest`      | Forest daemon, used to connect to the Filecoin network   | `forest --chain calibnet --encrypt-keystore false` |
+| `forest-cli`  | Human-friendly wrappers around the Filecoin JSON-RPC API | `forest-cli info show`                             |
+| `forest-tool` | Handle tasks not involving the Forest daemon             | `forest-tool snapshot fetch`                       |
+
 ### Detaching Forest process
 
 You can detach Forest process via the `--detach` flag so that it runs in the

scripts/db_params_hyperfine.sh

@@ -22,4 +22,4 @@ hyperfine \
     ./target/release/forest \
       --chain ${CHAIN} --config /tmp/forest.conf --rpc false --no-gc --encrypt-keystore false --halt-after-import \
       --import-snapshot ${SNAPSHOT}; \
-    ./target/release/forest-cli --chain ${CHAIN} db clean --force"
+    ./target/release/forest-tool db destroy --chain ${CHAIN} --force"

scripts/gen_coverage_report.sh

@@ -31,17 +31,17 @@ function cov {
 cargo llvm-cov --workspace clean
 cargo llvm-cov --workspace --no-report
 cov forest-cli --chain calibnet db clean --force
-cov forest-cli --chain calibnet snapshot fetch --aria2 --provider filecoin -s "$TMP_DIR"
+cov forest-tool snapshot fetch --chain calibnet --vendor filops -s "$TMP_DIR"
 SNAPSHOT_PATH=$(find "$TMP_DIR" -name \*.zst | head -n 1)
 cov forest --chain calibnet --encrypt-keystore false --import-snapshot "$SNAPSHOT_PATH" --halt-after-import --height=-200 --track-peak-rss
 cov forest-cli --chain calibnet db clean --force
-cov forest-cli --chain calibnet snapshot fetch --aria2 -s "$TMP_DIR"
+cov forest-tool snapshot fetch --chain calibnet -s "$TMP_DIR"
 SNAPSHOT_PATH=$(find "$TMP_DIR" -name \*.car | head -n 1)
 cov forest --chain calibnet --encrypt-keystore false --import-snapshot "$SNAPSHOT_PATH" --height=-200 --detach --track-peak-rss --save-token "$TOKEN_PATH"
 cov forest-cli sync wait
 cov forest-cli sync status
 cov forest-cli --chain calibnet db gc
-cov forest-cli --chain calibnet db stats
+cov forest-tool db stats --chain calibnet
 cov forest-cli snapshot export
 cov forest-cli snapshot export
 cov forest-cli attach --exec 'showPeers()'

scripts/tests/calibnet_export_check.sh

@@ -25,5 +25,5 @@ echo "Validating CAR files"
 zstd --decompress ./*.car.zst
 for f in *.car; do
   echo "Validating CAR file $f"
-  $FOREST_CLI_PATH --chain calibnet snapshot validate "$f"
+  $FOREST_TOOL_PATH snapshot validate "$f"
 done

scripts/tests/forest_cli_check.sh

@@ -19,17 +19,17 @@ function num-files-here() {
         | wc --lines
 }
 
-"$FOREST_CLI_PATH" fetch-params --keys
+"$FOREST_TOOL_PATH" fetch-params --keys
 
-: "cleaning an empty database doesn't fail (see #2811)"
-"$FOREST_CLI_PATH" --chain calibnet db clean --force
-"$FOREST_CLI_PATH" --chain calibnet db clean --force
+: "destroying an empty database doesn't fail (see #2811)"
+"$FOREST_TOOL_PATH" db destroy --chain calibnet --force
+"$FOREST_TOOL_PATH" db destroy --chain calibnet --force
 
 
 : fetch snapshot
 pushd "$(mktemp --directory)"
-    "$FOREST_CLI_PATH" --chain calibnet snapshot fetch --vendor forest
-    "$FOREST_CLI_PATH" --chain calibnet snapshot fetch --vendor filops
+    "$FOREST_TOOL_PATH" snapshot fetch --chain calibnet --vendor forest
+    "$FOREST_TOOL_PATH" snapshot fetch --chain calibnet --vendor filops
     # this will fail if they happen to have the same height - we should change the format of our filenames
     test "$(num-files-here)" -eq 2
 
@@ -59,7 +59,7 @@ pushd "$(mktemp --directory)"
     #assert_eq "$DIFF_STATE_ROOTS" 1100
 
     : Validate the union of a snapshot and a diff
-    "$FOREST_CLI_PATH" snapshot validate --check-network calibnet base_snapshot.forest.car.zst diff_snapshot.forest.car.zst
+    "$FOREST_TOOL_PATH" snapshot validate --check-network calibnet base_snapshot.forest.car.zst diff_snapshot.forest.car.zst
 rm -- *
 popd
 
@@ -68,7 +68,7 @@ popd
 : validate latest calibnet snapshot
 pushd "$(mktemp --directory)"
     : : fetch a compressed calibnet snapshot
-    "$FOREST_CLI_PATH" --chain calibnet snapshot fetch
+    "$FOREST_TOOL_PATH" snapshot fetch --chain calibnet
     test "$(num-files-here)" -eq 1
     uncompress_me=$(find . -type f | head -1)
 
@@ -77,10 +77,10 @@ pushd "$(mktemp --directory)"
 
     validate_me=$(find . -type f | head -1)
     : : validating under calibnet chain should succeed
-    "$FOREST_CLI_PATH" snapshot validate --check-network calibnet "$validate_me"
+    "$FOREST_TOOL_PATH" snapshot validate --check-network calibnet "$validate_me"
 
     : : validating under mainnet chain should fail
-    if "$FOREST_CLI_PATH" snapshot validate --check-network mainnet "$validate_me"; then
+    if "$FOREST_TOOL_PATH" snapshot validate --check-network mainnet "$validate_me"; then
         exit 1
     fi
 

scripts/tests/harness.sh

@@ -5,6 +5,7 @@
 
 FOREST_PATH="forest"
 FOREST_CLI_PATH="forest-cli"
+FOREST_TOOL_PATH="forest-tool"
 
 TMP_DIR=$(mktemp --directory)
 LOG_DIRECTORY=$TMP_DIR/logs
@@ -19,19 +20,19 @@ function forest_download_and_import_snapshot {
 
 function forest_check_db_stats {
   echo "Checking DB stats"
-  $FOREST_CLI_PATH --chain calibnet db stats
+  $FOREST_TOOL_PATH db stats --chain calibnet
 }
 
 function forest_query_epoch {
-  $FOREST_CLI_PATH archive info "$1" | grep Epoch | awk '{print $2}'
+  $FOREST_TOOL_PATH archive info "$1" | grep Epoch | awk '{print $2}'
 }
 
 function forest_query_state_roots {
-  $FOREST_CLI_PATH archive info "$1" | grep State-roots | awk '{print $2}'
+  $FOREST_TOOL_PATH archive info "$1" | grep State-roots | awk '{print $2}'
 }
 
 function forest_query_format {
-  $FOREST_CLI_PATH archive info "$1" | grep "CAR format" | awk '{print $3}'
+  $FOREST_TOOL_PATH archive info "$1" | grep "CAR format" | awk '{print $3}'
 }
 
 function forest_run_node_detached {

src/cli/main.rs

@@ -7,6 +7,7 @@ use std::sync::Arc;
 use crate::cli_shared::logger;
 use crate::networks::ChainConfig;
 use crate::shim::address::{CurrentNetwork, Network};
+use crate::utils::bail_moved_cmd;
 use crate::utils::io::ProgressBar;
 use crate::{
     cli::subcommands::{cli_error_and_die, Cli},
@@ -50,7 +51,9 @@ where
                     }
                     // Run command
                     match cmd {
-                        Subcommand::Fetch(cmd) => cmd.run(config).await,
+                        Subcommand::Fetch(_cmd) => {
+                            bail_moved_cmd("fetch-params", "forest-tool fetch-params")
+                        }
                         Subcommand::Chain(cmd) => cmd.run(config).await,
                         Subcommand::Auth(cmd) => cmd.run(config).await,
                         Subcommand::Net(cmd) => cmd.run(config).await,