Merge branch 'main' into elmattic/compress-cmd-tool-migration

.github/workflows/forest.yml

@@ -112,6 +112,29 @@ jobs:
       - name: forest-cli check
         run: ./scripts/tests/forest_cli_check.sh
 
+  # tool-specific tests
+  forest-tool-check:
+    needs:
+      - build-ubuntu
+    name: Forest TOOL checks
+    runs-on: ubuntu-latest
+    steps:
+      # To help investigate transient test failures
+      - run: lscpu
+      - name: Checkout Sources
+        uses: actions/checkout@v3
+      - uses: actions/download-artifact@v3
+        with:
+          name: forest-${{ runner.os }}
+          path: ~/.cargo/bin
+      # Permissions are lost during artifact-upload
+      # https://github.com/actions/upload-artifact#permission-loss
+      - name: Set permissions
+        run: |
+          chmod +x ~/.cargo/bin/forest*
+      - name: forest-tool check
+        run: ./scripts/tests/forest_tool_check.sh
+
   # miscallenous tests done on calibnet
   calibnet-check:
     needs:

CHANGELOG.md

@@ -55,6 +55,8 @@
   `forest-cli snapshot export`.
 - [#3348](https://github.com/ChainSafe/forest/pull/3348): Add `--diff-depth`
   flag to `forest-cli archive export`.
+- [#3325](https://github.com/ChainSafe/forest/pull/3325): Add
+  `forest-tool state-migration actor-bundle` subcommand.
 - [#3387](https://github.com/ChainSafe/forest/pull/3387): Add
   `forest-wallet delete` RPC command.
 - [#3322](https://github.com/ChainSafe/forest/issues/3322): Added prompt to

assets/ci_download.sh

@@ -5,4 +5,7 @@ set -eu
 
 SCRIPT_DIR=$(cd "$(dirname "$0")" ; pwd)
 
+# Note: When updating the bundle, to not break CI on main
+# do not replace `actor_bundles.car.zst` on DO space directly,
+# upload another `actor_bundles_yyyy_MM_dd.car.zst`
 curl -o "${SCRIPT_DIR}/actor_bundles.car.zst" https://forest-continuous-integration.fra1.cdn.digitaloceanspaces.com/assets/actor_bundles.car.zst

documentation/src/developer_documentation/state_migration_guide.md

@@ -36,15 +36,20 @@ HeightInfo {
 }
 ```
 
-- adding the bundle cid and url to the `ACTOR_BUNDLES` in the `src/mod.rs`.
+- adding the bundle cid and url to the `ACTOR_BUNDLES` in the
+  `src/networks/mod.rs`.
 
 ```rust
 ActorBundleInfo{
     manifest: Cid::try_from("bafy2bzacedbedgynklc4dgpyxippkxmba2mgtw7ecntoneclsvvl4klqwuyyy").unwrap(),
-    url: Url::parse("https://forest-continuous-integration.fra1.cdn.digitaloceanspaces.com/builtin-actors/calibnet/Shark.car").unwrap(),
+    url: Url::parse("https://github.com/filecoin-project/builtin-actors/releases/download/v9.0.3/builtin-actors-calibrationnet.car").unwrap(),
 },
 ```
 
+- regenerate a merged actor bundle with
+  `forest-tool state-migration actor-bundle` and replace
+  `assets/actor_bundles.car.zst`
+
 ### Implement the migration
 
 The next step is to implement the migration itself. In this guide, we will take

scripts/tests/forest_tool_check.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# This script is used to test the `forest-tool` commands that do not
+# require a running `forest` node.
+
+set -euxo pipefail
+
+FOREST_TOOL_PATH="forest-tool"
+
+"$FOREST_TOOL_PATH" state-migration actor-bundle

src/cli/subcommands/car_cmd.rs

@@ -4,12 +4,14 @@
 use std::path::PathBuf;
 
 use clap::Subcommand;
-use futures::{Stream, StreamExt, TryStreamExt};
+use futures::{StreamExt, TryStreamExt};
 use itertools::Itertools;
-use tokio::io::{AsyncBufRead, AsyncSeek, AsyncWriteExt};
+use tokio::io::AsyncWriteExt;
 
-use crate::ipld::CidHashSet;
-use crate::utils::db::car_stream::{Block, CarStream};
+use crate::utils::db::{
+    car_stream::CarStream,
+    car_util::{dedup_block_stream, merge_car_streams},
+};
 
 #[derive(Debug, Subcommand)]
 pub enum CarCommands {
@@ -52,148 +54,3 @@ impl CarCommands {
         Ok(())
     }
 }
-
-fn merge_car_streams<R>(
-    car_streams: Vec<CarStream<R>>,
-) -> impl Stream<Item = std::io::Result<Block>>
-where
-    R: AsyncSeek + AsyncBufRead + Unpin,
-{
-    futures::stream::iter(car_streams).flatten()
-}
-
-fn dedup_block_stream(
-    stream: impl Stream<Item = std::io::Result<Block>>,
-) -> impl Stream<Item = std::io::Result<Block>> {
-    let mut seen = CidHashSet::default();
-    stream.try_filter(move |Block { cid, data: _ }| futures::future::ready(seen.insert(*cid)))
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use ahash::HashSet;
-    use cid::multihash;
-    use cid::multihash::MultihashDigest;
-    use cid::Cid;
-    use futures::executor::{block_on, block_on_stream};
-    use fvm_ipld_encoding::DAG_CBOR;
-    use pretty_assertions::assert_eq;
-    use quickcheck::Arbitrary;
-    use quickcheck_macros::quickcheck;
-
-    #[derive(Debug, Clone)]
-    struct Blocks(Vec<Block>);
-
-    impl From<&Blocks> for HashSet<Cid> {
-        fn from(blocks: &Blocks) -> Self {
-            blocks.0.iter().map(|b| b.cid).collect()
-        }
-    }
-
-    impl Blocks {
-        async fn into_forest_car_zst_bytes(self) -> Vec<u8> {
-            let roots = vec![self.0[0].cid];
-            let frames = crate::db::car::forest::Encoder::compress_stream_default(
-                self.into_stream().map_err(anyhow::Error::from),
-            );
-            let mut writer = vec![];
-            crate::db::car::forest::Encoder::write(&mut writer, roots, frames)
-                .await
-                .unwrap();
-            writer
-        }
-
-        fn into_stream(self) -> impl Stream<Item = std::io::Result<Block>> {
-            futures::stream::iter(self.0).map(Ok)
-        }
-
-        /// Implicit clone is performed inside to simplify caller code
-        fn to_stream(&self) -> impl Stream<Item = std::io::Result<Block>> {
-            self.clone().into_stream()
-        }
-    }
-
-    impl Arbitrary for Blocks {
-        fn arbitrary(g: &mut quickcheck::Gen) -> Self {
-            // `CarReader` complains when n is 0: Error: Failed to parse CAR file: empty CAR file
-            let n = u8::arbitrary(g).saturating_add(1) as usize;
-            let mut blocks = Vec::with_capacity(n);
-            for _ in 0..n {
-                // use small len here to increase the chance of duplication
-                let data = [u8::arbitrary(g), u8::arbitrary(g)];
-                let cid = Cid::new_v1(DAG_CBOR, multihash::Code::Blake2b256.digest(&data));
-                let block = Block {
-                    cid,
-                    data: data.to_vec(),
-                };
-                blocks.push(block);
-            }
-            Self(blocks)
-        }
-    }
-
-    #[quickcheck]
-    fn blocks_roundtrip(blocks: Blocks) -> anyhow::Result<()> {
-        block_on(async move {
-            let car = blocks.into_forest_car_zst_bytes().await;
-            let reader = CarStream::new(std::io::Cursor::new(&car)).await?;
-            let blocks2 = Blocks(reader.try_collect().await?);
-            let car2 = blocks2.into_forest_car_zst_bytes().await;
-
-            assert_eq!(car, car2);
-
-            Ok::<_, anyhow::Error>(())
-        })
-    }
-
-    #[quickcheck]
-    fn dedup_block_stream_tests_a_a(a: Blocks) {
-        // ∀A. A∪A = A
-        assert_eq!(dedup_block_stream_wrapper(&a, &a), HashSet::from(&a));
-    }
-
-    #[quickcheck]
-    fn dedup_block_stream_tests_a_b(a: Blocks, b: Blocks) {
-        let union_a_b = dedup_block_stream_wrapper(&a, &b);
-        let union_b_a = dedup_block_stream_wrapper(&b, &a);
-        // ∀AB. A∪B = B∪A
-        assert_eq!(union_a_b, union_b_a);
-        // ∀AB. A⊆(A∪B)
-        union_a_b.is_superset(&HashSet::from(&a));
-        // ∀AB. B⊆(A∪B)
-        union_a_b.is_superset(&HashSet::from(&b));
-    }
-
-    fn dedup_block_stream_wrapper(a: &Blocks, b: &Blocks) -> HashSet<Cid> {
-        let blocks: Vec<Cid> =
-            block_on_stream(dedup_block_stream(a.to_stream().chain(b.to_stream())))
-                .map(|block| block.unwrap().cid)
-                .collect();
-
-        // Ensure `dedup_block_stream` works properly
-        assert!(blocks.iter().all_unique());
-
-        HashSet::from_iter(blocks)
-    }
-
-    #[quickcheck]
-    fn car_dedup_block_stream_tests(a: Blocks, b: Blocks) -> anyhow::Result<()> {
-        let cid_union = HashSet::from_iter(HashSet::from(&a).union(&HashSet::from(&b)).cloned());
-
-        block_on(async move {
-            let car_a = std::io::Cursor::new(a.into_forest_car_zst_bytes().await);
-            let car_b = std::io::Cursor::new(b.into_forest_car_zst_bytes().await);
-            let deduped = dedup_block_stream(merge_car_streams(vec![
-                CarStream::new(car_a).await?,
-                CarStream::new(car_b).await?,
-            ]));
-
-            let cid_union2: HashSet<Cid> = deduped.map_ok(|block| block.cid).try_collect().await?;
-
-            assert_eq!(cid_union, cid_union2);
-
-            Ok::<_, anyhow::Error>(())
-        })
-    }
-}

src/cli/subcommands/mod.rs

@@ -9,14 +9,14 @@
 mod archive_cmd;
 mod attach_cmd;
 mod auth_cmd;
-mod car_cmd;
+pub(crate) mod car_cmd;
 mod chain_cmd;
 mod config_cmd;
 mod db_cmd;
 mod info_cmd;
 mod mpool_cmd;
 mod net_cmd;
-pub mod send_cmd;
+pub(crate) mod send_cmd;
 mod shutdown_cmd;
 mod snapshot_cmd;
 mod state_cmd;

src/daemon/bundle.rs

@@ -22,7 +22,7 @@ pub async fn load_actor_bundles(db: &impl Blockstore) -> anyhow::Result<Vec<Cid>
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::build::ACTOR_BUNDLES;
+    use crate::networks::ACTOR_BUNDLES;
     use ahash::HashSet;
     use pretty_assertions::assert_eq;
 

src/lib.rs

@@ -36,7 +36,6 @@ mod lotus_json;
 mod message;
 mod message_pool;
 mod metrics;
-mod r#mod;
 mod networks;
 mod rpc;
 mod rpc_api;
@@ -51,10 +50,6 @@ mod tool;
 mod utils;
 mod wallet;
 
-pub mod build {
-    pub use super::r#mod::*;
-}
-
 /// These items are semver-exempt, and exist for forest author use only
 // We want to have doctests, but don't want our internals to be public because:
 // - We don't want to be concerned with library compat