diff --git a/Cargo.lock b/Cargo.lock index 0dc9f7c120aa..98ee7bfba646 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -310,7 +310,7 @@ dependencies = [ "num-traits", "rusticata-macros", "thiserror", - "time 0.3.23", + "time 0.3.24", ] [[package]] @@ -326,7 +326,7 @@ dependencies = [ "num-traits", "rusticata-macros", "thiserror", - "time 0.3.23", + "time 0.3.24", ] [[package]] @@ -454,7 +454,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -476,7 +476,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -493,7 +493,7 @@ checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -946,7 +946,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05" dependencies = [ "memchr", - "regex-automata 0.3.3", + "regex-automata 0.3.4", "serde", ] @@ -1202,7 +1202,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -1822,7 +1822,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -1844,7 +1844,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core 0.20.3", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -1921,6 +1921,15 @@ dependencies = [ "rusticata-macros", ] +[[package]] +name = "deranged" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8810e7e2cf385b1e9b50d68264908ec367ba642c96d02edfe61c39e88e2a3c01" +dependencies = [ + "serde", +] + [[package]] name = "derive-getters" version = "0.3.0" @@ -1940,7 +1949,7 @@ checksum = "53e0efad4403bfc52dc201159c4b842a246a14b98c64b55dfd0f2d89729dfeb8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -2091,7 +2100,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -2263,9 +2272,9 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" dependencies = [ "errno-dragonfly", "libc", @@ -2326,7 +2335,7 @@ checksum = "55a9a55d1dab3b07854648d48e366f684aefe2ac78ae28cec3bf65e3cd53d9a3" dependencies = [ "execute-command-tokens", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -3139,7 +3148,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -4778,9 +4787,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.10" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24e6ab01971eb092ffe6a7d42f49f9ff42662f17604681e2843ad65077ba47dc" +checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" dependencies = [ "cc", "pkg-config", @@ -4916,9 +4925,9 @@ checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" [[package]] name = "matchit" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" +checksum = "67827e6ea8ee8a7c4a72227ef4fc08957040acffdb5f122733b24fa12daff41b" [[package]] name = "matrixmultiply" @@ -5450,7 +5459,7 @@ checksum = "9e6a0fd4f737c707bd9086cc16c925f294943eb62eb71499e9fd4cf71f8b9f4e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -5790,7 +5799,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -5819,7 +5828,7 @@ checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -5954,9 +5963,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edc55135a600d700580e406b4de0d59cb9ad25e344a3a091a97ded2622ec4ec6" +checksum = "f32154ba0af3a075eefa1eda8bb414ee928f62303a54ea85b8d6638ff1a6ee9e" [[package]] name = "positioned-io" @@ -6456,7 +6465,7 @@ checksum = "6413f3de1edee53342e6138e75b56d32e7bc6e332b3bd62d497b1929d4cfbcdd" dependencies = [ "pem", "ring", - "time 0.3.23", + "time 0.3.24", "x509-parser 0.13.2", "yasna", ] @@ -6469,7 +6478,7 @@ checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" dependencies = [ "pem", "ring", - "time 0.3.23", + "time 0.3.24", "yasna", ] @@ -6523,7 +6532,7 @@ checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.3", + "regex-automata 0.3.4", "regex-syntax 0.7.4", ] @@ -6538,9 +6547,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310" +checksum = "b7b6d6190b7594385f61bd3911cd1be99dfddcfc365a4160cc2ab5bff4aed294" dependencies = [ "aho-corasick", "memchr", @@ -6842,9 +6851,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.1" +version = "0.101.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f36a6828982f422756984e47912a7a51dcbc2a197aa791158f8ca61cd8204e" +checksum = "513722fd73ad80a71f72b61009ea1b584bcfa1483ca93949c8f290298837fa59" dependencies = [ "ring", "untrusted", @@ -6989,9 +6998,9 @@ checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" [[package]] name = "serde" -version = "1.0.175" +version = "1.0.179" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d25439cd7397d044e2748a6fe2432b5e85db703d6d097bd014b3c0ad1ebff0b" +checksum = "0a5bf42b8d227d4abf38a1ddb08602e229108a517cd4e5bb28f9c7eaafdce5c0" dependencies = [ "serde_derive", ] @@ -7016,13 +7025,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.175" +version = "1.0.179" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b23f7ade6f110613c0d63858ddb8b94c1041f550eab58a16b371bdf2c9c80ab4" +checksum = "741e124f5485c7e60c03b043f79f320bff3527f4bbf12cf3831750dc46a0ec2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -7051,9 +7060,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b" +checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" dependencies = [ "itoa", "ryu", @@ -7072,13 +7081,13 @@ dependencies = [ [[package]] name = "serde_repr" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e168eaaf71e8f9bd6037feb05190485708e019f4fd87d161b3c0a0d37daf85e5" +checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -7136,7 +7145,7 @@ dependencies = [ "serde", "serde_json", "serde_with_macros", - "time 0.3.23", + "time 0.3.24", ] [[package]] @@ -7148,7 +7157,7 @@ dependencies = [ "darling 0.20.3", "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -7322,7 +7331,7 @@ dependencies = [ "num-bigint", "num-traits", "thiserror", - "time 0.3.23", + "time 0.3.24", ] [[package]] @@ -7369,7 +7378,7 @@ checksum = "0eb01866308440fc64d6c44d9e86c5cc17adfe33c4d6eed55da9145044d0ffc1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -7639,7 +7648,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -7689,9 +7698,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.27" +version = "2.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b60f673f44a8255b9c8c657daf66a596d435f2da81a555b06dc644d080ba45e0" +checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" dependencies = [ "proc-macro2", "quote", @@ -7794,7 +7803,7 @@ checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -7824,9 +7833,9 @@ checksum = "3f6821a2afe2700471d4572a25bcbfc091d6d596902a279ed41af139f350b76e" [[package]] name = "tikv-jemalloc-sys" -version = "0.5.3+5.3.0-patched" +version = "0.5.4+5.3.0-patched" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a678df20055b43e57ef8cddde41cdfda9a3c1a060b67f4c5836dfb1d78543ba8" +checksum = "9402443cb8fd499b6f327e40565234ff34dbda27460c5b47db0db77443dd85d1" dependencies = [ "cc", "libc", @@ -7834,9 +7843,9 @@ dependencies = [ [[package]] name = "tikv-jemallocator" -version = "0.5.0" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20612db8a13a6c06d57ec83953694185a367e16945f66565e8028d2c0bd76979" +checksum = "965fe0c26be5c56c94e38ba547249074803efd52adfb66de62107d95aab3eaca" dependencies = [ "libc", "tikv-jemalloc-sys", @@ -7855,10 +7864,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.23" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59e399c068f43a5d116fedaf73b203fa4f9c519f17e2b34f63221d3792f81446" +checksum = "b79eabcd964882a646b3584543ccabeae7869e9ac32a46f6f22b7a5bd405308b" dependencies = [ + "deranged", "itoa", "serde", "time-core", @@ -7873,9 +7883,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96ba15a897f3c86766b757e5ac7221554c6750054d74d5b28844fce5fb36a6c4" +checksum = "eb71511c991639bb078fd5bf97757e03914361c48100d52878b8e52b46fb92cd" dependencies = [ "time-core", ] @@ -7944,7 +7954,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -8145,7 +8155,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d48f71a791638519505cefafe162606f706c25592e4bde4d97600c0195312e" dependencies = [ "crossbeam-channel", - "time 0.3.23", + "time 0.3.24", "tracing-subscriber", ] @@ -8157,7 +8167,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -8594,7 +8604,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", "wasm-bindgen-shared", ] @@ -8628,7 +8638,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -8698,9 +8708,9 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.109.0" +version = "0.110.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf9564f29de2890ee34406af52d2a92dec6ef044c8ddfc5add5db8dcfd36e6c" +checksum = "1dfcdb72d96f01e6c85b6bf20102e7423bdbaad5c337301bab2bbf253d26413c" dependencies = [ "indexmap 2.0.0", "semver", @@ -8708,12 +8718,12 @@ dependencies = [ [[package]] name = "wasmprinter" -version = "0.2.61" +version = "0.2.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df06dc468a161167818d5333cc248f49b58218cc0b20eb036840ea4332cb1a4a" +checksum = "42cd12ed4d96a984e4b598a17457f1126d01640cc7461afbb319642111ff9e7f" dependencies = [ "anyhow", - "wasmparser 0.109.0", + "wasmparser 0.110.0", ] [[package]] @@ -8959,7 +8969,7 @@ dependencies = [ "sha2 0.10.7", "stun", "thiserror", - "time 0.3.23", + "time 0.3.24", "tokio", "turn", "url", @@ -9395,9 +9405,9 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winnow" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25b5872fa2e10bd067ae946f927e726d7d603eaeb6e02fa6a350e0722d2b8c11" +checksum = "8bd122eb777186e60c3fdf765a58ac76e41c582f1f535fbf3314434c6b58f3f7" dependencies = [ "memchr", ] @@ -9468,7 +9478,7 @@ dependencies = [ "ring", "rusticata-macros", "thiserror", - "time 0.3.23", + "time 0.3.24", ] [[package]] @@ -9486,7 +9496,7 @@ dependencies = [ "oid-registry 0.6.1", "rusticata-macros", "thiserror", - "time 0.3.23", + "time 0.3.24", ] [[package]] @@ -9528,7 +9538,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" dependencies = [ - "time 0.3.23", + "time 0.3.24", ] [[package]] @@ -9558,7 +9568,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1f080b1c3f8c..ec6c63f6dcb7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -192,8 +192,19 @@ tokio-test = "0.4.2" [build-dependencies] anyhow = "1" -protobuf-codegen = "3.2" -walkdir = "2.3" +async-compression = { version = "0.4", features = ["futures-io", "zstd"] } +async-fs = "1" +cid = { version = "0.10", default-features = false, features = ["std"] } +futures = "0.3" +fvm_ipld_car = "0.7" +itertools = "0.11" +once_cell = "1" +protobuf-codegen = "3" +reqwest = { version = "0.11", default-features = false, features = ["stream", "rustls-tls"] } +tempfile = "3" +tokio = { version = "1", features = ['full'] } +url = "2" +walkdir = "2" # This needs to be set as default. Otherwise, a regular build or test will produce # gargantuan artifacts (around 70G for all tests). For a debugging session, you can diff --git a/build.rs b/build.rs index b122c56958da..af2fd95764e7 100644 --- a/build.rs +++ b/build.rs @@ -1,26 +1,187 @@ // Copyright 2019-2023 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT -use std::path::PathBuf; +mod src; +use src::{ActorBundleInfo, ACTOR_BUNDLES}; +use std::{ + env, fs, io, + path::{Path, PathBuf}, + pin::pin, +}; + +use anyhow::Context; +use async_compression::futures::write::ZstdEncoder; +use cid::Cid; +use futures::{ + io::{BufReader, BufWriter}, + AsyncRead, AsyncWriteExt, Stream, StreamExt, TryStreamExt, +}; +use fvm_ipld_car::{CarHeader, CarReader}; +use itertools::Itertools; +use once_cell::sync::Lazy; use protobuf_codegen::Customize; +use reqwest::Url; +use tokio::task::JoinSet; use walkdir::WalkDir; const PROTO_DIR: &str = "proto"; const CARGO_OUT_DIR: &str = "proto"; -fn main() -> anyhow::Result<()> { +// Using a local path instead of `OUT_DIR` to reuse the cache as much as possible +static ACTOR_BUNDLE_CACHE_DIR: Lazy = + Lazy::new(|| Path::new("target/actor_bundles/").to_owned()); + +pub fn global_http_client() -> reqwest::Client { + static CLIENT: Lazy = Lazy::new(reqwest::Client::new); + CLIENT.clone() +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + generate_protobuf_code()?; + + generate_compressed_actor_bundles().await?; + + Ok(()) +} + +async fn generate_compressed_actor_bundles() -> anyhow::Result<()> { + println!("cargo:rerun-if-changed=src/mod.rs"); + println!( + "cargo:rerun-if-changed={}", + ACTOR_BUNDLE_CACHE_DIR.display() + ); + + let mut tasks = JoinSet::new(); + for ActorBundleInfo { manifest, url } in ACTOR_BUNDLES.iter() { + tasks.spawn(async move { + download_bundle_if_needed(manifest, url) + .await + .with_context(|| format!("Failed to get {manifest}.car from {url}")) + }); + } + + let mut car_roots = vec![]; + let mut car_readers = vec![]; + while let Some(path) = tasks.join_next().await { + let car_reader = fvm_ipld_car::CarReader::new(futures::io::BufReader::new( + async_fs::File::open(path??).await?, + )) + .await?; + car_roots.extend_from_slice(car_reader.header.roots.as_slice()); + car_readers.push(car_reader); + } + + let car_writer = CarHeader::from( + car_readers + .iter() + .flat_map(|it| it.header.roots.iter()) + .unique() + .cloned() + .collect::>(), + ); + + let mut zstd_encoder = ZstdEncoder::with_quality( + async_fs::File::create(Path::new(&env::var("OUT_DIR")?).join("actor_bundles.car.zst")) + .await?, + if cfg!(debug_assertions) { + async_compression::Level::Default + } else { + async_compression::Level::Precise(17) + }, + ); + + car_writer + .write_stream_async(&mut zstd_encoder, &mut pin!(merge_car_readers(car_readers))) + .await?; + + Ok(()) +} + +fn read_car_as_stream(reader: CarReader) -> impl Stream)> +where + R: AsyncRead + Send + Unpin, +{ + futures::stream::unfold(reader, move |mut reader| async { + reader + .next_block() + .await + .expect("Failed to call CarReader::next_block") + .map(|b| ((b.cid, b.data), reader)) + }) +} + +fn merge_car_readers(readers: Vec>) -> impl Stream)> +where + R: AsyncRead + Send + Unpin, +{ + futures::stream::iter(readers).flat_map(read_car_as_stream) +} + +async fn download_bundle_if_needed(root: &Cid, url: &Url) -> anyhow::Result { + if !ACTOR_BUNDLE_CACHE_DIR.is_dir() { + fs::create_dir_all(ACTOR_BUNDLE_CACHE_DIR.as_path())?; + } + let cached_car_path = ACTOR_BUNDLE_CACHE_DIR.join(format!("{root}.car")); + if cached_car_path.is_file() { + if let Ok(file) = async_fs::File::open(&cached_car_path).await { + if let Ok(true) = is_bundle_valid(root, BufReader::new(file)).await { + return Ok(cached_car_path); + } + } + } + + let tmp = tempfile::NamedTempFile::new_in(ACTOR_BUNDLE_CACHE_DIR.as_path())?.into_temp_path(); + { + let response = global_http_client().get(url.clone()).send().await?; + let mut writer = BufWriter::new(async_fs::File::create(&tmp).await?); + futures::io::copy( + response + .bytes_stream() + .map_err(|reqwest_error| io::Error::new(io::ErrorKind::Other, reqwest_error)) + .into_async_read(), + &mut writer, + ) + .await?; + writer.flush().await?; + } + if is_bundle_valid(root, BufReader::new(async_fs::File::open(&tmp).await?)).await? { + tmp.persist(&cached_car_path)?; + Ok(cached_car_path) + } else { + anyhow::bail!("Invalid bundle: {url}"); + } +} + +async fn is_bundle_valid(root: &Cid, reader: R) -> anyhow::Result +where + R: AsyncRead + Send + Unpin, +{ + is_bundle_car_valid(root, CarReader::new(reader).await?) +} + +fn is_bundle_car_valid(root: &Cid, car_reader: CarReader) -> anyhow::Result +where + R: AsyncRead + Send + Unpin, +{ + Ok(car_reader.header.roots.len() == 1 && &car_reader.header.roots[0] == root) +} + +fn generate_protobuf_code() -> anyhow::Result<()> { + println!("cargo:rerun-if-changed=proto"); + protobuf_codegen::Codegen::new() .pure() .cargo_out_dir(CARGO_OUT_DIR) - .inputs(get_inputs()?.as_slice()) + .inputs(get_proto_inputs()?.as_slice()) .include(PROTO_DIR) .customize(Customize::default().lite_runtime(true)) .run()?; Ok(()) } -fn get_inputs() -> anyhow::Result> { +fn get_proto_inputs() -> anyhow::Result> { let mut inputs = Vec::new(); for entry in WalkDir::new(PROTO_DIR).into_iter().flatten() { let path = entry.path(); diff --git a/documentation/src/developer_documentation/state_migration_guide.md b/documentation/src/developer_documentation/state_migration_guide.md index 0b243f03da3a..c5c93358134a 100644 --- a/documentation/src/developer_documentation/state_migration_guide.md +++ b/documentation/src/developer_documentation/state_migration_guide.md @@ -24,18 +24,25 @@ NV19. The first step is to import the actor bundle into Forest. This is done by: -- adding the bundle to the `HeightInfos` struct in the network definitions files - (e.g., +- adding the bundle cid to the `HeightInfos` struct in the network definitions + files (e.g., [calibnet](https://github.com/ChainSafe/forest/blob/main/src/networks/calibnet/mod.rs)). ```rust HeightInfo { height: Height::Hygge, epoch: 322_354, - bundle: Some(ActorBundleInfo { - manifest: Cid::try_from("bafy2bzaced25ta3j6ygs34roprilbtb3f6mxifyfnm7z7ndquaruxzdq3y7lo").unwrap(), - url: Url::parse("https://github.com/filecoin-project/builtin-actors/releases/download/v10.0.0-rc.1/builtin-actors-calibrationnet.car").unwrap() -}) + bundle: Some(Cid::try_from("bafy2bzaced25ta3j6ygs34roprilbtb3f6mxifyfnm7z7ndquaruxzdq3y7lo").unwrap()), +} +``` + +- adding the bundle cid and url to the `ACTOR_BUNDLES` in the `src/mod.rs`. + +```rust +ActorBundleInfo{ + manifest: Cid::try_from("bafy2bzacedbedgynklc4dgpyxippkxmba2mgtw7ecntoneclsvvl4klqwuyyy").unwrap(), + url: Url::parse("https://forest-continuous-integration.fra1.cdn.digitaloceanspaces.com/builtin-actors/calibnet/Shark.car").unwrap(), +}, ``` ### Implement the migration diff --git a/src/cli/subcommands/snapshot_cmd.rs b/src/cli/subcommands/snapshot_cmd.rs index 7771167acc59..a859da8a1d60 100644 --- a/src/cli/subcommands/snapshot_cmd.rs +++ b/src/cli/subcommands/snapshot_cmd.rs @@ -6,7 +6,7 @@ use crate::blocks::{tipset_keys_json::TipsetKeysJson, Tipset, TipsetKeys}; use crate::chain::index::ChainIndex; use crate::cli::subcommands::{cli_error_and_die, handle_rpc_err}; use crate::cli_shared::snapshot::{self, TrustedVendor}; -use crate::daemon::bundle::load_bundles; +use crate::daemon::bundle::load_actor_bundles; use crate::db::car::AnyCar; use crate::fil_cns::composition as cns; use crate::ipld::{recurse_links_hash, CidHashSet}; @@ -443,17 +443,8 @@ where let last_epoch = ts.epoch() - epochs as i64; - // Bundles are required when doing state migrations. Download any bundles - // that may be necessary after `last_epoch`. - load_bundles( - last_epoch, - &Config { - chain: chain_config.clone(), - ..Default::default() - }, - &db, - ) - .await?; + // Bundles are required when doing state migrations. + load_actor_bundles(&db).await?; // Set proof parameter data dir and make sure the proofs are available if cns::FETCH_PARAMS { diff --git a/src/daemon/bundle.rs b/src/daemon/bundle.rs index 680e0cc97f0a..28129c6c55e9 100644 --- a/src/daemon/bundle.rs +++ b/src/daemon/bundle.rs @@ -1,84 +1,33 @@ // Copyright 2019-2023 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT -use crate::cli_shared::cli::Config; -use crate::networks::Height; -use crate::shim::clock::ChainEpoch; +use async_compression::futures::bufread::ZstdDecoder; +use cid::Cid; use fvm_ipld_blockstore::Blockstore; -use tokio::{fs::File, io::BufWriter}; -use tracing::info; -pub async fn load_bundles( - epoch: ChainEpoch, - config: &Config, - db: &impl Blockstore, -) -> anyhow::Result<()> { - // collect bundles to load into the database. - let mut bundles = Vec::new(); - for info in &config.chain.height_infos { - if epoch < config.chain.epoch(info.height) { - if let Some(bundle) = &info.bundle { - bundles.push(( - bundle.manifest, - get_actors_bundle(config, info.height).await?, - )); - } - } - } +pub async fn load_actor_bundles(db: &impl Blockstore) -> anyhow::Result> { + pub const ACTOR_BUNDLES_CAR_ZST: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/actor_bundles.car.zst")); - for (manifest_cid, reader) in bundles { - let roots = fvm_ipld_car::load_car(db, reader).await?; - assert_eq!( - roots.len(), - 1, - "expected one root when loading actors bundle" - ); - info!("Loaded actors bundle with CID: {}", roots[0]); - anyhow::ensure!( - manifest_cid == roots[0], - "manifest cid in config '{manifest_cid}' does not match manifest cid from bundle '{}'", - roots[0] - ); - } - Ok(()) + Ok(fvm_ipld_car::load_car( + db, + ZstdDecoder::new(futures::io::BufReader::new(ACTOR_BUNDLES_CAR_ZST)), + ) + .await?) } -/// Downloads the actors bundle (if not already downloaded) and returns a reader -/// to it. -// TODO Get it from IPFS instead of GitHub. -pub async fn get_actors_bundle( - config: &Config, - height: Height, -) -> anyhow::Result> { - let bundle_info = config.chain.height_infos[height as usize] - .bundle - .as_ref() - .ok_or_else(|| anyhow::anyhow!("no bundle for epoch {}", config.chain.epoch(height)))?; - - // This is the path where the actors bundle will be stored. - let bundle_path_dir = config - .client - .data_dir - .join("bundles") - .join(config.chain.network.to_string()); - - tokio::fs::create_dir_all(&bundle_path_dir).await?; - let bundle_path = bundle_path_dir.join(format!("bundle_{height}.car")); - - // If the bundle already exists, return a reader to it. - if bundle_path.exists() { - let file = async_fs::File::open(bundle_path).await?; - return Ok(futures::io::BufReader::new(file)); +#[cfg(test)] +mod tests { + use super::*; + use crate::build::ACTOR_BUNDLES; + use ahash::HashSet; + use pretty_assertions::assert_eq; + + #[tokio::test] + async fn test_load_actor_bundles() { + let db = fvm_ipld_blockstore::MemoryBlockstore::new(); + let roots = HashSet::from_iter(load_actor_bundles(&db).await.unwrap()); + let roots_expected: HashSet = ACTOR_BUNDLES.iter().map(|b| b.manifest).collect(); + assert_eq!(roots, roots_expected); } - - // Otherwise, download it. - info!("Downloading actors bundle..."); - let mut reader = crate::utils::net::reader(bundle_info.url.as_str()).await?; - - let file = File::create(&bundle_path).await?; - let mut writer = BufWriter::new(file); - tokio::io::copy(&mut reader, &mut writer).await?; - - let file = async_fs::File::open(bundle_path).await?; - Ok(futures::io::BufReader::new(file)) } diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index 18ccb4b6d56f..0cb8bd25bc3a 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -36,7 +36,7 @@ use crate::utils::{ version::FOREST_VERSION_STRING, RetryArgs, }; use anyhow::{bail, Context}; -use bundle::load_bundles; +use bundle::load_actor_bundles; use dialoguer::{console::Term, theme::ColorfulTheme}; use futures::{select, Future, FutureExt}; use lazy_static::lazy_static; @@ -261,7 +261,7 @@ pub(super) async fn start( let epoch = chain_store.heaviest_tipset().epoch(); - load_bundles(epoch, &config, &db).await?; + load_actor_bundles(&db).await?; let peer_manager = Arc::new(PeerManager::default()); services.spawn(peer_manager.clone().peer_operation_event_loop_task()); diff --git a/src/lib.rs b/src/lib.rs index 0ad41c729ffd..1c6c48c8a200 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -36,6 +36,7 @@ mod libp2p_bitswap; mod message; mod message_pool; mod metrics; +mod r#mod; mod networks; mod rpc; mod rpc_api; @@ -48,6 +49,10 @@ mod statediff; mod test_utils; mod utils; +pub mod build { + pub use super::r#mod::*; +} + /// These items are semver-exempt, and exist for forest author use only // We want to have doctests, but don't want our internals to be public because: // - We don't want to be concerned with library compat diff --git a/src/mod.rs b/src/mod.rs new file mode 100644 index 000000000000..460e0accd451 --- /dev/null +++ b/src/mod.rs @@ -0,0 +1,54 @@ +// Copyright 2019-2023 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +//! This module contains code that is shared between the library and the build script + +use cid::Cid; +use once_cell::sync::Lazy; +use reqwest::Url; + +#[derive(Debug)] +pub struct ActorBundleInfo { + pub manifest: Cid, + pub url: Url, +} + +pub static ACTOR_BUNDLES: Lazy<[ActorBundleInfo; 8]> = Lazy::new(|| { + [ + // calibnet + ActorBundleInfo{ + manifest: Cid::try_from("bafy2bzacedbedgynklc4dgpyxippkxmba2mgtw7ecntoneclsvvl4klqwuyyy").unwrap(), + url: Url::parse("https://forest-continuous-integration.fra1.cdn.digitaloceanspaces.com/builtin-actors/calibnet/Shark.car").unwrap(), + }, + ActorBundleInfo{ + manifest: Cid::try_from("bafy2bzaced25ta3j6ygs34roprilbtb3f6mxifyfnm7z7ndquaruxzdq3y7lo").unwrap(), + url: Url::parse("https://forest-continuous-integration.fra1.cdn.digitaloceanspaces.com/builtin-actors/calibnet/Hygge.car").unwrap(), + }, + ActorBundleInfo{ + manifest: Cid::try_from("bafy2bzacedhuowetjy2h4cxnijz2l64h4mzpk5m256oywp4evarpono3cjhco").unwrap(), + url: Url::parse("https://forest-continuous-integration.fra1.cdn.digitaloceanspaces.com/builtin-actors/calibnet/Lightning.car").unwrap(), + }, + // devnet + ActorBundleInfo{ + manifest: Cid::try_from("bafy2bzacebzz376j5kizfck56366kdz5aut6ktqrvqbi3efa2d4l2o2m653ts").unwrap(), + url: Url::parse("https://forest-continuous-integration.fra1.cdn.digitaloceanspaces.com/builtin-actors/devnet/Hygge.car").unwrap(), + }, + ActorBundleInfo{ + manifest: Cid::try_from("bafy2bzaceay35go4xbjb45km6o46e5bib3bi46panhovcbedrynzwmm3drr4i").unwrap(), + url: Url::parse("https://forest-continuous-integration.fra1.cdn.digitaloceanspaces.com/builtin-actors/devnet/Lightning.car").unwrap(), + }, + // mainnet + ActorBundleInfo{ + manifest: Cid::try_from("bafy2bzaceb6j6666h36xnhksu3ww4kxb6e25niayfgkdnifaqi6m6ooc66i6i").unwrap(), + url: Url::parse("https://forest-continuous-integration.fra1.cdn.digitaloceanspaces.com/builtin-actors/mainnet/Shark.car").unwrap(), + }, + ActorBundleInfo{ + manifest: Cid::try_from("bafy2bzacecsuyf7mmvrhkx2evng5gnz5canlnz2fdlzu2lvcgptiq2pzuovos").unwrap(), + url: Url::parse("https://forest-continuous-integration.fra1.cdn.digitaloceanspaces.com/builtin-actors/mainnet/Hygge.car").unwrap(), + }, + ActorBundleInfo{ + manifest: Cid::try_from("bafy2bzacecnhaiwcrpyjvzl4uv4q3jzoif26okl3m66q3cijp3dfwlcxwztwo").unwrap(), + url: Url::parse("https://forest-continuous-integration.fra1.cdn.digitaloceanspaces.com/builtin-actors/mainnet/Lightning.car").unwrap(), + }, + ] +}); diff --git a/src/networks/calibnet/mod.rs b/src/networks/calibnet/mod.rs index 66c2ce394ea5..f2ac6bf82259 100644 --- a/src/networks/calibnet/mod.rs +++ b/src/networks/calibnet/mod.rs @@ -2,14 +2,11 @@ // SPDX-License-Identifier: Apache-2.0, MIT use cid::Cid; -use lazy_static::lazy_static; use libp2p::Multiaddr; use once_cell::sync::Lazy; use std::str::FromStr; -use url::Url; use super::{drand::DRAND_MAINNET, parse_bootstrap_peers, DrandPoint, Height, HeightInfo}; -use crate::networks::ActorBundleInfo; /// Default genesis car file bytes. pub const DEFAULT_GENESIS: &[u8] = include_bytes!("genesis.car"); @@ -32,125 +29,125 @@ const LIGHTNING_ROLLOVER_PERIOD: i64 = 3120; // https://github.com/ethereum-lists/chains/blob/4731f6713c6fc2bf2ae727388642954a6545b3a9/_data/chains/eip155-314159.json pub const ETH_CHAIN_ID: u64 = 314159; -lazy_static! { /// Height epochs. -pub static ref HEIGHT_INFOS: [HeightInfo; 21] = [ - HeightInfo { - height: Height::Breeze, - epoch: -1, - bundle: None, - }, - HeightInfo { - height: Height::Smoke, - epoch: -2, - bundle: None, - }, - HeightInfo { - height: Height::Ignition, - epoch: -3, - bundle: None, - }, - HeightInfo { - height: Height::ActorsV2, - epoch: 30, - bundle: None, - }, - HeightInfo { - height: Height::Tape, - epoch: 60, - bundle: None, - }, - HeightInfo { - height: Height::Liftoff, - epoch: -5, - bundle: None, - }, - HeightInfo { - height: Height::Kumquat, - epoch: 90, - bundle: None, - }, - HeightInfo { - height: Height::Calico, - epoch: 120, - bundle: None, - }, - HeightInfo { - height: Height::Persian, - epoch: 130, - bundle: None, - }, - HeightInfo { - height: Height::Orange, - epoch: 300, - bundle: None, - }, - HeightInfo { - height: Height::Trust, - epoch: 330, - bundle: None, - }, - HeightInfo { - height: Height::Norwegian, - epoch: 360, - bundle: None, - }, - HeightInfo { - height: Height::Turbo, - epoch: 390, - bundle: None, - }, - HeightInfo { - height: Height::Hyperdrive, - epoch: 420, - bundle: None, - }, - HeightInfo { - height: Height::Chocolate, - epoch: 450, - bundle: None, - }, - HeightInfo { - height: Height::OhSnap, - epoch: 480, - bundle: None, - }, - HeightInfo { - height: Height::Skyr, - epoch: 510, - bundle: None, - }, - HeightInfo { - height: Height::Shark, - epoch: 16_800, - bundle: Some(ActorBundleInfo { - manifest: Cid::try_from("bafy2bzacedbedgynklc4dgpyxippkxmba2mgtw7ecntoneclsvvl4klqwuyyy").unwrap(), - url: Url::parse("https://github.com/filecoin-project/builtin-actors/releases/download/v9.0.3/builtin-actors-calibrationnet.car").unwrap() - }) - }, - HeightInfo { - height: Height::Hygge, - epoch: 322_354, - bundle: Some(ActorBundleInfo { - manifest: Cid::try_from("bafy2bzaced25ta3j6ygs34roprilbtb3f6mxifyfnm7z7ndquaruxzdq3y7lo").unwrap(), - url: Url::parse("https://github.com/filecoin-project/builtin-actors/releases/download/v10.0.0-rc.1/builtin-actors-calibrationnet.car").unwrap() - }) - }, - HeightInfo { - height: Height::Lightning, - epoch: LIGHTNING_EPOCH, - bundle: Some(ActorBundleInfo { - manifest: Cid::try_from("bafy2bzacedhuowetjy2h4cxnijz2l64h4mzpk5m256oywp4evarpono3cjhco").unwrap(), - url: Url::parse("https://github.com/filecoin-project/builtin-actors/releases/download/v11.0.0-rc2/builtin-actors-calibrationnet.car").unwrap() - }), - }, - HeightInfo { - height: Height::Thunder, - epoch: LIGHTNING_EPOCH + LIGHTNING_ROLLOVER_PERIOD, - bundle: None, - }, -]; -} +pub static HEIGHT_INFOS: Lazy<[HeightInfo; 21]> = Lazy::new(|| { + [ + HeightInfo { + height: Height::Breeze, + epoch: -1, + bundle: None, + }, + HeightInfo { + height: Height::Smoke, + epoch: -2, + bundle: None, + }, + HeightInfo { + height: Height::Ignition, + epoch: -3, + bundle: None, + }, + HeightInfo { + height: Height::ActorsV2, + epoch: 30, + bundle: None, + }, + HeightInfo { + height: Height::Tape, + epoch: 60, + bundle: None, + }, + HeightInfo { + height: Height::Liftoff, + epoch: -5, + bundle: None, + }, + HeightInfo { + height: Height::Kumquat, + epoch: 90, + bundle: None, + }, + HeightInfo { + height: Height::Calico, + epoch: 120, + bundle: None, + }, + HeightInfo { + height: Height::Persian, + epoch: 130, + bundle: None, + }, + HeightInfo { + height: Height::Orange, + epoch: 300, + bundle: None, + }, + HeightInfo { + height: Height::Trust, + epoch: 330, + bundle: None, + }, + HeightInfo { + height: Height::Norwegian, + epoch: 360, + bundle: None, + }, + HeightInfo { + height: Height::Turbo, + epoch: 390, + bundle: None, + }, + HeightInfo { + height: Height::Hyperdrive, + epoch: 420, + bundle: None, + }, + HeightInfo { + height: Height::Chocolate, + epoch: 450, + bundle: None, + }, + HeightInfo { + height: Height::OhSnap, + epoch: 480, + bundle: None, + }, + HeightInfo { + height: Height::Skyr, + epoch: 510, + bundle: None, + }, + HeightInfo { + height: Height::Shark, + epoch: 16_800, + bundle: Some( + Cid::try_from("bafy2bzacedbedgynklc4dgpyxippkxmba2mgtw7ecntoneclsvvl4klqwuyyy") + .unwrap(), + ), + }, + HeightInfo { + height: Height::Hygge, + epoch: 322_354, + bundle: Some( + Cid::try_from("bafy2bzaced25ta3j6ygs34roprilbtb3f6mxifyfnm7z7ndquaruxzdq3y7lo") + .unwrap(), + ), + }, + HeightInfo { + height: Height::Lightning, + epoch: LIGHTNING_EPOCH, + bundle: Some( + Cid::try_from("bafy2bzacedhuowetjy2h4cxnijz2l64h4mzpk5m256oywp4evarpono3cjhco") + .unwrap(), + ), + }, + HeightInfo { + height: Height::Thunder, + epoch: LIGHTNING_EPOCH + LIGHTNING_ROLLOVER_PERIOD, + bundle: None, + }, + ] +}); pub(super) static DRAND_SCHEDULE: [DrandPoint<'static>; 1] = [DrandPoint { height: 0, diff --git a/src/networks/devnet/mod.rs b/src/networks/devnet/mod.rs index ae7ecceb206b..e48dcb2f60a3 100644 --- a/src/networks/devnet/mod.rs +++ b/src/networks/devnet/mod.rs @@ -2,131 +2,129 @@ // SPDX-License-Identifier: Apache-2.0, MIT use cid::Cid; -use lazy_static::lazy_static; -use url::Url; +use once_cell::sync::Lazy; use super::{drand::DRAND_MAINNET, DrandPoint, Height, HeightInfo}; -use crate::networks::ActorBundleInfo; // https://github.com/ethereum-lists/chains/blob/6b1e3ccad1cfcaae5aa1ab917960258f0ef1a6b6/_data/chains/eip155-31415926.json pub const ETH_CHAIN_ID: u64 = 31415926; -lazy_static! { /// Height epochs. -pub static ref HEIGHT_INFOS: [HeightInfo; 21] = [ - HeightInfo { - height: Height::Breeze, - epoch: -50, - bundle: None, - }, - HeightInfo { - height: Height::Smoke, - epoch: -2, - bundle: None, - }, - HeightInfo { - height: Height::Ignition, - epoch: -3, - bundle: None, - }, - HeightInfo { - height: Height::ActorsV2, - epoch: -3, - bundle: None, - }, - HeightInfo { - height: Height::Tape, - epoch: -4, - bundle: None, - }, - HeightInfo { - height: Height::Liftoff, - epoch: -6, - bundle: None, - }, - HeightInfo { - height: Height::Kumquat, - epoch: -7, - bundle: None, - }, - HeightInfo { - height: Height::Calico, - epoch: -9, - bundle: None, - }, - HeightInfo { - height: Height::Persian, - epoch: -10, - bundle: None, - }, - HeightInfo { - height: Height::Orange, - epoch: -11, - bundle: None, - }, - HeightInfo { - height: Height::Trust, - epoch: -13, - bundle: None, - }, - HeightInfo { - height: Height::Norwegian, - epoch: -14, - bundle: None, - }, - HeightInfo { - height: Height::Turbo, - epoch: -15, - bundle: None, - }, - HeightInfo { - height: Height::Hyperdrive, - epoch: -16, - bundle: None, - }, - HeightInfo { - height: Height::Chocolate, - epoch: -17, - bundle: None, - }, - HeightInfo { - height: Height::OhSnap, - epoch: -18, - bundle: None, - }, - HeightInfo { - height: Height::Skyr, - epoch: -19, - bundle: None, - }, - HeightInfo { - height: Height::Shark, - epoch: -20, - bundle: None, - }, - HeightInfo { - height: Height::Hygge, - epoch: -1, - bundle: Some(ActorBundleInfo { - manifest: Cid::try_from("bafy2bzacebzz376j5kizfck56366kdz5aut6ktqrvqbi3efa2d4l2o2m653ts").unwrap(), - url: Url::parse("https://github.com/filecoin-project/builtin-actors/releases/download/v10.0.0/builtin-actors-devnet.car").unwrap() - }), - }, - HeightInfo { - height: Height::Lightning, - epoch: 30, - bundle: Some(ActorBundleInfo { - manifest: Cid::try_from("bafy2bzaceay35go4xbjb45km6o46e5bib3bi46panhovcbedrynzwmm3drr4i").unwrap(), - url: Url::parse("https://github.com/filecoin-project/builtin-actors/releases/download/v11.0.0/builtin-actors-devnet.car").unwrap() - }), - }, - HeightInfo { - height: Height::Thunder, - epoch: 1000, - bundle: None, - }, -]; -} +pub static HEIGHT_INFOS: Lazy<[HeightInfo; 21]> = Lazy::new(|| { + [ + HeightInfo { + height: Height::Breeze, + epoch: -50, + bundle: None, + }, + HeightInfo { + height: Height::Smoke, + epoch: -2, + bundle: None, + }, + HeightInfo { + height: Height::Ignition, + epoch: -3, + bundle: None, + }, + HeightInfo { + height: Height::ActorsV2, + epoch: -3, + bundle: None, + }, + HeightInfo { + height: Height::Tape, + epoch: -4, + bundle: None, + }, + HeightInfo { + height: Height::Liftoff, + epoch: -6, + bundle: None, + }, + HeightInfo { + height: Height::Kumquat, + epoch: -7, + bundle: None, + }, + HeightInfo { + height: Height::Calico, + epoch: -9, + bundle: None, + }, + HeightInfo { + height: Height::Persian, + epoch: -10, + bundle: None, + }, + HeightInfo { + height: Height::Orange, + epoch: -11, + bundle: None, + }, + HeightInfo { + height: Height::Trust, + epoch: -13, + bundle: None, + }, + HeightInfo { + height: Height::Norwegian, + epoch: -14, + bundle: None, + }, + HeightInfo { + height: Height::Turbo, + epoch: -15, + bundle: None, + }, + HeightInfo { + height: Height::Hyperdrive, + epoch: -16, + bundle: None, + }, + HeightInfo { + height: Height::Chocolate, + epoch: -17, + bundle: None, + }, + HeightInfo { + height: Height::OhSnap, + epoch: -18, + bundle: None, + }, + HeightInfo { + height: Height::Skyr, + epoch: -19, + bundle: None, + }, + HeightInfo { + height: Height::Shark, + epoch: -20, + bundle: None, + }, + HeightInfo { + height: Height::Hygge, + epoch: -1, + bundle: Some( + Cid::try_from("bafy2bzacebzz376j5kizfck56366kdz5aut6ktqrvqbi3efa2d4l2o2m653ts") + .unwrap(), + ), + }, + HeightInfo { + height: Height::Lightning, + epoch: 30, + bundle: Some( + Cid::try_from("bafy2bzaceay35go4xbjb45km6o46e5bib3bi46panhovcbedrynzwmm3drr4i") + .unwrap(), + ), + }, + HeightInfo { + height: Height::Thunder, + epoch: 1000, + bundle: None, + }, + ] +}); pub(super) static DRAND_SCHEDULE: [DrandPoint<'static>; 1] = [DrandPoint { height: 0, diff --git a/src/networks/mainnet/mod.rs b/src/networks/mainnet/mod.rs index 4219698e80dd..7af51451f6f3 100644 --- a/src/networks/mainnet/mod.rs +++ b/src/networks/mainnet/mod.rs @@ -3,17 +3,14 @@ use crate::shim::clock::ChainEpoch; use cid::Cid; -use lazy_static::lazy_static; use libp2p::Multiaddr; use once_cell::sync::Lazy; use std::str::FromStr; -use url::Url; use super::{ drand::{DRAND_INCENTINET, DRAND_MAINNET}, parse_bootstrap_peers, DrandPoint, Height, HeightInfo, }; -use crate::networks::ActorBundleInfo; const SMOKE_HEIGHT: ChainEpoch = 51000; @@ -35,125 +32,125 @@ const LIGHTNING_ROLLOVER_PERIOD: i64 = 2880 * 21; // https://github.com/ethereum-lists/chains/blob/4731f6713c6fc2bf2ae727388642954a6545b3a9/_data/chains/eip155-314.json pub const ETH_CHAIN_ID: u64 = 314; -lazy_static! { /// Height epochs. -pub static ref HEIGHT_INFOS: [HeightInfo; 21] = [ - HeightInfo { - height: Height::Breeze, - epoch: 41_280, - bundle: None, - }, - HeightInfo { - height: Height::Smoke, - epoch: SMOKE_HEIGHT, - bundle: None, - }, - HeightInfo { - height: Height::Ignition, - epoch: 94_000, - bundle: None, - }, - HeightInfo { - height: Height::ActorsV2, - epoch: 138_720, - bundle: None, - }, - HeightInfo { - height: Height::Tape, - epoch: 140_760, - bundle: None, - }, - HeightInfo { - height: Height::Liftoff, - epoch: 148_888, - bundle: None, - }, - HeightInfo { - height: Height::Kumquat, - epoch: 170_000, - bundle: None, - }, - HeightInfo { - height: Height::Calico, - epoch: 265_200, - bundle: None, - }, - HeightInfo { - height: Height::Persian, - epoch: 272_400, - bundle: None, - }, - HeightInfo { - height: Height::Orange, - epoch: 336_458, - bundle: None, - }, - HeightInfo { - height: Height::Trust, - epoch: 550_321, - bundle: None, - }, - HeightInfo { - height: Height::Norwegian, - epoch: 665_280, - bundle: None, - }, - HeightInfo { - height: Height::Turbo, - epoch: 712_320, - bundle: None, - }, - HeightInfo { - height: Height::Hyperdrive, - epoch: 892_800, - bundle: None, - }, - HeightInfo { - height: Height::Chocolate, - epoch: 1_231_620, - bundle: None, - }, - HeightInfo { - height: Height::OhSnap, - epoch: 1_594_680, - bundle: None, - }, - HeightInfo { - height: Height::Skyr, - epoch: 1_960_320, - bundle: None, - }, - HeightInfo { - height: Height::Shark, - epoch: 2_383_680, - bundle: Some(ActorBundleInfo { - manifest: Cid::try_from("bafy2bzaceb6j6666h36xnhksu3ww4kxb6e25niayfgkdnifaqi6m6ooc66i6i").unwrap(), - url: Url::parse("https://github.com/filecoin-project/builtin-actors/releases/download/v9.0.3/builtin-actors-mainnet.car").unwrap() - }) - }, - HeightInfo { - height: Height::Hygge, - epoch: 2_683_348, - bundle: Some(ActorBundleInfo { - manifest: Cid::try_from("bafy2bzacecsuyf7mmvrhkx2evng5gnz5canlnz2fdlzu2lvcgptiq2pzuovos").unwrap(), - url: Url::parse("https://github.com/filecoin-project/builtin-actors/releases/download/v10.0.0/builtin-actors-mainnet.car").unwrap() - }), - }, - HeightInfo { - height: Height::Lightning, - epoch: 2_809_800, - bundle: Some(ActorBundleInfo { - manifest: Cid::try_from("bafy2bzacecnhaiwcrpyjvzl4uv4q3jzoif26okl3m66q3cijp3dfwlcxwztwo").unwrap(), - url: Url::parse("https://github.com/filecoin-project/builtin-actors/releases/download/v11.0.0/builtin-actors-mainnet.car").unwrap() - }), - }, - HeightInfo { - height: Height::Thunder, - epoch: 2_809_800 + LIGHTNING_ROLLOVER_PERIOD, - bundle: None, - }, -]; -} +pub static HEIGHT_INFOS: Lazy<[HeightInfo; 21]> = Lazy::new(|| { + [ + HeightInfo { + height: Height::Breeze, + epoch: 41_280, + bundle: None, + }, + HeightInfo { + height: Height::Smoke, + epoch: SMOKE_HEIGHT, + bundle: None, + }, + HeightInfo { + height: Height::Ignition, + epoch: 94_000, + bundle: None, + }, + HeightInfo { + height: Height::ActorsV2, + epoch: 138_720, + bundle: None, + }, + HeightInfo { + height: Height::Tape, + epoch: 140_760, + bundle: None, + }, + HeightInfo { + height: Height::Liftoff, + epoch: 148_888, + bundle: None, + }, + HeightInfo { + height: Height::Kumquat, + epoch: 170_000, + bundle: None, + }, + HeightInfo { + height: Height::Calico, + epoch: 265_200, + bundle: None, + }, + HeightInfo { + height: Height::Persian, + epoch: 272_400, + bundle: None, + }, + HeightInfo { + height: Height::Orange, + epoch: 336_458, + bundle: None, + }, + HeightInfo { + height: Height::Trust, + epoch: 550_321, + bundle: None, + }, + HeightInfo { + height: Height::Norwegian, + epoch: 665_280, + bundle: None, + }, + HeightInfo { + height: Height::Turbo, + epoch: 712_320, + bundle: None, + }, + HeightInfo { + height: Height::Hyperdrive, + epoch: 892_800, + bundle: None, + }, + HeightInfo { + height: Height::Chocolate, + epoch: 1_231_620, + bundle: None, + }, + HeightInfo { + height: Height::OhSnap, + epoch: 1_594_680, + bundle: None, + }, + HeightInfo { + height: Height::Skyr, + epoch: 1_960_320, + bundle: None, + }, + HeightInfo { + height: Height::Shark, + epoch: 2_383_680, + bundle: Some( + Cid::try_from("bafy2bzaceb6j6666h36xnhksu3ww4kxb6e25niayfgkdnifaqi6m6ooc66i6i") + .unwrap(), + ), + }, + HeightInfo { + height: Height::Hygge, + epoch: 2_683_348, + bundle: Some( + Cid::try_from("bafy2bzacecsuyf7mmvrhkx2evng5gnz5canlnz2fdlzu2lvcgptiq2pzuovos") + .unwrap(), + ), + }, + HeightInfo { + height: Height::Lightning, + epoch: 2_809_800, + bundle: Some( + Cid::try_from("bafy2bzacecnhaiwcrpyjvzl4uv4q3jzoif26okl3m66q3cijp3dfwlcxwztwo") + .unwrap(), + ), + }, + HeightInfo { + height: Height::Thunder, + epoch: 2_809_800 + LIGHTNING_ROLLOVER_PERIOD, + bundle: None, + }, + ] +}); pub(super) static DRAND_SCHEDULE: [DrandPoint<'static>; 2] = [ DrandPoint { diff --git a/src/networks/mod.rs b/src/networks/mod.rs index 2006a935a434..7e8d11435a31 100644 --- a/src/networks/mod.rs +++ b/src/networks/mod.rs @@ -13,7 +13,6 @@ use fil_actors_shared::v10::runtime::Policy; use libp2p::Multiaddr; use serde::{Deserialize, Serialize}; use strum_macros::Display; -use url::Url; mod drand; @@ -128,17 +127,11 @@ impl From for NetworkVersion { } } -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub struct ActorBundleInfo { - pub manifest: Cid, - pub url: Url, -} - #[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] pub struct HeightInfo { pub height: Height, pub epoch: ChainEpoch, - pub bundle: Option, + pub bundle: Option, } pub fn sort_by_epoch(height_info_slice: &[HeightInfo]) -> Vec { diff --git a/src/state_migration/nv17/migration.rs b/src/state_migration/nv17/migration.rs index 88ba3fe372ec..607eb9db6d58 100644 --- a/src/state_migration/nv17/migration.rs +++ b/src/state_migration/nv17/migration.rs @@ -146,17 +146,16 @@ where .ok_or_else(|| anyhow!("no height info for network version NV17"))? .bundle .as_ref() - .ok_or_else(|| anyhow!("no bundle info for network version NV17"))? - .manifest; + .ok_or_else(|| anyhow!("no bundle info for network version NV17"))?; - blockstore.get(&new_manifest_cid)?.ok_or_else(|| { + blockstore.get(new_manifest_cid)?.ok_or_else(|| { anyhow!( "manifest for network version NV17 not found in blockstore: {}", new_manifest_cid ) })?; - let new_manifest = Manifest::load(&blockstore, &new_manifest_cid)?; + let new_manifest = Manifest::load(&blockstore, new_manifest_cid)?; let mut actors_in = StateTree::new_from_root(blockstore.clone(), state)?; diff --git a/src/state_migration/nv17/miner.rs b/src/state_migration/nv17/miner.rs index ba93230c1456..fcf840d7f205 100644 --- a/src/state_migration/nv17/miner.rs +++ b/src/state_migration/nv17/miner.rs @@ -589,7 +589,7 @@ mod tests { let mut chain_config = ChainConfig::calibnet(); if let Some(bundle) = &mut chain_config.height_infos[Height::Shark as usize].bundle { - bundle.manifest = new_manifest_cid; + *bundle = new_manifest_cid; } let new_state_cid = super::super::run_migration(&chain_config, &store, &tree_root, 200)?; let actors_out_state_root: StateRoot = store.get_cbor(&new_state_cid)?.unwrap(); @@ -622,7 +622,7 @@ mod tests { let (new_manifest_cid, _new_manifest) = make_test_manifest(&store, "fil/9/")?; let mut chain_config = ChainConfig::calibnet(); if let Some(bundle) = &mut chain_config.height_infos[Height::Shark as usize].bundle { - bundle.manifest = new_manifest_cid; + *bundle = new_manifest_cid; } let new_state_cid = super::super::run_migration(&chain_config, &store, &state_tree_old_root, 200)?; diff --git a/src/state_migration/nv18/migration.rs b/src/state_migration/nv18/migration.rs index 475488bee785..bc849811767a 100644 --- a/src/state_migration/nv18/migration.rs +++ b/src/state_migration/nv18/migration.rs @@ -80,10 +80,9 @@ where .ok_or_else(|| anyhow!("no height info for network version NV18"))? .bundle .as_ref() - .ok_or_else(|| anyhow!("no bundle info for network version NV18"))? - .manifest; + .ok_or_else(|| anyhow!("no bundle info for network version NV18"))?; - blockstore.get(&new_manifest_cid)?.ok_or_else(|| { + blockstore.get(new_manifest_cid)?.ok_or_else(|| { anyhow!( "manifest for network version NV18 not found in blockstore: {}", new_manifest_cid @@ -94,7 +93,7 @@ where let verifier = Arc::new(Verifier::default()); let mut migration = StateMigration::::new(Some(verifier)); - migration.add_nv18_migrations(blockstore.clone(), state, &new_manifest_cid)?; + migration.add_nv18_migrations(blockstore.clone(), state, new_manifest_cid)?; let actors_in = StateTree::new_from_root(blockstore.clone(), state)?; let actors_out = StateTree::new(blockstore.clone(), StateTreeVersion::V5)?; diff --git a/src/state_migration/nv19/migration.rs b/src/state_migration/nv19/migration.rs index 4d0f34adf229..e18b76d7f601 100644 --- a/src/state_migration/nv19/migration.rs +++ b/src/state_migration/nv19/migration.rs @@ -79,10 +79,9 @@ where .ok_or_else(|| anyhow!("no height info for network version NV19"))? .bundle .as_ref() - .ok_or_else(|| anyhow!("no bundle info for network version NV19"))? - .manifest; + .ok_or_else(|| anyhow!("no bundle info for network version NV19"))?; - blockstore.get(&new_manifest_cid)?.ok_or_else(|| { + blockstore.get(new_manifest_cid)?.ok_or_else(|| { anyhow!( "manifest for network version NV19 not found in blockstore: {}", new_manifest_cid @@ -93,7 +92,7 @@ where let verifier = Arc::new(Verifier::default()); let mut migration = StateMigration::::new(Some(verifier)); - migration.add_nv19_migrations(blockstore.clone(), state, &new_manifest_cid)?; + migration.add_nv19_migrations(blockstore.clone(), state, new_manifest_cid)?; let actors_in = StateTree::new_from_root(blockstore.clone(), state)?; let actors_out = StateTree::new(blockstore.clone(), StateTreeVersion::V5)?; diff --git a/src/state_migration/tests/go-test/state_migration_test.go b/src/state_migration/tests/go-test/state_migration_test.go index 3ea542076e6b..8909dfb5a441 100644 --- a/src/state_migration/tests/go-test/state_migration_test.go +++ b/src/state_migration/tests/go-test/state_migration_test.go @@ -3,7 +3,6 @@ package main import ( "context" "fmt" - "os" "testing" "github.com/filecoin-project/go-state-types/abi" @@ -20,7 +19,7 @@ func TestStateMigrationNV17(t *testing.T) { bs := migration9Test.NewSyncBlockStoreInMemory() ctx := context.Background() - LoadCar(t, ctx, bs, fmt.Sprintf("%s/.local/share/forest/bundles/calibnet/bundle_Shark.car", os.Getenv("HOME"))) + LoadCar(t, ctx, bs, fmt.Sprintf("../../../../target/actor_bundles/%s.car", newManifestCid)) LoadCar(t, ctx, bs, fmt.Sprintf("../data/%s.car", startRoot)) // Or LoadCompressedCar for .car.zst // LoadCompressedCar(t, ctx, bs, fmt.Sprintf("../data/%s.car.zst", startRoot)) diff --git a/src/state_migration/tests/mod.rs b/src/state_migration/tests/mod.rs index e7cbf448125c..99fe9e97c272 100644 --- a/src/state_migration/tests/mod.rs +++ b/src/state_migration/tests/mod.rs @@ -1,10 +1,10 @@ // Copyright 2019-2023 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT -use crate::shim::state_tree::StateRoot; use crate::{ - daemon::bundle::get_actors_bundle, + daemon::bundle::load_actor_bundles, networks::{ChainConfig, Height, NetworkChain}, + shim::state_tree::StateRoot, state_migration::run_state_migrations, }; use anyhow::*; @@ -15,7 +15,6 @@ use std::path::Path; use std::{str::FromStr, sync::Arc}; use tokio::io::AsyncWriteExt; -#[ignore = "https://github.com/ChainSafe/forest/issues/2765"] #[tokio::test] async fn test_nv17_state_migration_calibnet() -> Result<()> { // forest_filecoin::state_migration: State migration at height Shark(epoch 16800) was successful, @@ -32,7 +31,6 @@ async fn test_nv17_state_migration_calibnet() -> Result<()> { .await } -#[ignore = "https://github.com/ChainSafe/forest/issues/2765"] #[tokio::test] async fn test_nv18_state_migration_calibnet() -> Result<()> { // State migration at height Hygge(epoch 322354) was successful, @@ -49,7 +47,6 @@ async fn test_nv18_state_migration_calibnet() -> Result<()> { .await } -#[ignore = "https://github.com/ChainSafe/forest/issues/2765"] #[tokio::test] async fn test_nv19_state_migration_calibnet() -> Result<()> { // State migration at height Lightning(epoch 489094) was successful, @@ -91,22 +88,11 @@ async fn test_state_migration( let store = Arc::new(crate::db::car::plain::PlainCar::new( std::io::BufReader::new(std::fs::File::open(&car_path)?), )?); + load_actor_bundles(&store).await?; + let chain_config = Arc::new(ChainConfig::from_chain(&network)); let height_info = &chain_config.height_infos[height as usize]; - fvm_ipld_car::load_car( - &store, - get_actors_bundle( - &crate::Config { - chain: chain_config.clone(), - ..Default::default() - }, - height, - ) - .await?, - ) - .await?; - let state_root: StateRoot = store.get_cbor(&old_state)?.unwrap(); println!("Actor root (for Go test): {}", state_root.actors);