Offline RPC

CHANGELOG.md

@@ -29,6 +29,9 @@
 
 ### Added
 
+- [#3866](https://github.com/ChainSafe/forest/pull/3866) Implement Offline RPC
+  API.
+
 ### Changed
 
 ### Removed

scripts/tests/calibnet_offline_rpc_check.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# This script is used to test the offline RPC API server against itself.
+# It's run in CI, and uses forest-tool api compare subcommand to test RPC endpoints.
+
+set -euxo pipefail
+
+FOREST_TOOL_PATH="forest-tool"
+PORTS=(8080 8081)
+
+# Function to stop services on specified ports
+stop_services() {
+    for port in "${PORTS[@]}"; do
+        fuser -k "$port/tcp" || true
+    done
+}
+
+# Fetch latest calibnet snapshot
+snapshot="$("$FOREST_TOOL_PATH" snapshot fetch --chain calibnet)"
+
+# Start Offline RPC servers on ports
+for i in "${!PORTS[@]}"; do
+  port=${PORTS[$i]}
+  data_dir="offline-rpc-db-$((i + 1))"
+  "$FOREST_TOOL_PATH" api serve "$snapshot" --chain calibnet --port "$port" --data-dir "$data_dir" &
+done
+
+# Check if services on ports have started
+for port in "${PORTS[@]}"; do
+    until nc -z localhost "$port"; do
+        sleep 30
+    done
+done
+
+# Compare
+$FOREST_TOOL_PATH api compare "$snapshot" --forest /ip4/127.0.0.1/tcp/8080/http --lotus /ip4/127.0.0.1/tcp/8081/http --n-tipsets 5
+exit_code=$?
+
+# Check the result
+if [ $exit_code -ne 0 ]; then
+    stop_services
+    exit 1
+fi
+
+# Stop services on ports
+stop_services

src/cli_shared/logger/mod.rs

@@ -126,7 +126,7 @@ fn default_env_filter() -> EnvFilter {
 
 fn default_tool_filter() -> EnvFilter {
     let default_directives = [
-        "warn",
+        "info",
         "forest::snapshot=info",
         "forest::progress=info",
         "libp2p_bitswap=off",

src/daemon/db_util.rs

@@ -94,7 +94,7 @@ pub async fn import_chain_as_forest_car(
     Ok((forest_car_db_path, ts))
 }
 
-async fn download_to(url: &Url, destination: &Path) -> anyhow::Result<()> {
+pub async fn download_to(url: &Url, destination: &Path) -> anyhow::Result<()> {
     snapshot::download_file_with_retry(
         url,
         destination.parent().with_context(|| {

src/daemon/mod.rs

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0, MIT
 
 pub mod bundle;
-mod db_util;
+pub mod db_util;
 pub mod main;
 
 use crate::auth::{create_token, generate_priv_key, ADMIN, JWT_IDENTIFIER};

src/rpc/auth_api.rs

@@ -8,7 +8,7 @@ use fvm_ipld_blockstore::Blockstore;
 use jsonrpc_v2::{Data, Error as JsonRpcError, Params};
 
 /// RPC call to create a new JWT Token
-pub(in crate::rpc) async fn auth_new<DB: Blockstore>(
+pub async fn auth_new<DB: Blockstore>(
     data: Data<RPCState<DB>>,
     Params(params): Params<AuthNewParams>,
 ) -> Result<LotusJson<Vec<u8>>, JsonRpcError> {
@@ -20,7 +20,7 @@ pub(in crate::rpc) async fn auth_new<DB: Blockstore>(
 }
 
 /// RPC call to verify JWT Token and return the token's permissions
-pub(in crate::rpc) async fn auth_verify<DB>(
+pub async fn auth_verify<DB>(
     data: Data<RPCState<DB>>,
     Params((header_raw,)): Params<(String,)>,
 ) -> Result<Vec<String>, JsonRpcError>

src/rpc/beacon_api.rs

@@ -11,7 +11,7 @@ use jsonrpc_v2::{Data, Error as JsonRpcError, Params};
 /// `BeaconGetEntry` returns the beacon entry for the given Filecoin epoch. If
 /// the entry has not yet been produced, the call will block until the entry
 /// becomes available
-pub(in crate::rpc) async fn beacon_get_entry<DB: Blockstore>(
+pub async fn beacon_get_entry<DB: Blockstore>(
     data: Data<RPCState<DB>>,
     Params((first,)): Params<(ChainEpoch,)>,
 ) -> Result<LotusJson<BeaconEntry>, JsonRpcError> {

src/rpc/chain_api.rs

@@ -26,7 +26,7 @@ use sha2::Sha256;
 use std::sync::Arc;
 use tokio::sync::Mutex;
 
-pub(in crate::rpc) async fn chain_get_message<DB: Blockstore>(
+pub async fn chain_get_message<DB: Blockstore>(
     data: Data<RPCState<DB>>,
     Params(LotusJson((msg_cid,))): Params<LotusJson<(Cid,)>>,
 ) -> Result<LotusJson<Message>, JsonRpcError> {
@@ -41,7 +41,7 @@ pub(in crate::rpc) async fn chain_get_message<DB: Blockstore>(
     }))
 }
 
-pub(in crate::rpc) async fn chain_get_parent_message<DB: Blockstore>(
+pub async fn chain_get_parent_message<DB: Blockstore>(
     data: Data<RPCState<DB>>,
     Params(LotusJson((block_cid,))): Params<LotusJson<(Cid,)>>,
 ) -> Result<LotusJson<Vec<ApiMessage>>, JsonRpcError> {
@@ -58,7 +58,7 @@ pub(in crate::rpc) async fn chain_get_parent_message<DB: Blockstore>(
     }
 }
 
-pub(in crate::rpc) async fn chain_get_parent_receipts<DB: Blockstore + Send + Sync + 'static>(
+pub async fn chain_get_parent_receipts<DB: Blockstore + Send + Sync + 'static>(
     data: Data<RPCState<DB>>,
     Params(LotusJson((block_cid,))): Params<LotusJson<(Cid,)>>,
 ) -> Result<LotusJson<Vec<ApiReceipt>>, JsonRpcError> {
@@ -128,7 +128,7 @@ pub(crate) async fn chain_get_messages_in_tipset<DB: Blockstore>(
     Ok(LotusJson(messages))
 }
 
-pub(in crate::rpc) async fn chain_export<DB>(
+pub async fn chain_export<DB>(
     data: Data<RPCState<DB>>,
     Params(ChainExportParams {
         epoch,
@@ -192,7 +192,7 @@ where
     }
 }
 
-pub(in crate::rpc) async fn chain_read_obj<DB: Blockstore>(
+pub async fn chain_read_obj<DB: Blockstore>(
     data: Data<RPCState<DB>>,
     Params(LotusJson((obj_cid,))): Params<LotusJson<(Cid,)>>,
 ) -> Result<LotusJson<Vec<u8>>, JsonRpcError> {
@@ -204,14 +204,14 @@ pub(in crate::rpc) async fn chain_read_obj<DB: Blockstore>(
     Ok(LotusJson(bytes))
 }
 
-pub(in crate::rpc) async fn chain_has_obj<DB: Blockstore>(
+pub async fn chain_has_obj<DB: Blockstore>(
     data: Data<RPCState<DB>>,
     Params(LotusJson((obj_cid,))): Params<LotusJson<(Cid,)>>,
 ) -> Result<bool, JsonRpcError> {
     Ok(data.state_manager.blockstore().get(&obj_cid)?.is_some())
 }
 
-pub(in crate::rpc) async fn chain_get_block_messages<DB: Blockstore>(
+pub async fn chain_get_block_messages<DB: Blockstore>(
     data: Data<RPCState<DB>>,
     Params(LotusJson((blk_cid,))): Params<LotusJson<(Cid,)>>,
 ) -> Result<BlockMessages, JsonRpcError> {
@@ -241,7 +241,7 @@ pub(in crate::rpc) async fn chain_get_block_messages<DB: Blockstore>(
     Ok(ret)
 }
 
-pub(in crate::rpc) async fn chain_get_tipset_by_height<DB: Blockstore>(
+pub async fn chain_get_tipset_by_height<DB: Blockstore>(
     data: Data<RPCState<DB>>,
     Params(LotusJson((height, tsk))): Params<LotusJson<(ChainEpoch, TipsetKey)>>,
 ) -> Result<LotusJson<Tipset>, JsonRpcError> {
@@ -257,21 +257,21 @@ pub(in crate::rpc) async fn chain_get_tipset_by_height<DB: Blockstore>(
     Ok((*tss).clone().into())
 }
 
-pub(in crate::rpc) async fn chain_get_genesis<DB: Blockstore>(
+pub async fn chain_get_genesis<DB: Blockstore>(
     data: Data<RPCState<DB>>,
 ) -> Result<Option<LotusJson<Tipset>>, JsonRpcError> {
     let genesis = data.state_manager.chain_store().genesis_block_header();
     Ok(Some(Tipset::from(genesis).into()))
 }
 
-pub(in crate::rpc) async fn chain_head<DB: Blockstore>(
+pub async fn chain_head<DB: Blockstore>(
     data: Data<RPCState<DB>>,
 ) -> Result<LotusJson<Tipset>, JsonRpcError> {
     let heaviest = data.state_manager.chain_store().heaviest_tipset();
     Ok((*heaviest).clone().into())
 }
 
-pub(in crate::rpc) async fn chain_get_block<DB: Blockstore>(
+pub async fn chain_get_block<DB: Blockstore>(
     data: Data<RPCState<DB>>,
     Params(LotusJson((blk_cid,))): Params<LotusJson<(Cid,)>>,
 ) -> Result<LotusJson<CachingBlockHeader>, JsonRpcError> {
@@ -283,7 +283,7 @@ pub(in crate::rpc) async fn chain_get_block<DB: Blockstore>(
     Ok(blk.into())
 }
 
-pub(in crate::rpc) async fn chain_get_tipset<DB: Blockstore>(
+pub async fn chain_get_tipset<DB: Blockstore>(
     data: Data<RPCState<DB>>,
     Params(LotusJson((tsk,))): Params<LotusJson<(TipsetKey,)>>,
 ) -> Result<LotusJson<Tipset>, JsonRpcError> {
@@ -296,7 +296,7 @@ pub(in crate::rpc) async fn chain_get_tipset<DB: Blockstore>(
 
 // This is basically a port of the reference implementation at
 // https://github.com/filecoin-project/lotus/blob/v1.23.0/node/impl/full/chain.go#L321
-pub(in crate::rpc) async fn chain_set_head<DB: Blockstore>(
+pub async fn chain_set_head<DB: Blockstore>(
     data: Data<RPCState<DB>>,
     Params(LotusJson((tsk,))): Params<LotusJson<(TipsetKey,)>>,
 ) -> Result<(), JsonRpcError> {

src/rpc/common_api.rs

@@ -14,11 +14,11 @@ use uuid::Uuid;
 static SESSION_UUID: Lazy<Uuid> = Lazy::new(Uuid::new_v4);
 
 /// The session UUID uniquely identifies the API node.
-pub(in crate::rpc) async fn session() -> Result<String, JsonRpcError> {
+pub async fn session() -> Result<String, JsonRpcError> {
     Ok(SESSION_UUID.to_string())
 }
 
-pub(in crate::rpc) async fn version(
+pub async fn version(
     block_delay: u64,
     forest_version: &'static str,
 ) -> Result<APIVersion, JsonRpcError> {
@@ -30,7 +30,7 @@ pub(in crate::rpc) async fn version(
     })
 }
 
-pub(in crate::rpc) async fn shutdown(shutdown_send: Sender<()>) -> Result<(), JsonRpcError> {
+pub async fn shutdown(shutdown_send: Sender<()>) -> Result<(), JsonRpcError> {
     // Trigger graceful shutdown
     if let Err(err) = shutdown_send.send(()).await {
         return Err(JsonRpcError::from(err));
@@ -39,7 +39,7 @@ pub(in crate::rpc) async fn shutdown(shutdown_send: Sender<()>) -> Result<(), Js
 }
 
 /// gets start time from network
-pub(in crate::rpc) async fn start_time<DB: Blockstore>(
+pub async fn start_time<DB: Blockstore>(
     data: Data<RPCState<DB>>,
 ) -> Result<chrono::DateTime<chrono::Utc>, JsonRpcError> {
     Ok(data.start_time)

src/rpc/eth_api.rs

@@ -18,12 +18,12 @@ use jsonrpc_v2::{Data, Error as JsonRpcError, Params};
 use num_bigint::BigInt;
 use num_traits::Zero as _;
 
-pub(in crate::rpc) async fn eth_accounts() -> Result<Vec<String>, JsonRpcError> {
+pub async fn eth_accounts() -> Result<Vec<String>, JsonRpcError> {
     // EthAccounts will always return [] since we don't expect Forest to manage private keys
     Ok(vec![])
 }
 
-pub(in crate::rpc) async fn eth_block_number<DB: Blockstore>(
+pub async fn eth_block_number<DB: Blockstore>(
     data: Data<RPCState<DB>>,
 ) -> Result<String, JsonRpcError> {
     // `eth_block_number` needs to return the height of the latest committed tipset.
@@ -51,7 +51,7 @@ pub(in crate::rpc) async fn eth_block_number<DB: Blockstore>(
     }
 }
 
-pub(in crate::rpc) async fn eth_chain_id<DB: Blockstore>(
+pub async fn eth_chain_id<DB: Blockstore>(
     data: Data<RPCState<DB>>,
 ) -> Result<String, JsonRpcError> {
     Ok(format!(
@@ -60,7 +60,7 @@ pub(in crate::rpc) async fn eth_chain_id<DB: Blockstore>(
     ))
 }
 
-pub(in crate::rpc) async fn eth_gas_price<DB: Blockstore>(
+pub async fn eth_gas_price<DB: Blockstore>(
     data: Data<RPCState<DB>>,
 ) -> Result<GasPriceResult, JsonRpcError> {
     let ts = data.state_manager.chain_store().heaviest_tipset();
@@ -74,7 +74,7 @@ pub(in crate::rpc) async fn eth_gas_price<DB: Blockstore>(
     }
 }
 
-pub(in crate::rpc) async fn eth_get_balance<DB: Blockstore>(
+pub async fn eth_get_balance<DB: Blockstore>(
     data: Data<RPCState<DB>>,
     Params(LotusJson((address, block_param))): Params<LotusJson<(Address, BlockNumberOrHash)>>,
 ) -> Result<EthBigInt, JsonRpcError> {

src/rpc/gas_api.rs

@@ -19,7 +19,7 @@ use rand_distr::{Distribution, Normal};
 const MIN_GAS_PREMIUM: f64 = 100000.0;
 
 /// Estimate the fee cap
-pub(in crate::rpc) async fn gas_estimate_fee_cap<DB: Blockstore>(
+pub async fn gas_estimate_fee_cap<DB: Blockstore>(
     data: Data<RPCState<DB>>,
     Params(params): Params<LotusJson<(Message, i64, TipsetKey)>>,
 ) -> Result<String, JsonRpcError> {
@@ -49,7 +49,7 @@ fn estimate_fee_cap<DB: Blockstore>(
 }
 
 /// Estimate the fee cap
-pub(in crate::rpc) async fn gas_estimate_gas_premium<DB: Blockstore>(
+pub async fn gas_estimate_gas_premium<DB: Blockstore>(
     data: Data<RPCState<DB>>,
     Params(params): Params<LotusJson<(u64, Address, i64, TipsetKey)>>,
 ) -> Result<String, JsonRpcError> {
@@ -141,7 +141,7 @@ pub async fn estimate_gas_premium<DB: Blockstore>(
 }
 
 /// Estimate the gas limit
-pub(in crate::rpc) async fn gas_estimate_gas_limit<DB>(
+pub async fn gas_estimate_gas_limit<DB>(
     data: Data<RPCState<DB>>,
     Params(LotusJson((msg, tsk))): Params<LotusJson<(Message, TipsetKey)>>,
 ) -> Result<i64, JsonRpcError>
@@ -195,7 +195,7 @@ where
 }
 
 /// Estimates the gas parameters for a given message
-pub(in crate::rpc) async fn gas_estimate_message_gas<DB>(
+pub async fn gas_estimate_message_gas<DB>(
     data: Data<RPCState<DB>>,
     Params(LotusJson((msg, spec, tsk))): Params<
         LotusJson<(Message, Option<MessageSendSpec>, TipsetKey)>,
@@ -209,7 +209,7 @@ where
         .map(Into::into)
 }
 
-pub(in crate::rpc) async fn estimate_message_gas<DB>(
+pub async fn estimate_message_gas<DB>(
     data: &Data<RPCState<DB>>,
     msg: Message,
     _spec: Option<MessageSendSpec>,

src/rpc/mpool_api.rs

@@ -21,7 +21,7 @@ use jsonrpc_v2::{Data, Error as JsonRpcError, Params};
 use super::gas_api::estimate_message_gas;
 
 /// Gets next nonce for the specified sender.
-pub(in crate::rpc) async fn mpool_get_nonce<DB>(
+pub async fn mpool_get_nonce<DB>(
     data: Data<RPCState<DB>>,
     Params(LotusJson((address,))): Params<LotusJson<(Address,)>>,
 ) -> Result<u64, JsonRpcError>
@@ -32,7 +32,7 @@ where
 }
 
 /// Return `Vec` of pending messages in `mpool`
-pub(in crate::rpc) async fn mpool_pending<DB>(
+pub async fn mpool_pending<DB>(
     data: Data<RPCState<DB>>,
     Params(LotusJson((cid_vec,))): Params<LotusJson<(Vec<Cid>,)>>,
 ) -> Result<LotusJson<Vec<SignedMessage>>, JsonRpcError>
@@ -100,7 +100,7 @@ where
 }
 
 /// Add `SignedMessage` to `mpool`, return message CID
-pub(in crate::rpc) async fn mpool_push<DB>(
+pub async fn mpool_push<DB>(
     data: Data<RPCState<DB>>,
     Params(LotusJson((signed_message,))): Params<LotusJson<(SignedMessage,)>>,
 ) -> Result<LotusJson<Cid>, JsonRpcError>
@@ -113,7 +113,7 @@ where
 }
 
 /// Sign given `UnsignedMessage` and add it to `mpool`, return `SignedMessage`
-pub(in crate::rpc) async fn mpool_push_message<DB>(
+pub async fn mpool_push_message<DB>(
     data: Data<RPCState<DB>>,
     Params(LotusJson((umsg, spec))): Params<LotusJson<(Message, Option<MessageSendSpec>)>>,
 ) -> Result<LotusJson<SignedMessage>, JsonRpcError>