diff --git a/src/chain/store/chain_store.rs b/src/chain/store/chain_store.rs index 61b2ae5b374..a3b6592dbdd 100644 --- a/src/chain/store/chain_store.rs +++ b/src/chain/store/chain_store.rs @@ -234,7 +234,7 @@ where &self .settings .require_obj::(HEAD_KEY) - .expect("failed to load heaviest tipset"), + .expect("failed to load heaviest tipset key"), ) .expect("failed to load heaviest tipset") } diff --git a/src/chain/store/index.rs b/src/chain/store/index.rs index a9383e396e3..17d59dceceb 100644 --- a/src/chain/store/index.rs +++ b/src/chain/store/index.rs @@ -7,6 +7,7 @@ use crate::beacon::{BeaconEntry, IGNORE_DRAND_VAR}; use crate::blocks::{Tipset, TipsetKey}; use crate::metrics; use crate::shim::clock::ChainEpoch; +use crate::utils::misc::env::is_env_truthy; use fvm_ipld_blockstore::Blockstore; use itertools::Itertools; use lru::LruCache; @@ -47,11 +48,13 @@ impl ChainIndex { /// Loads a tipset from memory given the tipset keys and cache. Semantically /// identical to [`Tipset::load`] but the result is cached. pub fn load_tipset(&self, tsk: &TipsetKey) -> Result>, Error> { - if let Some(ts) = self.ts_cache.lock().get(tsk) { - metrics::LRU_CACHE_HIT - .get_or_create(&metrics::values::TIPSET) - .inc(); - return Ok(Some(ts.clone())); + if !is_env_truthy("FOREST_TIPSET_CACHE_DISABLED") { + if let Some(ts) = self.ts_cache.lock().get(tsk) { + metrics::LRU_CACHE_HIT + .get_or_create(&metrics::values::TIPSET) + .inc(); + return Ok(Some(ts.clone())); + } } let ts_opt = Tipset::load(&self.db, tsk)?.map(Arc::new); diff --git a/src/cli_shared/snapshot.rs b/src/cli_shared/snapshot.rs index 87de6098d47..2c9e3e92900 100644 --- a/src/cli_shared/snapshot.rs +++ b/src/cli_shared/snapshot.rs @@ -134,6 +134,9 @@ fn parse_content_disposition(value: &reqwest::header::HeaderValue) -> Option anyhow::Result { + if !directory.is_dir() { + std::fs::create_dir_all(directory)?; + } let dst_path = directory.join(filename); let destination = dst_path.display(); event!(target: "forest::snapshot", tracing::Level::INFO, %url, %destination, "downloading snapshot"); diff --git a/src/db/memory.rs b/src/db/memory.rs index 6d0abe23b00..d813b2eb9b4 100644 --- a/src/db/memory.rs +++ b/src/db/memory.rs @@ -11,6 +11,7 @@ use cid::Cid; use fvm_ipld_blockstore::Blockstore; use itertools::Itertools; use parking_lot::RwLock; +use std::ops::Deref; use super::{EthMappingsStore, SettingsStore}; @@ -22,6 +23,44 @@ pub struct MemoryDB { eth_mappings_db: RwLock>>, } +impl MemoryDB { + #[allow(dead_code)] + pub fn serialize(&self) -> anyhow::Result> { + let blockchain_db = self.blockchain_db.read(); + let blockchain_persistent_db = self.blockchain_persistent_db.read(); + let settings_db = self.settings_db.read(); + let eth_mappings_db = self.eth_mappings_db.read(); + let tuple = ( + blockchain_db.deref(), + blockchain_persistent_db.deref(), + settings_db.deref(), + eth_mappings_db.deref(), + ); + Ok(fvm_ipld_encoding::to_vec(&tuple)?) + } + + pub fn deserialize_from(bytes: &[u8]) -> anyhow::Result { + let (blockchain_db, blockchain_persistent_db, settings_db, eth_mappings_db) = + fvm_ipld_encoding::from_slice(bytes)?; + Ok(Self { + blockchain_db: RwLock::new(blockchain_db), + blockchain_persistent_db: RwLock::new(blockchain_persistent_db), + settings_db: RwLock::new(settings_db), + eth_mappings_db: RwLock::new(eth_mappings_db), + }) + } + + pub fn deserialize_from_legacy(bytes: &[u8]) -> anyhow::Result { + let (blockchain_db, settings_db, eth_mappings_db) = fvm_ipld_encoding::from_slice(bytes)?; + Ok(Self { + blockchain_db: RwLock::new(blockchain_db), + blockchain_persistent_db: Default::default(), + settings_db: RwLock::new(settings_db), + eth_mappings_db: RwLock::new(eth_mappings_db), + }) + } +} + impl GarbageCollectable for MemoryDB { fn get_keys(&self) -> anyhow::Result { let mut set = CidHashSet::new(); diff --git a/src/rpc/mod.rs b/src/rpc/mod.rs index cb1c9856f0e..4eb3ddf9064 100644 --- a/src/rpc/mod.rs +++ b/src/rpc/mod.rs @@ -20,6 +20,7 @@ mod error; mod reflect; pub mod types; pub use methods::*; +use serde::{Deserialize, Serialize}; /// Protocol or transport-specific error pub use jsonrpsee::core::ClientError; @@ -31,237 +32,238 @@ pub use jsonrpsee::core::ClientError; /// trait. /// /// All methods should be entered here. +#[macro_export] macro_rules! for_each_method { ($callback:path) => { // auth vertical - $callback!(crate::rpc::auth::AuthNew); - $callback!(crate::rpc::auth::AuthVerify); + $callback!($crate::rpc::auth::AuthNew); + $callback!($crate::rpc::auth::AuthVerify); // beacon vertical - $callback!(crate::rpc::beacon::BeaconGetEntry); + $callback!($crate::rpc::beacon::BeaconGetEntry); // chain vertical - $callback!(crate::rpc::chain::ChainExport); - $callback!(crate::rpc::chain::ChainGetBlock); - $callback!(crate::rpc::chain::ChainGetBlockMessages); - $callback!(crate::rpc::chain::ChainGetEvents); - $callback!(crate::rpc::chain::ChainGetGenesis); - $callback!(crate::rpc::chain::ChainGetMessage); - $callback!(crate::rpc::chain::ChainGetMessagesInTipset); - $callback!(crate::rpc::chain::ChainGetMinBaseFee); - $callback!(crate::rpc::chain::ChainGetParentMessages); - $callback!(crate::rpc::chain::ChainGetParentReceipts); - $callback!(crate::rpc::chain::ChainGetPath); - $callback!(crate::rpc::chain::ChainGetTipSet); - $callback!(crate::rpc::chain::ChainGetTipSetAfterHeight); - $callback!(crate::rpc::chain::ChainGetTipSetByHeight); - $callback!(crate::rpc::chain::ChainHasObj); - $callback!(crate::rpc::chain::ChainHead); - $callback!(crate::rpc::chain::ChainReadObj); - $callback!(crate::rpc::chain::ChainSetHead); - $callback!(crate::rpc::chain::ChainStatObj); - $callback!(crate::rpc::chain::ChainTipSetWeight); + $callback!($crate::rpc::chain::ChainExport); + $callback!($crate::rpc::chain::ChainGetBlock); + $callback!($crate::rpc::chain::ChainGetBlockMessages); + $callback!($crate::rpc::chain::ChainGetEvents); + $callback!($crate::rpc::chain::ChainGetGenesis); + $callback!($crate::rpc::chain::ChainGetMessage); + $callback!($crate::rpc::chain::ChainGetMessagesInTipset); + $callback!($crate::rpc::chain::ChainGetMinBaseFee); + $callback!($crate::rpc::chain::ChainGetParentMessages); + $callback!($crate::rpc::chain::ChainGetParentReceipts); + $callback!($crate::rpc::chain::ChainGetPath); + $callback!($crate::rpc::chain::ChainGetTipSet); + $callback!($crate::rpc::chain::ChainGetTipSetAfterHeight); + $callback!($crate::rpc::chain::ChainGetTipSetByHeight); + $callback!($crate::rpc::chain::ChainHasObj); + $callback!($crate::rpc::chain::ChainHead); + $callback!($crate::rpc::chain::ChainReadObj); + $callback!($crate::rpc::chain::ChainSetHead); + $callback!($crate::rpc::chain::ChainStatObj); + $callback!($crate::rpc::chain::ChainTipSetWeight); // common vertical - $callback!(crate::rpc::common::Session); - $callback!(crate::rpc::common::Shutdown); - $callback!(crate::rpc::common::StartTime); - $callback!(crate::rpc::common::Version); + $callback!($crate::rpc::common::Session); + $callback!($crate::rpc::common::Shutdown); + $callback!($crate::rpc::common::StartTime); + $callback!($crate::rpc::common::Version); // eth vertical - $callback!(crate::rpc::eth::EthAccounts); - $callback!(crate::rpc::eth::EthAddressToFilecoinAddress); - $callback!(crate::rpc::eth::EthBlockNumber); - $callback!(crate::rpc::eth::EthCall); - $callback!(crate::rpc::eth::EthChainId); - $callback!(crate::rpc::eth::EthEstimateGas); - $callback!(crate::rpc::eth::EthFeeHistory); - $callback!(crate::rpc::eth::EthGasPrice); - $callback!(crate::rpc::eth::EthGetBalance); - $callback!(crate::rpc::eth::EthGetBlockByHash); - $callback!(crate::rpc::eth::EthGetBlockByNumber); - $callback!(crate::rpc::eth::EthGetBlockReceipts); - $callback!(crate::rpc::eth::EthGetBlockReceiptsLimited); - $callback!(crate::rpc::eth::EthGetBlockTransactionCountByHash); - $callback!(crate::rpc::eth::EthGetBlockTransactionCountByNumber); - $callback!(crate::rpc::eth::EthGetCode); - $callback!(crate::rpc::eth::EthGetLogs); - $callback!(crate::rpc::eth::EthGetMessageCidByTransactionHash); - $callback!(crate::rpc::eth::EthGetStorageAt); - $callback!(crate::rpc::eth::EthGetTransactionByHash); - $callback!(crate::rpc::eth::EthGetTransactionByHashLimited); - $callback!(crate::rpc::eth::EthGetTransactionCount); - $callback!(crate::rpc::eth::EthGetTransactionHashByCid); - $callback!(crate::rpc::eth::EthGetTransactionByBlockNumberAndIndex); - $callback!(crate::rpc::eth::EthGetTransactionByBlockHashAndIndex); - $callback!(crate::rpc::eth::EthMaxPriorityFeePerGas); - $callback!(crate::rpc::eth::EthProtocolVersion); - $callback!(crate::rpc::eth::EthGetTransactionReceipt); - $callback!(crate::rpc::eth::EthGetTransactionReceiptLimited); - $callback!(crate::rpc::eth::EthNewFilter); - $callback!(crate::rpc::eth::EthNewPendingTransactionFilter); - $callback!(crate::rpc::eth::EthNewBlockFilter); - $callback!(crate::rpc::eth::EthUninstallFilter); - $callback!(crate::rpc::eth::EthSyncing); - $callback!(crate::rpc::eth::Web3ClientVersion); - $callback!(crate::rpc::eth::EthSendRawTransaction); + $callback!($crate::rpc::eth::EthAccounts); + $callback!($crate::rpc::eth::EthAddressToFilecoinAddress); + $callback!($crate::rpc::eth::EthBlockNumber); + $callback!($crate::rpc::eth::EthCall); + $callback!($crate::rpc::eth::EthChainId); + $callback!($crate::rpc::eth::EthEstimateGas); + $callback!($crate::rpc::eth::EthFeeHistory); + $callback!($crate::rpc::eth::EthGasPrice); + $callback!($crate::rpc::eth::EthGetBalance); + $callback!($crate::rpc::eth::EthGetBlockByHash); + $callback!($crate::rpc::eth::EthGetBlockByNumber); + $callback!($crate::rpc::eth::EthGetBlockReceipts); + $callback!($crate::rpc::eth::EthGetBlockReceiptsLimited); + $callback!($crate::rpc::eth::EthGetBlockTransactionCountByHash); + $callback!($crate::rpc::eth::EthGetBlockTransactionCountByNumber); + $callback!($crate::rpc::eth::EthGetCode); + $callback!($crate::rpc::eth::EthGetLogs); + $callback!($crate::rpc::eth::EthGetMessageCidByTransactionHash); + $callback!($crate::rpc::eth::EthGetStorageAt); + $callback!($crate::rpc::eth::EthGetTransactionByHash); + $callback!($crate::rpc::eth::EthGetTransactionByHashLimited); + $callback!($crate::rpc::eth::EthGetTransactionCount); + $callback!($crate::rpc::eth::EthGetTransactionHashByCid); + $callback!($crate::rpc::eth::EthGetTransactionByBlockNumberAndIndex); + $callback!($crate::rpc::eth::EthGetTransactionByBlockHashAndIndex); + $callback!($crate::rpc::eth::EthMaxPriorityFeePerGas); + $callback!($crate::rpc::eth::EthProtocolVersion); + $callback!($crate::rpc::eth::EthGetTransactionReceipt); + $callback!($crate::rpc::eth::EthGetTransactionReceiptLimited); + $callback!($crate::rpc::eth::EthNewFilter); + $callback!($crate::rpc::eth::EthNewPendingTransactionFilter); + $callback!($crate::rpc::eth::EthNewBlockFilter); + $callback!($crate::rpc::eth::EthUninstallFilter); + $callback!($crate::rpc::eth::EthSyncing); + $callback!($crate::rpc::eth::Web3ClientVersion); + $callback!($crate::rpc::eth::EthSendRawTransaction); // gas vertical - $callback!(crate::rpc::gas::GasEstimateFeeCap); - $callback!(crate::rpc::gas::GasEstimateGasLimit); - $callback!(crate::rpc::gas::GasEstimateGasPremium); - $callback!(crate::rpc::gas::GasEstimateMessageGas); + $callback!($crate::rpc::gas::GasEstimateFeeCap); + $callback!($crate::rpc::gas::GasEstimateGasLimit); + $callback!($crate::rpc::gas::GasEstimateGasPremium); + $callback!($crate::rpc::gas::GasEstimateMessageGas); // market vertical - $callback!(crate::rpc::market::MarketAddBalance); + $callback!($crate::rpc::market::MarketAddBalance); // miner vertical - $callback!(crate::rpc::miner::MinerCreateBlock); - $callback!(crate::rpc::miner::MinerGetBaseInfo); + $callback!($crate::rpc::miner::MinerCreateBlock); + $callback!($crate::rpc::miner::MinerGetBaseInfo); // mpool vertical - $callback!(crate::rpc::mpool::MpoolBatchPush); - $callback!(crate::rpc::mpool::MpoolBatchPushUntrusted); - $callback!(crate::rpc::mpool::MpoolGetNonce); - $callback!(crate::rpc::mpool::MpoolPending); - $callback!(crate::rpc::mpool::MpoolPush); - $callback!(crate::rpc::mpool::MpoolPushMessage); - $callback!(crate::rpc::mpool::MpoolPushUntrusted); - $callback!(crate::rpc::mpool::MpoolSelect); + $callback!($crate::rpc::mpool::MpoolBatchPush); + $callback!($crate::rpc::mpool::MpoolBatchPushUntrusted); + $callback!($crate::rpc::mpool::MpoolGetNonce); + $callback!($crate::rpc::mpool::MpoolPending); + $callback!($crate::rpc::mpool::MpoolPush); + $callback!($crate::rpc::mpool::MpoolPushMessage); + $callback!($crate::rpc::mpool::MpoolPushUntrusted); + $callback!($crate::rpc::mpool::MpoolSelect); // msig vertical - $callback!(crate::rpc::msig::MsigGetAvailableBalance); - $callback!(crate::rpc::msig::MsigGetPending); - $callback!(crate::rpc::msig::MsigGetVested); - $callback!(crate::rpc::msig::MsigGetVestingSchedule); + $callback!($crate::rpc::msig::MsigGetAvailableBalance); + $callback!($crate::rpc::msig::MsigGetPending); + $callback!($crate::rpc::msig::MsigGetVested); + $callback!($crate::rpc::msig::MsigGetVestingSchedule); // net vertical - $callback!(crate::rpc::net::NetAddrsListen); - $callback!(crate::rpc::net::NetAgentVersion); - $callback!(crate::rpc::net::NetAutoNatStatus); - $callback!(crate::rpc::net::NetConnect); - $callback!(crate::rpc::net::NetDisconnect); - $callback!(crate::rpc::net::NetFindPeer); - $callback!(crate::rpc::net::NetInfo); - $callback!(crate::rpc::net::NetListening); - $callback!(crate::rpc::net::NetPeers); - $callback!(crate::rpc::net::NetProtectAdd); - $callback!(crate::rpc::net::NetProtectList); - $callback!(crate::rpc::net::NetProtectRemove); - $callback!(crate::rpc::net::NetVersion); + $callback!($crate::rpc::net::NetAddrsListen); + $callback!($crate::rpc::net::NetAgentVersion); + $callback!($crate::rpc::net::NetAutoNatStatus); + $callback!($crate::rpc::net::NetConnect); + $callback!($crate::rpc::net::NetDisconnect); + $callback!($crate::rpc::net::NetFindPeer); + $callback!($crate::rpc::net::NetInfo); + $callback!($crate::rpc::net::NetListening); + $callback!($crate::rpc::net::NetPeers); + $callback!($crate::rpc::net::NetProtectAdd); + $callback!($crate::rpc::net::NetProtectList); + $callback!($crate::rpc::net::NetProtectRemove); + $callback!($crate::rpc::net::NetVersion); // node vertical - $callback!(crate::rpc::node::NodeStatus); + $callback!($crate::rpc::node::NodeStatus); // state vertical - $callback!(crate::rpc::state::StateAccountKey); - $callback!(crate::rpc::state::StateCall); - $callback!(crate::rpc::state::StateCirculatingSupply); - $callback!(crate::rpc::state::StateCompute); - $callback!(crate::rpc::state::StateDealProviderCollateralBounds); - $callback!(crate::rpc::state::StateFetchRoot); - $callback!(crate::rpc::state::StateGetActor); - $callback!(crate::rpc::state::StateGetAllAllocations); - $callback!(crate::rpc::state::StateGetAllClaims); - $callback!(crate::rpc::state::StateGetAllocation); - $callback!(crate::rpc::state::StateGetAllocationForPendingDeal); - $callback!(crate::rpc::state::StateGetAllocationIdForPendingDeal); - $callback!(crate::rpc::state::StateGetAllocations); - $callback!(crate::rpc::state::StateGetBeaconEntry); - $callback!(crate::rpc::state::StateGetClaim); - $callback!(crate::rpc::state::StateGetClaims); - $callback!(crate::rpc::state::StateGetNetworkParams); - $callback!(crate::rpc::state::StateGetRandomnessDigestFromBeacon); - $callback!(crate::rpc::state::StateGetRandomnessDigestFromTickets); - $callback!(crate::rpc::state::StateGetRandomnessFromBeacon); - $callback!(crate::rpc::state::StateGetRandomnessFromTickets); - $callback!(crate::rpc::state::StateGetReceipt); - $callback!(crate::rpc::state::StateListActors); - $callback!(crate::rpc::state::StateListMessages); - $callback!(crate::rpc::state::StateListMiners); - $callback!(crate::rpc::state::StateLookupID); - $callback!(crate::rpc::state::StateLookupRobustAddress); - $callback!(crate::rpc::state::StateMarketBalance); - $callback!(crate::rpc::state::StateMarketDeals); - $callback!(crate::rpc::state::StateMarketParticipants); - $callback!(crate::rpc::state::StateMarketStorageDeal); - $callback!(crate::rpc::state::StateMinerActiveSectors); - $callback!(crate::rpc::state::StateMinerAllocated); - $callback!(crate::rpc::state::StateMinerAvailableBalance); - $callback!(crate::rpc::state::StateMinerDeadlines); - $callback!(crate::rpc::state::StateMinerFaults); - $callback!(crate::rpc::state::StateMinerInfo); - $callback!(crate::rpc::state::StateMinerInitialPledgeCollateral); - $callback!(crate::rpc::state::StateMinerPartitions); - $callback!(crate::rpc::state::StateMinerPower); - $callback!(crate::rpc::state::StateMinerPreCommitDepositForPower); - $callback!(crate::rpc::state::StateMinerProvingDeadline); - $callback!(crate::rpc::state::StateMinerRecoveries); - $callback!(crate::rpc::state::StateMinerSectorAllocated); - $callback!(crate::rpc::state::StateMinerSectorCount); - $callback!(crate::rpc::state::StateMinerSectors); - $callback!(crate::rpc::state::StateNetworkName); - $callback!(crate::rpc::state::StateNetworkVersion); - $callback!(crate::rpc::state::StateReadState); - $callback!(crate::rpc::state::StateReplay); - $callback!(crate::rpc::state::StateSearchMsg); - $callback!(crate::rpc::state::StateSearchMsgLimited); - $callback!(crate::rpc::state::StateSectorExpiration); - $callback!(crate::rpc::state::StateSectorGetInfo); - $callback!(crate::rpc::state::StateSectorPartition); - $callback!(crate::rpc::state::StateSectorPreCommitInfo); - $callback!(crate::rpc::state::StateSectorPreCommitInfoV0); - $callback!(crate::rpc::state::StateVerifiedClientStatus); - $callback!(crate::rpc::state::StateVerifiedRegistryRootKey); - $callback!(crate::rpc::state::StateVerifierStatus); - $callback!(crate::rpc::state::StateVMCirculatingSupplyInternal); - $callback!(crate::rpc::state::StateWaitMsg); - $callback!(crate::rpc::state::StateWaitMsgV0); + $callback!($crate::rpc::state::StateAccountKey); + $callback!($crate::rpc::state::StateCall); + $callback!($crate::rpc::state::StateCirculatingSupply); + $callback!($crate::rpc::state::StateCompute); + $callback!($crate::rpc::state::StateDealProviderCollateralBounds); + $callback!($crate::rpc::state::StateFetchRoot); + $callback!($crate::rpc::state::StateGetActor); + $callback!($crate::rpc::state::StateGetAllAllocations); + $callback!($crate::rpc::state::StateGetAllClaims); + $callback!($crate::rpc::state::StateGetAllocation); + $callback!($crate::rpc::state::StateGetAllocationForPendingDeal); + $callback!($crate::rpc::state::StateGetAllocationIdForPendingDeal); + $callback!($crate::rpc::state::StateGetAllocations); + $callback!($crate::rpc::state::StateGetBeaconEntry); + $callback!($crate::rpc::state::StateGetClaim); + $callback!($crate::rpc::state::StateGetClaims); + $callback!($crate::rpc::state::StateGetNetworkParams); + $callback!($crate::rpc::state::StateGetRandomnessDigestFromBeacon); + $callback!($crate::rpc::state::StateGetRandomnessDigestFromTickets); + $callback!($crate::rpc::state::StateGetRandomnessFromBeacon); + $callback!($crate::rpc::state::StateGetRandomnessFromTickets); + $callback!($crate::rpc::state::StateGetReceipt); + $callback!($crate::rpc::state::StateListActors); + $callback!($crate::rpc::state::StateListMessages); + $callback!($crate::rpc::state::StateListMiners); + $callback!($crate::rpc::state::StateLookupID); + $callback!($crate::rpc::state::StateLookupRobustAddress); + $callback!($crate::rpc::state::StateMarketBalance); + $callback!($crate::rpc::state::StateMarketDeals); + $callback!($crate::rpc::state::StateMarketParticipants); + $callback!($crate::rpc::state::StateMarketStorageDeal); + $callback!($crate::rpc::state::StateMinerActiveSectors); + $callback!($crate::rpc::state::StateMinerAllocated); + $callback!($crate::rpc::state::StateMinerAvailableBalance); + $callback!($crate::rpc::state::StateMinerDeadlines); + $callback!($crate::rpc::state::StateMinerFaults); + $callback!($crate::rpc::state::StateMinerInfo); + $callback!($crate::rpc::state::StateMinerInitialPledgeCollateral); + $callback!($crate::rpc::state::StateMinerPartitions); + $callback!($crate::rpc::state::StateMinerPower); + $callback!($crate::rpc::state::StateMinerPreCommitDepositForPower); + $callback!($crate::rpc::state::StateMinerProvingDeadline); + $callback!($crate::rpc::state::StateMinerRecoveries); + $callback!($crate::rpc::state::StateMinerSectorAllocated); + $callback!($crate::rpc::state::StateMinerSectorCount); + $callback!($crate::rpc::state::StateMinerSectors); + $callback!($crate::rpc::state::StateNetworkName); + $callback!($crate::rpc::state::StateNetworkVersion); + $callback!($crate::rpc::state::StateReadState); + $callback!($crate::rpc::state::StateReplay); + $callback!($crate::rpc::state::StateSearchMsg); + $callback!($crate::rpc::state::StateSearchMsgLimited); + $callback!($crate::rpc::state::StateSectorExpiration); + $callback!($crate::rpc::state::StateSectorGetInfo); + $callback!($crate::rpc::state::StateSectorPartition); + $callback!($crate::rpc::state::StateSectorPreCommitInfo); + $callback!($crate::rpc::state::StateSectorPreCommitInfoV0); + $callback!($crate::rpc::state::StateVerifiedClientStatus); + $callback!($crate::rpc::state::StateVerifiedRegistryRootKey); + $callback!($crate::rpc::state::StateVerifierStatus); + $callback!($crate::rpc::state::StateVMCirculatingSupplyInternal); + $callback!($crate::rpc::state::StateWaitMsg); + $callback!($crate::rpc::state::StateWaitMsgV0); // sync vertical - $callback!(crate::rpc::sync::SyncCheckBad); - $callback!(crate::rpc::sync::SyncMarkBad); - $callback!(crate::rpc::sync::SyncState); - $callback!(crate::rpc::sync::SyncSubmitBlock); + $callback!($crate::rpc::sync::SyncCheckBad); + $callback!($crate::rpc::sync::SyncMarkBad); + $callback!($crate::rpc::sync::SyncState); + $callback!($crate::rpc::sync::SyncSubmitBlock); // wallet vertical - $callback!(crate::rpc::wallet::WalletBalance); - $callback!(crate::rpc::wallet::WalletDefaultAddress); - $callback!(crate::rpc::wallet::WalletDelete); - $callback!(crate::rpc::wallet::WalletExport); - $callback!(crate::rpc::wallet::WalletHas); - $callback!(crate::rpc::wallet::WalletImport); - $callback!(crate::rpc::wallet::WalletList); - $callback!(crate::rpc::wallet::WalletNew); - $callback!(crate::rpc::wallet::WalletSetDefault); - $callback!(crate::rpc::wallet::WalletSign); - $callback!(crate::rpc::wallet::WalletSignMessage); - $callback!(crate::rpc::wallet::WalletValidateAddress); - $callback!(crate::rpc::wallet::WalletVerify); + $callback!($crate::rpc::wallet::WalletBalance); + $callback!($crate::rpc::wallet::WalletDefaultAddress); + $callback!($crate::rpc::wallet::WalletDelete); + $callback!($crate::rpc::wallet::WalletExport); + $callback!($crate::rpc::wallet::WalletHas); + $callback!($crate::rpc::wallet::WalletImport); + $callback!($crate::rpc::wallet::WalletList); + $callback!($crate::rpc::wallet::WalletNew); + $callback!($crate::rpc::wallet::WalletSetDefault); + $callback!($crate::rpc::wallet::WalletSign); + $callback!($crate::rpc::wallet::WalletSignMessage); + $callback!($crate::rpc::wallet::WalletValidateAddress); + $callback!($crate::rpc::wallet::WalletVerify); // f3 - $callback!(crate::rpc::f3::F3GetCertificate); - $callback!(crate::rpc::f3::F3GetECPowerTable); - $callback!(crate::rpc::f3::F3GetF3PowerTable); - $callback!(crate::rpc::f3::F3IsRunning); - $callback!(crate::rpc::f3::F3GetProgress); - $callback!(crate::rpc::f3::F3GetManifest); - $callback!(crate::rpc::f3::F3ListParticipants); - $callback!(crate::rpc::f3::F3GetLatestCertificate); - $callback!(crate::rpc::f3::F3GetOrRenewParticipationTicket); - $callback!(crate::rpc::f3::F3Participate); - $callback!(crate::rpc::f3::GetHead); - $callback!(crate::rpc::f3::GetParent); - $callback!(crate::rpc::f3::GetParticipatingMinerIDs); - $callback!(crate::rpc::f3::GetPowerTable); - $callback!(crate::rpc::f3::GetTipset); - $callback!(crate::rpc::f3::GetTipsetByEpoch); - $callback!(crate::rpc::f3::Finalize); - $callback!(crate::rpc::f3::ProtectPeer); - $callback!(crate::rpc::f3::SignMessage); + $callback!($crate::rpc::f3::F3GetCertificate); + $callback!($crate::rpc::f3::F3GetECPowerTable); + $callback!($crate::rpc::f3::F3GetF3PowerTable); + $callback!($crate::rpc::f3::F3IsRunning); + $callback!($crate::rpc::f3::F3GetProgress); + $callback!($crate::rpc::f3::F3GetManifest); + $callback!($crate::rpc::f3::F3ListParticipants); + $callback!($crate::rpc::f3::F3GetLatestCertificate); + $callback!($crate::rpc::f3::F3GetOrRenewParticipationTicket); + $callback!($crate::rpc::f3::F3Participate); + $callback!($crate::rpc::f3::GetHead); + $callback!($crate::rpc::f3::GetParent); + $callback!($crate::rpc::f3::GetParticipatingMinerIDs); + $callback!($crate::rpc::f3::GetPowerTable); + $callback!($crate::rpc::f3::GetTipset); + $callback!($crate::rpc::f3::GetTipsetByEpoch); + $callback!($crate::rpc::f3::Finalize); + $callback!($crate::rpc::f3::ProtectPeer); + $callback!($crate::rpc::f3::SignMessage); // misc - $callback!(crate::rpc::misc::GetActorEventsRaw); + $callback!($crate::rpc::misc::GetActorEventsRaw); }; } pub(crate) use for_each_method; @@ -431,6 +433,14 @@ impl RPCState { } } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RpcCallSnapshot { + pub name: String, + pub params: Option, + pub response: serde_json::Value, + pub db: String, +} + #[derive(Clone)] struct PerConnection { methods: Methods, diff --git a/src/rpc/reflect/mod.rs b/src/rpc/reflect/mod.rs index 1185a135428..ead3e5e23ab 100644 --- a/src/rpc/reflect/mod.rs +++ b/src/rpc/reflect/mod.rs @@ -149,6 +149,21 @@ pub trait RpcMethodExt: RpcMethod { )), } } + + fn parse_params( + params_raw: Option>, + calling_convention: ParamStructure, + ) -> anyhow::Result { + Ok(Self::Params::parse( + params_raw + .map(|s| serde_json::from_str(s.as_ref())) + .transpose()?, + Self::PARAM_NAMES, + calling_convention, + Self::N_REQUIRED_PARAMS, + )?) + } + /// Generate a full `OpenRPC` method definition for this endpoint. fn openrpc<'de>( gen: &mut SchemaGenerator, @@ -213,17 +228,8 @@ pub trait RpcMethodExt: RpcMethod { ); module.register_async_method(Self::NAME, move |params, ctx, _extensions| async move { - let raw = params - .as_str() - .map(serde_json::from_str) - .transpose() + let params = Self::parse_params(params.as_str(), calling_convention) .map_err(|e| Error::invalid_params(e, None))?; - let params = Self::Params::parse( - raw, - Self::PARAM_NAMES, - calling_convention, - Self::N_REQUIRED_PARAMS, - )?; let ok = Self::handle(ctx, params).await?; Result::<_, jsonrpsee::types::ErrorObjectOwned>::Ok(ok.into_lotus_json()) }) diff --git a/src/shim/address.rs b/src/shim/address.rs index 3e5493f60f4..778fd1d8a46 100644 --- a/src/shim/address.rs +++ b/src/shim/address.rs @@ -49,7 +49,7 @@ thread_local! { /// /// The thread-local network variable is initialized to the value of the global network. This global /// network variable is set once when Forest has figured out which network it is using. -pub struct CurrentNetwork(); +pub struct CurrentNetwork; impl CurrentNetwork { pub fn get() -> Network { FromPrimitive::from_u8(LOCAL_NETWORK.with(|ident| ident.load(Ordering::Acquire))) diff --git a/src/tool/subcommands/api_cmd.rs b/src/tool/subcommands/api_cmd.rs index 745f2699f0b..e753952b10d 100644 --- a/src/tool/subcommands/api_cmd.rs +++ b/src/tool/subcommands/api_cmd.rs @@ -1,6 +1,8 @@ // Copyright 2019-2024 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT +mod test_snapshot; + use crate::blocks::{ElectionProof, Ticket, Tipset}; use crate::db::car::ManyCar; use crate::eth::{EthChainId as EthChainIdType, SAFE_EPOCH_DELAY}; @@ -149,6 +151,10 @@ pub enum ApiCommands { #[arg(long)] include_ignored: bool, }, + Test { + #[arg(num_args = 1.., required = true)] + files: Vec, + }, } impl ApiCommands { @@ -255,6 +261,13 @@ impl ApiCommands { println!(); } } + Self::Test { files } => { + for path in files { + print!("Running RPC test with snapshot {} ...", path.display()); + test_snapshot::run_test_from_snapshot(&path).await?; + println!(" Success"); + } + } } Ok(()) } diff --git a/src/tool/subcommands/api_cmd/test_snapshot.rs b/src/tool/subcommands/api_cmd/test_snapshot.rs new file mode 100644 index 00000000000..bcf10b83e75 --- /dev/null +++ b/src/tool/subcommands/api_cmd/test_snapshot.rs @@ -0,0 +1,160 @@ +// Copyright 2019-2024 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +use super::filter::EthEventHandler; +use crate::{ + chain::ChainStore, + chain_sync::{network_context::SyncNetworkContext, SyncConfig, SyncStage}, + db::MemoryDB, + genesis::{get_network_name_from_genesis, read_genesis_header}, + libp2p::{NetworkMessage, PeerManager}, + lotus_json::HasLotusJson, + message_pool::{MessagePool, MpoolRpcProvider}, + networks::ChainConfig, + rpc::{RPCState, RpcCallSnapshot, RpcMethod as _, RpcMethodExt as _}, + shim::address::{CurrentNetwork, Network}, + state_manager::StateManager, + KeyStore, KeyStoreConfig, +}; +use base64::prelude::*; +use openrpc_types::ParamStructure; +use parking_lot::RwLock; +use std::{path::Path, sync::Arc}; +use tokio::{sync::mpsc, task::JoinSet}; + +pub async fn run_test_from_snapshot(path: &Path) -> anyhow::Result<()> { + CurrentNetwork::set_global(Network::Testnet); + let mut run = false; + let snapshot_bytes = std::fs::read(path)?; + let snapshot_bytes = if let Ok(bytes) = zstd::decode_all(snapshot_bytes.as_slice()) { + bytes + } else { + snapshot_bytes + }; + let snapshot: RpcCallSnapshot = serde_json::from_slice(snapshot_bytes.as_slice())?; + let db_bytes = BASE64_STANDARD.decode(&snapshot.db)?; + let db = Arc::new(match MemoryDB::deserialize_from(db_bytes.as_slice()) { + Ok(db) => db, + Err(_) => MemoryDB::deserialize_from_legacy(db_bytes.as_slice())?, + }); + let chain_config = Arc::new(ChainConfig::calibnet()); + let (ctx, _, _) = ctx(db, chain_config).await?; + let params_raw = if let Some(params) = &snapshot.params { + Some(serde_json::to_string(params)?) + } else { + None + }; + + macro_rules! run_test { + ($ty:ty) => { + if snapshot.name.as_str() == <$ty>::NAME { + let params = <$ty>::parse_params(params_raw.clone(), ParamStructure::Either)?; + let result = <$ty>::handle(ctx.clone(), params).await?; + assert_eq!(snapshot.response, result.into_lotus_json_value()?); + run = true; + } + }; + } + + crate::for_each_method!(run_test); + + assert!(run, "RPC method not found"); + + Ok(()) +} + +async fn ctx( + db: Arc, + chain_config: Arc, +) -> anyhow::Result<( + Arc>, + flume::Receiver, + tokio::sync::mpsc::Receiver<()>, +)> { + let (network_send, network_rx) = flume::bounded(5); + let (tipset_send, _) = flume::bounded(5); + let sync_config = Arc::new(SyncConfig::default()); + let genesis_header = + read_genesis_header(None, chain_config.genesis_bytes(&db).await?.as_deref(), &db).await?; + + let chain_store = Arc::new( + ChainStore::new( + db.clone(), + db.clone(), + db, + chain_config.clone(), + genesis_header.clone(), + ) + .unwrap(), + ); + + let state_manager = + Arc::new(StateManager::new(chain_store.clone(), chain_config, sync_config).unwrap()); + let network_name = get_network_name_from_genesis(&genesis_header, &state_manager)?; + let message_pool = MessagePool::new( + MpoolRpcProvider::new(chain_store.publisher().clone(), state_manager.clone()), + network_name.clone(), + network_send.clone(), + Default::default(), + state_manager.chain_config().clone(), + &mut JoinSet::new(), + )?; + + let peer_manager = Arc::new(PeerManager::default()); + let sync_network_context = + SyncNetworkContext::new(network_send, peer_manager, state_manager.blockstore_owned()); + let (shutdown, shutdown_recv) = mpsc::channel(1); + let rpc_state = Arc::new(RPCState { + state_manager, + keystore: Arc::new(tokio::sync::RwLock::new(KeyStore::new( + KeyStoreConfig::Memory, + )?)), + mpool: Arc::new(message_pool), + bad_blocks: Default::default(), + sync_state: Arc::new(RwLock::new(Default::default())), + eth_event_handler: Arc::new(EthEventHandler::new()), + sync_network_context, + network_name, + start_time: chrono::Utc::now(), + shutdown, + tipset_send, + }); + rpc_state.sync_state.write().set_stage(SyncStage::Idle); + Ok((rpc_state, network_rx, shutdown_recv)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::daemon::db_util::download_to; + use itertools::Itertools as _; + use url::Url; + + #[tokio::test] + async fn rpc_regression_tests() -> anyhow::Result<()> { + let urls = include_str!("test_snapshots.txt") + .trim() + .split("\n") + .filter_map(|n| { + Url::parse( + format!( + "https://forest-snapshots.fra1.cdn.digitaloceanspaces.com/rpc_test/{n}" + ) + .as_str(), + ) + .ok() + }) + .collect_vec(); + for url in urls { + print!("Testing {url} ..."); + let tmp_dir = tempfile::tempdir()?; + let tmp = tempfile::NamedTempFile::new_in(&tmp_dir)?.into_temp_path(); + println!("start downloading at {}", tmp.display()); + download_to(&url, &tmp).await?; + println!("done downloading {}", tmp.display()); + run_test_from_snapshot(&tmp).await?; + println!(" succeeded."); + } + Ok(()) + } +} diff --git a/src/tool/subcommands/api_cmd/test_snapshots.txt b/src/tool/subcommands/api_cmd/test_snapshots.txt new file mode 100644 index 00000000000..e2da66fd254 --- /dev/null +++ b/src/tool/subcommands/api_cmd/test_snapshots.txt @@ -0,0 +1,2 @@ +f3_gettipsetbyepoch_1730952732441851.json.zst +filecoin_statelistactors_1730953255032189.json.zst