From 1b4287bd3e42e26e5096d0ac4ff22f9ea41a46c1 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Mon, 29 Aug 2022 14:12:15 -0300 Subject: [PATCH 01/25] add grpc mempool test research --- zebra-chain/src/transaction/hash.rs | 2 +- zebra-node-services/src/mempool/gossip.rs | 10 +++- zebra-test/src/vectors.rs | 13 +++++ zebrad/src/components/mempool.rs | 10 +++- .../common/lightwalletd/wallet_grpc_test.rs | 50 +++++++++++++++++-- 5 files changed, 77 insertions(+), 8 deletions(-) diff --git a/zebra-chain/src/transaction/hash.rs b/zebra-chain/src/transaction/hash.rs index 68a0fb78694..51827aa2e48 100644 --- a/zebra-chain/src/transaction/hash.rs +++ b/zebra-chain/src/transaction/hash.rs @@ -109,7 +109,7 @@ impl Hash { /// /// Zebra displays transaction and block hashes in big-endian byte-order, /// following the u256 convention set by Bitcoin and zcashd. - fn bytes_in_display_order(&self) -> [u8; 32] { + pub fn bytes_in_display_order(&self) -> [u8; 32] { let mut reversed_bytes = self.0; reversed_bytes.reverse(); reversed_bytes diff --git a/zebra-node-services/src/mempool/gossip.rs b/zebra-node-services/src/mempool/gossip.rs index 2e344893a57..51b48968e12 100644 --- a/zebra-node-services/src/mempool/gossip.rs +++ b/zebra-node-services/src/mempool/gossip.rs @@ -3,7 +3,7 @@ use zebra_chain::transaction::{UnminedTx, UnminedTxId}; /// A gossiped transaction, which can be the transaction itself or just its ID. -#[derive(Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq)] pub enum Gossip { /// Just the ID of an unmined transaction. Id(UnminedTxId), @@ -20,6 +20,14 @@ impl Gossip { Gossip::Tx(tx) => tx.id, } } + + /// Return the [`UnminedTx`] of a gossiped transaction, if we have it. + pub fn tx(&self) -> Option { + match self { + Gossip::Id(_) => None, + Gossip::Tx(tx) => Some(tx.clone()), + } + } } impl From for Gossip { diff --git a/zebra-test/src/vectors.rs b/zebra-test/src/vectors.rs index cfcd48e811b..9c0640de96b 100644 --- a/zebra-test/src/vectors.rs +++ b/zebra-test/src/vectors.rs @@ -188,4 +188,17 @@ lazy_static! { /// pub static ref ZIP243_3: Vec = >::from_hex("0400008085202f8901a8c685478265f4c14dada651969c45a65e1aeb8cd6791f2f5bb6a1d9952104d9010000006b483045022100a61e5d557568c2ddc1d9b03a7173c6ce7c996c4daecab007ac8f34bee01e6b9702204d38fdc0bcf2728a69fde78462a10fb45a9baa27873e6a5fc45fb5c76764202a01210365ffea3efa3908918a8b8627724af852fc9b86d7375b103ab0543cf418bcaa7ffeffffff02005a6202000000001976a9148132712c3ff19f3a151234616777420a6d7ef22688ac8b959800000000001976a9145453e4698f02a38abdaa521cd1ff2dee6fac187188ac29b0040048b004000000000000000000000000") .expect("Transaction bytes are in valid hex representation"); + + /// https://explorer.zcha.in/transactions/91c61ef8770986f27651a73fe8ea35607c6ebe91201894f3cd7c6a97359a2751 but with the + /// expiration height modified to hex 00093d (4000000 decimal in little endian). + /// v5 transaction with transparent inputs and outpouts, no shielded. + pub static ref DUMMY_TX2: Vec = >::from_hex("050000800a27a726b4d0d6c23fbe190000093d0001ce6336775066a26ef5b38e91c73c907d255da7488d40cb0682284aeac84c8af4000000006b483045022100eaf7fdcaff8f0716385cefb997dd1b64a6d2cf4cdfc25b04352376ea610e937702202c17f82f3ce506bdcc3e5d74a103305cfdd838f75095495dc5e4995ecab7ae37012103f68dd6a595fa8381470462eef99803eccbc24aea0cdda5a88c2010ab10943db8feffffff0258252800000000001976a9141cb70d902612db048aa9b0462cea068f81aed9cc88ac491a2400000000001976a914fe4637b0103af2339fb69105e2f17a6d37c282a788ac000000") + .expect("Transaction bytes are in valid hex representation"); + + /// https://explorer.zcha.in/transactions/babaf52aaafbee607bce231178be756121e839c4c99e2a15eedad84b1288a396 but with the + /// expiration height modified to hex 00093d (4000000 decimal in little endian). + /// v5 transaction, no transparent data, only shielded sapling. + pub static ref DUMMY_TX3: Vec = >::from_hex("050000800a27a726b4d0d6c20000000000093d00000001918291c84ae675708527da9b7f5e836df9df29b126f9f0e0afa01c7b794e01267f0d85b3a51a242883f4b3786b6ecddf67d9c935c4cce59f038ce13451bbc259cd14a2edc26a42478e15680dbafc9bb5ea6f05af911150d72c8abf3b973658ed02dbdf3941aae0b511102a718851429fffab6b761b2ff03c5182ff091b9363d1c4b2eae6fd2e4ade1b7f2126a4ec6167c76828fc1006bedeafcf6ba18eebdaf166b8a50eb85ed95abae3f8425e8c4859e57d2335e66be322046705f7db6fe01669fc6cd051be33ff2dbcd6ac1ff3791f474380dec36b56f9364b1e748bde301b2c1824841e3516636a9986e8c832a084b363ed92de75858f9014b7f7b987449bfea635007e949a9f2d74f64d393e302876764172ee90862df1fe743dbe5242b8d6c34882dbc240b95586bc79001356fb08952d3d336355dbbc10a92bd189181f469aae920269c6efaa7b2a1ecdcfd9efe978ed50af0b26c130436469cf13812071c557ef140b3d1028e200662f74a4e9ece1510282a473eb7820cbdd2af8a10dfca03c567d093d209334b935c35958bc17bce51dfda9c6231fea4fcac2720b8096b263c40e0fbe9e4c30abe0e814307a3c40dec9b8b508ccbf907579fbf9da953ab2ea634918ee350905bed666b67bfb74a808a927b74c5ccddaa08d4215e501278b0deb1955dd073bc9fe4688567ed7880b00322f6092e8cb794cefafd572c8b61fa89bee8ead90aa02b667c9b29bfd0a4111db62518795c7f6f2f940f5f6e2610c50d0ced13a070ed08eb781fe9102572aa46bd0fc2ff8b09a5bc29b8cf755f8ab62ea09bdc23dcb6bf08fe99b4031aeaec61bbc55add8cac4d9e8f35395ae0043199831a4bffac91988b711a9abb5c135c907c40fcd36917eceeedfcc4cd172bce8bbe5e05496cb8c2d45c5e779d20f2ff93a7deffcb8f8fac15e83e02924dee7a19b0e5ea9cc3a7397b3e53572d3d55b73d8029651f30a118ac722669db7e8d1effd70bf5acf69bca10af12f7c1b44cdabf7c013bfc8bc16b891ce94548335258bad5d2ad189d413141dff8342639cd9b7f199cb7c81e7c4900c35e75f934fa407a7c3e2c4cf5d6bb0eee3449ece95efce0a4fbb4c1b97f209398ba4c32092ade4f435b82e50038b25c503afc00f9f9a772bf7f04cb8aba32704efe0a10ce1bd2c7a669da3dec4cee5aa7f4a7dcdc367b4f62275eb08bd6a32457e5f28ddd56d6518049f60754b0b37fc6e3d714dfbc63cadceb643705be09893af0c26ce2dd742fab7a4c462500a6aa3bd82b9b01cd2c2935c31b16b195adcebc981b6413009450db84005b49a4da62893ee3a36cce886eee2796da3f8767760abf40d5f93bc560f06696e670b6edaa93b1e25c228684baebfaf538d1da43a180cd491d330a4046d7b6a51dd15c20cf5a414aa59829288003741af558be0079740a94059ed6cdce6c133bfbaa700751176e71743657d0b077f6ca12e3cc062e5d613ecb9c36d63129e7c4a6a1188c3dbb9e954126b15592b06d0d3c1c7b232866a4cb3398fc11496180df5a928091d056a216fa577e230824de875ad77902d10eebd6c6e755f59fc32b3ee892cb213e4c81acd4312e7f0e486b1446cea00b2caa87565334d638267c2810f02bca09b799d6a01940d17c48fc2afff18e25984d8f7ee9b102c88e610e61f9695c2475d3c97a7f9a32f0af730f4848ea855fa2f963dd633d4c0edecfde477503940d86caf449185b15efb278e08c428135848fd2bc312c8eb51f001b5e5e3f61fb5ac8d57d92e3c604697beb53368641101b382fdb0edf1cac0053c5ea1f5732e557060f74c5d14ef8e2c906938d15bbf5423e4226f510a938d7be7769275f6461c2ef65db98b18cc62a38be82ce997265e1b585099903be8308ac0d5bd66ff210658263dc5fb2267b7a8d2b3dd2fc25c25d18a2b5ab9b5e9a43ab7c900f2a63e2950c711b33eedfcb725d133e67259a5f867a6f69eed537e48edae77cf3d0b0c7e60f61f6c1c418318d2508fdce67039892db855561506ec534a2120de338f6d165dcf8f0e8e28c1a529395c2b068129c4b49b31f0328de95475c83bb625193203d20c9df9a762f3e7229c0a6a31ec703dc772729248cd9fd979839d1e67b41ed06083574dca9070aedf183da902a71c64021af018a596be977f3fb55d797b80a9a4ef37f9446aa522db786db700162085ad4200a4275382301c7fb1882159bdec2867942951fde2549f0e8d2bf2f70461ec7c6de9be52938e4bebca393bedddf9e8030000000000009f600fe22c07fd60d49559791edcf74ecef542f800ab0adcfeda49684655044c990dc78dab58805cbe2d2b088c3946383a5f145a42501fde7c42b44b14d0126547e8375a067c6c5877f8faebb131e5dead4946c3200080706698f5f1695b986a77e7e5ba44bf64e8cbff66cd63449f67304542c403c367940d6e8c81f8e977990ec7f30ce82b238ee24e4a0e2e5472a0a4342d8d3227fd47af48c5bfb0f28dc97fb9baa88fbbbb8630773fa023f4db2f92f413a8babd112e3a7d11321c3135296f7851f3be628a183d42627f859dfe2d9ccf66a6f5749fbce5bb9c87353950ae4031115744c2393f38f88349c668028bfaf0b7bd3a5088f48b500cc905bb443cc60d80a1eb21b6da0202f673a570ec4940b162d3c67c04a4e796fe6aef0ca4089502f0d129707ada789abeeba5c09f572f32489498d588a5f2a02c03b64bc9ab9558c38236b32f7312977530547d03e5a2f6374cabe7809aece49bd3b5c40890a7e276e66d7d3672ca3d610f83e1be09ffde1a9debabfdecb1a5dec7b5d9d3a505949ba33e5c5e7c06fd32c162bc82a6398e1adbfeff7f549d7dfc8f510a21fcfd7baa2e4a295afcc064c5189ebeef37b580d01ae34d1fcd571f2c89c8f566c4cc5abedb4653ba52edc6a9d480092f6765bbcba9acebc9f11e3bfbe2afd22fd9a9d1dd1afec6a0777324f325b64fe34cbf4f57444d70cd20e53466aab657dcbfc37309ce0ff4df6cd120ed9ea564be50b94355ee97bfd403aba15466d2475df2827947185dfb1c7c2eacbbf11622c81e86400f89bced44ffd6f84f3e18645aab168a3b5920ace730cfb0a1b939b00815c2a97b78317fd22e7e17b5630282b84bebe72d23b634c7d1501dde3a6d92147e94d5227937b5975341d2f9f4ae2b15ffcf6690a8bef76afe0ff4a157a08adc727fd69b78f896b2011607cd8229e38d3e813ed4c5728d8c3b3431738d5860b3692def17096122f7771b545764a83ea2512973187b565e4ee96838b3295dd33d1cf67bb759510e8fffdbdb73cde9c15b0900") + .expect("Transaction bytes are in valid hex representation"); + } diff --git a/zebrad/src/components/mempool.rs b/zebrad/src/components/mempool.rs index 844624bb92e..bc0fe7c8a0f 100644 --- a/zebrad/src/components/mempool.rs +++ b/zebrad/src/components/mempool.rs @@ -413,7 +413,15 @@ impl Service for Mempool { .into_iter() .map(|gossiped_tx| -> Result<(), MempoolError> { storage.should_download_or_verify(gossiped_tx.id())?; - tx_downloads.download_if_needed_and_verify(gossiped_tx)?; + tx_downloads.download_if_needed_and_verify(gossiped_tx.clone())?; + if let Some(tx) = gossiped_tx.tx() { + storage.insert( + zebra_chain::transaction::VerifiedUnminedTx::new( + tx, + zebra_chain::amount::Amount::zero(), + ), + )?; + } Ok(()) }) .map(|result| result.map_err(BoxError::from)) diff --git a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs index 6669c2b46bb..47b00f65031 100644 --- a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs +++ b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs @@ -37,7 +37,8 @@ use zebra_chain::{ block::Block, parameters::Network, parameters::NetworkUpgrade::{self, Canopy}, - serialization::ZcashDeserializeInto, + serialization::{ZcashDeserialize, ZcashDeserializeInto, ZcashSerialize}, + transaction::Transaction, }; use zebra_network::constants::USER_AGENT; @@ -47,7 +48,7 @@ use crate::common::{ lightwalletd::{ wallet_grpc::{ connect_to_lightwalletd, spawn_lightwalletd_with_rpc_server, Address, AddressList, - BlockId, BlockRange, ChainSpec, Empty, GetAddressUtxosArg, + BlockId, BlockRange, ChainSpec, Empty, Exclude, GetAddressUtxosArg, RawTransaction, TransparentAddressBlockFilter, TxFilter, }, zebra_skip_lightwalletd_tests, @@ -110,7 +111,7 @@ pub async fn run() -> Result<()> { )?; // Give lightwalletd a few seconds to open its grpc port before connecting to it - tokio::time::sleep(std::time::Duration::from_secs(10)).await; + tokio::time::sleep(std::time::Duration::from_secs(60)).await; tracing::info!( ?lightwalletd_rpc_port, @@ -277,8 +278,6 @@ pub async fn run() -> Result<()> { balance_zf.value_zat + balance_mg.value_zat ); - // TODO: Create call and checks for `GetMempoolTx` and `GetMempoolTxStream`? - let sapling_treestate_init_height = sapling_activation_height + 1; // Call `GetTreeState`. @@ -344,5 +343,46 @@ pub async fn run() -> Result<()> { // Make sure the subversion field is zebra the user agent assert_eq!(lightd_info.zcashd_subversion, USER_AGENT); + // Transactions in mempool + + let transaction = Transaction::zcash_deserialize(&zebra_test::vectors::DUMMY_TX3[..]).unwrap(); + let transaction_bytes = transaction.zcash_serialize_to_vec().unwrap(); + + // Send transaction by calling `SendTransaction` + let request1 = RawTransaction { + data: transaction_bytes, + height: -1, + }; + let _ = rpc_client.send_transaction(request1).await?.into_inner(); + + // Wait a bit to query the mempool + tokio::time::sleep(std::time::Duration::from_secs(10)).await; + + // Call `GetMempoolTx` and get a stream of transactions + let mut transactions_stream = rpc_client + .get_mempool_tx(Exclude { txid: vec![] }) + .await? + .into_inner(); + + // Make sure our transaction, and only our transaction was inserted to the mempool. + let mut counter = 0; + while let Some(tx) = transactions_stream.message().await? { + assert_eq!(tx.hash, transaction.hash().bytes_in_display_order()); + counter += 1; + } + + assert_eq!(counter, 1); + + // Get the mempool by calling `GetMempoolStream`. + let mut transaction_stream = rpc_client.get_mempool_stream(Empty {}).await?.into_inner(); + + let mut counter = 0; + while let Some(_tx) = transaction_stream.message().await? { + counter += 1; + } + + // This one is not working. + assert_eq!(counter, 0); + Ok(()) } From 67dc9a85aa7db31b5ed12d091c76a6b472641b6f Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Mon, 29 Aug 2022 19:00:43 -0300 Subject: [PATCH 02/25] add a config flag for mempool injection of transactions in test --- zebra-test/src/vectors.rs | 2 +- zebrad/src/components/mempool.rs | 23 ++++++++++++++++------- zebrad/src/components/mempool/config.rs | 4 ++++ zebrad/tests/common/launch.rs | 1 + 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/zebra-test/src/vectors.rs b/zebra-test/src/vectors.rs index 9c0640de96b..3973028b2a1 100644 --- a/zebra-test/src/vectors.rs +++ b/zebra-test/src/vectors.rs @@ -191,7 +191,7 @@ lazy_static! { /// https://explorer.zcha.in/transactions/91c61ef8770986f27651a73fe8ea35607c6ebe91201894f3cd7c6a97359a2751 but with the /// expiration height modified to hex 00093d (4000000 decimal in little endian). - /// v5 transaction with transparent inputs and outpouts, no shielded. + /// v5 transaction with transparent inputs and outputs, no shielded. pub static ref DUMMY_TX2: Vec = >::from_hex("050000800a27a726b4d0d6c23fbe190000093d0001ce6336775066a26ef5b38e91c73c907d255da7488d40cb0682284aeac84c8af4000000006b483045022100eaf7fdcaff8f0716385cefb997dd1b64a6d2cf4cdfc25b04352376ea610e937702202c17f82f3ce506bdcc3e5d74a103305cfdd838f75095495dc5e4995ecab7ae37012103f68dd6a595fa8381470462eef99803eccbc24aea0cdda5a88c2010ab10943db8feffffff0258252800000000001976a9141cb70d902612db048aa9b0462cea068f81aed9cc88ac491a2400000000001976a914fe4637b0103af2339fb69105e2f17a6d37c282a788ac000000") .expect("Transaction bytes are in valid hex representation"); diff --git a/zebrad/src/components/mempool.rs b/zebrad/src/components/mempool.rs index bc0fe7c8a0f..f1fc96b526e 100644 --- a/zebrad/src/components/mempool.rs +++ b/zebrad/src/components/mempool.rs @@ -280,6 +280,11 @@ impl Mempool { .copied() .collect() } + + /// TBA + pub fn skip_full_validation(&self) -> bool { + self.config.skip_full_validation + } } impl Service for Mempool { @@ -384,6 +389,8 @@ impl Service for Mempool { /// and will cause callers to disconnect from the remote peer. #[instrument(name = "mempool", skip(self, req))] fn call(&mut self, req: Request) -> Self::Future { + let skip_full_validation = self.skip_full_validation(); + match &mut self.active_state { ActiveState::Enabled { storage, @@ -414,13 +421,15 @@ impl Service for Mempool { .map(|gossiped_tx| -> Result<(), MempoolError> { storage.should_download_or_verify(gossiped_tx.id())?; tx_downloads.download_if_needed_and_verify(gossiped_tx.clone())?; - if let Some(tx) = gossiped_tx.tx() { - storage.insert( - zebra_chain::transaction::VerifiedUnminedTx::new( - tx, - zebra_chain::amount::Amount::zero(), - ), - )?; + if skip_full_validation { + if let Some(tx) = gossiped_tx.tx() { + storage.insert( + zebra_chain::transaction::VerifiedUnminedTx::new( + tx, + zebra_chain::amount::Amount::zero(), + ), + )?; + } } Ok(()) }) diff --git a/zebrad/src/components/mempool/config.rs b/zebrad/src/components/mempool/config.rs index 2b4f20f6a1b..25d4a7fce97 100644 --- a/zebrad/src/components/mempool/config.rs +++ b/zebrad/src/components/mempool/config.rs @@ -41,6 +41,9 @@ pub struct Config { // - allow the mempool to be enabled before the genesis block is committed? // we could replace `Option` with an enum that has an `AlwaysEnable` variant pub debug_enable_at_height: Option, + + /// TBA + pub skip_full_validation: bool, } impl Default for Config { @@ -59,6 +62,7 @@ impl Default for Config { eviction_memory_time: Duration::from_secs(60 * 60), debug_enable_at_height: None, + skip_full_validation: false, } } } diff --git a/zebrad/tests/common/launch.rs b/zebrad/tests/common/launch.rs index 6231200e199..49a674a5b43 100644 --- a/zebrad/tests/common/launch.rs +++ b/zebrad/tests/common/launch.rs @@ -224,6 +224,7 @@ pub fn spawn_zebrad_for_rpc_without_initial_peers Date: Tue, 30 Aug 2022 11:29:26 +1000 Subject: [PATCH 03/25] Only copy the inner state directory in the send transactions test --- zebrad/tests/common/cached_state.rs | 24 +++++++++++++++---- .../lightwalletd/send_transaction_test.rs | 4 ++-- zebrad/tests/common/sync.rs | 4 ++-- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/zebrad/tests/common/cached_state.rs b/zebrad/tests/common/cached_state.rs index 3323c3ac716..a2e97dbd32d 100644 --- a/zebrad/tests/common/cached_state.rs +++ b/zebrad/tests/common/cached_state.rs @@ -63,21 +63,37 @@ pub async fn load_tip_height_from_state_directory( Ok(chain_tip_height) } -/// Recursively copy a chain state directory into a new temporary directory. -pub async fn copy_state_directory(source: impl AsRef) -> Result { +/// Recursively copy a chain state database directory into a new temporary directory. +pub async fn copy_state_directory(network: Network, source: impl AsRef) -> Result { + // Copy the database files for this state and network, excluding testnet and other state versions let source = source.as_ref(); + let state_config = zebra_state::Config { + cache_dir: source.into(), + ..Default::default() + }; + let source_net_dir = state_config.db_path(network); + let source_net_dir = source_net_dir.as_path(); + let state_suffix = source_net_dir + .strip_prefix(source) + .expect("db_path() is a subdirectory"); + let destination = testdir()?; + let destination_net_dir = destination.path().join(state_suffix); tracing::info!( ?source, + ?source_net_dir, + ?state_suffix, ?destination, + ?destination_net_dir, "copying cached state files (this may take some time)...", ); - let mut remaining_directories = vec![PathBuf::from(source)]; + let mut remaining_directories = vec![PathBuf::from(source_net_dir)]; while let Some(directory) = remaining_directories.pop() { - let sub_directories = copy_directory(&directory, source, destination.as_ref()).await?; + let sub_directories = + copy_directory(&directory, source_net_dir, destination_net_dir.as_ref()).await?; remaining_directories.extend(sub_directories); } diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index db638204c67..ba93f1c87b7 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -39,7 +39,7 @@ use crate::common::{ zebra_skip_lightwalletd_tests, LightwalletdTestType::*, }, - sync::perform_full_sync_starting_from, + sync::copy_state_and_perform_full_sync, }; /// The test entry point. @@ -160,7 +160,7 @@ async fn load_transactions_from_a_future_block( ); let full_sync_path = - perform_full_sync_starting_from(network, zebrad_state_path.as_ref()).await?; + copy_state_and_perform_full_sync(network, zebrad_state_path.as_ref()).await?; tracing::info!(?full_sync_path, "loading transactions..."); diff --git a/zebrad/tests/common/sync.rs b/zebrad/tests/common/sync.rs index 743a830eb92..871a1815585 100644 --- a/zebrad/tests/common/sync.rs +++ b/zebrad/tests/common/sync.rs @@ -300,11 +300,11 @@ pub fn sync_until( /// is returned afterwards, containing the fully synchronized chain state. #[allow(dead_code)] #[tracing::instrument] -pub async fn perform_full_sync_starting_from( +pub async fn copy_state_and_perform_full_sync( network: Network, partial_sync_path: &Path, ) -> Result { - let fully_synced_path = copy_state_directory(&partial_sync_path).await?; + let fully_synced_path = copy_state_directory(network, &partial_sync_path).await?; sync_until( Height::MAX, From c19473ceea68f5bcc3ee80d3fac27751eddc25c4 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 30 Aug 2022 12:48:32 +1000 Subject: [PATCH 04/25] Preload Zcash parameters in some transaction verification tests --- zebrad/tests/acceptance.rs | 8 ++++++-- zebrad/tests/common/launch.rs | 3 +++ zebrad/tests/common/lightwalletd/send_transaction_test.rs | 3 ++- zebrad/tests/common/lightwalletd/wallet_grpc_test.rs | 4 +++- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index 47058a2e254..54ebc22f491 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -1819,8 +1819,12 @@ async fn fully_synced_rpc_test() -> Result<()> { let network = Network::Mainnet; - let (_zebrad, zebra_rpc_address) = - spawn_zebrad_for_rpc_without_initial_peers(network, cached_state_path.unwrap(), test_type)?; + let (_zebrad, zebra_rpc_address) = spawn_zebrad_for_rpc_without_initial_peers( + network, + cached_state_path.unwrap(), + test_type, + true, + )?; // Make a getblock test that works only on synced node (high block number). // The block is before the mandatory checkpoint, so the checkpoint cached state can be used diff --git a/zebrad/tests/common/launch.rs b/zebrad/tests/common/launch.rs index 49a674a5b43..bb044f74269 100644 --- a/zebrad/tests/common/launch.rs +++ b/zebrad/tests/common/launch.rs @@ -214,6 +214,7 @@ pub fn spawn_zebrad_for_rpc_without_initial_peers Result<(TestChild

, SocketAddr)> { // This is what we recommend our users configure. let mut config = random_known_rpc_port_config(true) @@ -224,6 +225,8 @@ pub fn spawn_zebrad_for_rpc_without_initial_peers Result<()> { "got transactions to send", ); + // TODO: change debug_skip_parameter_preload to true if we do the mempool test in the wallet gRPC test let (_zebrad, zebra_rpc_address) = - spawn_zebrad_for_rpc_without_initial_peers(Network::Mainnet, zebrad_state_path, test_type)?; + spawn_zebrad_for_rpc_without_initial_peers(Network::Mainnet, zebrad_state_path, test_type, false)?; tracing::info!( ?zebra_rpc_address, diff --git a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs index 47b00f65031..687782be7d0 100644 --- a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs +++ b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs @@ -94,8 +94,10 @@ pub async fn run() -> Result<()> { ); // Launch zebra using a predefined zebrad state path + // + // TODO: change debug_skip_parameter_preload to true if we do the mempool test in the send transaction test let (_zebrad, zebra_rpc_address) = - spawn_zebrad_for_rpc_without_initial_peers(network, zebrad_state_path.unwrap(), test_type)?; + spawn_zebrad_for_rpc_without_initial_peers(network, zebrad_state_path.unwrap(), test_type, false)?; tracing::info!( ?zebra_rpc_address, From a690fca5aa520eedfa78d9b52759e8307acde84d Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 30 Aug 2022 12:51:34 +1000 Subject: [PATCH 05/25] Add a block and transaction Hash method to convert from display order bytes --- zebra-chain/src/block/hash.rs | 13 ++++++++++++- zebra-chain/src/transaction/hash.rs | 11 +++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/zebra-chain/src/block/hash.rs b/zebra-chain/src/block/hash.rs index ccf3217ea6f..741ccc724f1 100644 --- a/zebra-chain/src/block/hash.rs +++ b/zebra-chain/src/block/hash.rs @@ -29,11 +29,22 @@ impl Hash { /// /// Zebra displays transaction and block hashes in big-endian byte-order, /// following the u256 convention set by Bitcoin and zcashd. - fn bytes_in_display_order(&self) -> [u8; 32] { + pub fn bytes_in_display_order(&self) -> [u8; 32] { let mut reversed_bytes = self.0; reversed_bytes.reverse(); reversed_bytes } + + /// Convert bytes in big-endian byte-order into a [`Hash`]. + /// + /// Zebra displays transaction and block hashes in big-endian byte-order, + /// following the u256 convention set by Bitcoin and zcashd. + pub fn from_bytes_in_display_order(bytes_in_display_order: &[u8; 32]) -> Hash { + let mut internal_byte_order = *bytes_in_display_order; + internal_byte_order.reverse(); + + Hash(internal_byte_order) + } } impl fmt::Display for Hash { diff --git a/zebra-chain/src/transaction/hash.rs b/zebra-chain/src/transaction/hash.rs index 51827aa2e48..b6ad5efd9cf 100644 --- a/zebra-chain/src/transaction/hash.rs +++ b/zebra-chain/src/transaction/hash.rs @@ -114,6 +114,17 @@ impl Hash { reversed_bytes.reverse(); reversed_bytes } + + /// Convert bytes in big-endian byte-order into a [`Hash`]. + /// + /// Zebra displays transaction and block hashes in big-endian byte-order, + /// following the u256 convention set by Bitcoin and zcashd. + pub fn from_bytes_in_display_order(bytes_in_display_order: &[u8; 32]) -> Hash { + let mut internal_byte_order = *bytes_in_display_order; + internal_byte_order.reverse(); + + Hash(internal_byte_order) + } } impl ToHex for &Hash { From b8d9abff2bf0cdc8baa71feac42e6e89e3d44bf3 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 30 Aug 2022 12:58:14 +1000 Subject: [PATCH 06/25] Update test coverage docs --- zebrad/tests/common/lightwalletd/wallet_grpc_test.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs index 687782be7d0..cdc47245b13 100644 --- a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs +++ b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs @@ -13,14 +13,15 @@ //! - `GetBlockRange`: Covered. //! //! - `GetTransaction`: Covered. -//! - `SendTransaction`: Not covered and it will never be, it has its own test. +//! - `SendTransaction`: Covered by the send_transaction_test. //! //! - `GetTaddressTxids`: Covered. //! - `GetTaddressBalance`: Covered. //! - `GetTaddressBalanceStream`: Covered. //! -//! - `GetMempoolTx`: Not covered. -//! - `GetMempoolStream`: Not covered. +//! - `GetMempoolTx`: Covered by the send_transaction_test. +//! - `GetMempoolStream`: Covered by the send_transaction_test, +//! currently disabled by `lightwalletd`. //! //! - `GetTreeState`: Covered. //! @@ -28,6 +29,7 @@ //! - `GetAddressUtxosStream`: Covered. //! //! - `GetLightdInfo`: Covered. +//! //! - `Ping`: Not covered and it will never be. `Ping` is only used for testing //! purposes. From cc57ec45e9494c8a10e11e42469b51592173105e Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 30 Aug 2022 12:51:01 +1000 Subject: [PATCH 07/25] Add debugging output for mempool transaction verification --- zebra-consensus/src/transaction.rs | 19 ++++- zebra-rpc/src/methods.rs | 2 + zebrad/src/components/mempool.rs | 74 ++++++++++++++++--- zebrad/src/components/mempool/downloads.rs | 19 ++++- zebrad/src/components/mempool/storage.rs | 20 +++++ .../mempool/storage/verified_set.rs | 2 +- 6 files changed, 119 insertions(+), 17 deletions(-) diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index ef95ae28fa6..7f13bf3ee80 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -304,8 +304,10 @@ where let tx_id = req.tx_id(); let span = tracing::debug_span!("tx", ?tx_id); + let is_mempool = req.is_mempool(); + async move { - tracing::trace!(?req); + tracing::trace!(?tx_id, ?req, "got tx verify request"); // Do basic checks first if let Some(block_time) = req.block_time() { @@ -315,7 +317,7 @@ where check::has_inputs_and_outputs(&tx)?; check::has_enough_orchard_flags(&tx)?; - if req.is_mempool() && tx.is_coinbase() { + if is_mempool && tx.is_coinbase() { return Err(TransactionError::CoinbaseInMempool); } if tx.is_coinbase() { @@ -344,6 +346,8 @@ where check::spend_conflicts(&tx)?; + tracing::trace!(?tx_id, "passed quick checks"); + // "The consensus rules applied to valueBalance, vShieldedOutput, and bindingSig // in non-coinbase transactions MUST also be applied to coinbase transactions." // @@ -360,6 +364,9 @@ where let cached_ffi_transaction = Arc::new(CachedFfiTransaction::new(tx.clone(), spent_outputs)); + + tracing::trace!(?tx_id, "got state UTXOs"); + let async_checks = match tx.as_ref() { Transaction::V1 { .. } | Transaction::V2 { .. } | Transaction::V3 { .. } => { tracing::debug!(?tx, "got transaction with wrong version"); @@ -391,10 +398,14 @@ where )?, }; + tracing::trace!(?tx_id, "awaiting async checks..."); + // If the Groth16 parameter download hangs, // Zebra will timeout here, waiting for the async checks. async_checks.check().await?; + tracing::trace!(?tx_id, "finished async checks"); + // Get the `value_balance` to calculate the transaction fee. let value_balance = tx.value_balance(&spent_utxos); @@ -429,6 +440,10 @@ where Ok(rsp) } + .inspect(move |result| { + // Hide the transaction data to avoid filling the logs + tracing::trace!(?tx_id, result = ?result.as_ref().map(|_tx| ()), "got tx verify result"); + }) .instrument(span) .boxed() } diff --git a/zebra-rpc/src/methods.rs b/zebra-rpc/src/methods.rs index e9ce2c889ae..6da888f16f2 100644 --- a/zebra-rpc/src/methods.rs +++ b/zebra-rpc/src/methods.rs @@ -506,6 +506,8 @@ where "mempool service returned more results than expected" ); + tracing::info!("sent transaction to mempool: {:?}", &queue_results[0]); + match &queue_results[0] { Ok(()) => Ok(SentTransactionHash(transaction_hash)), Err(error) => Err(Error { diff --git a/zebrad/src/components/mempool.rs b/zebrad/src/components/mempool.rs index f1fc96b526e..8279050ef28 100644 --- a/zebrad/src/components/mempool.rs +++ b/zebrad/src/components/mempool.rs @@ -296,6 +296,8 @@ impl Service for Mempool { fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { let is_state_changed = self.update_state(); + tracing::info!(is_enabled = ?self.is_enabled(), ?is_state_changed, "started polling the mempool..."); + // When the mempool is disabled we still return that the service is ready. // Otherwise, callers could block waiting for the mempool to be enabled. if !self.is_enabled() { @@ -343,21 +345,31 @@ impl Service for Mempool { while let Poll::Ready(Some(r)) = tx_downloads.as_mut().poll_next(cx) { match r { Ok(tx) => { - if let Ok(inserted_id) = storage.insert(tx.clone()) { + let insert_result = storage.insert(tx.clone()); + + tracing::info!( + ?insert_result, + "got Ok(_) transaction verify, tried to store", + ); + + if let Ok(inserted_id) = insert_result { // Save transaction ids that we will send to peers send_to_peers_ids.insert(inserted_id); } } - Err((txid, e)) => { - metrics::counter!("mempool.failed.verify.tasks.total", 1, "reason" => e.to_string()); - storage.reject_if_needed(txid, e); - // TODO: should we also log the result? + Err((txid, error)) => { + tracing::info!(?txid, ?error, "mempool transaction failed to verify"); + + metrics::counter!("mempool.failed.verify.tasks.total", 1, "reason" => error.to_string()); + storage.reject_if_needed(txid, error); } }; } // Handle best chain tip changes if let Some(TipAction::Grow { block }) = tip_action { + tracing::info!(block_height = ?block.height, "handling blocks added to tip"); + // Cancel downloads/verifications/storage of transactions // with the same mined IDs as recently mined transactions. let mined_ids = block.transaction_hashes.iter().cloned().collect(); @@ -372,10 +384,19 @@ impl Service for Mempool { // Remove transactions that are expired from the peers list send_to_peers_ids = Self::remove_expired_from_peer_list(&send_to_peers_ids, &expired_transactions); + + if !expired_transactions.is_empty() { + tracing::info!( + ?expired_transactions, + "removed expired transactions from the mempool", + ); + } } // Send transactions that were not rejected nor expired to peers if !send_to_peers_ids.is_empty() { + tracing::info!(?send_to_peers_ids, "sending new transactions to peers"); + self.transaction_sender.send(send_to_peers_ids)?; } } @@ -398,24 +419,49 @@ impl Service for Mempool { } => match req { // Queries Request::TransactionIds => { + info!(?req, "got mempool request"); + let res = storage.tx_ids().collect(); + + info!(?req, ?res, "answered mempool request"); + async move { Ok(Response::TransactionIds(res)) }.boxed() } - Request::TransactionsById(ids) => { - let res = storage.transactions_exact(ids).cloned().collect(); + Request::TransactionsById(ref ids) => { + info!(?req, "got mempool request"); + + let res: Vec<_> = storage.transactions_exact(ids.clone()).cloned().collect(); + + info!(?req, res_count = ?res.len(), "answered mempool request"); + async move { Ok(Response::Transactions(res)) }.boxed() } - Request::TransactionsByMinedId(ids) => { - let res = storage.transactions_same_effects(ids).cloned().collect(); + Request::TransactionsByMinedId(ref ids) => { + info!(?req, "got mempool request"); + + let res: Vec<_> = storage + .transactions_same_effects(ids.clone()) + .cloned() + .collect(); + + info!(?req, res_count = ?res.len(), "answered mempool request"); + async move { Ok(Response::Transactions(res)) }.boxed() } - Request::RejectedTransactionIds(ids) => { - let res = storage.rejected_transactions(ids).collect(); + Request::RejectedTransactionIds(ref ids) => { + info!(?req, "got mempool request"); + + let res = storage.rejected_transactions(ids.clone()).collect(); + + info!(?req, ?res, "answered mempool request"); + async move { Ok(Response::RejectedTransactionIds(res)) }.boxed() } // Queue mempool candidates Request::Queue(gossiped_txs) => { + info!(req_count = ?gossiped_txs.len(), "got mempool Queue request"); + let rsp: Vec> = gossiped_txs .into_iter() .map(|gossiped_tx| -> Result<(), MempoolError> { @@ -440,11 +486,17 @@ impl Service for Mempool { // Store successfully downloaded and verified transactions in the mempool Request::CheckForVerifiedTransactions => { + info!(?req, "got mempool request"); + // all the work for this request is done in poll_ready async move { Ok(Response::CheckedForVerifiedTransactions) }.boxed() } }, ActiveState::Disabled => { + // TODO: add the name of the request, but not the content, + // like the command() or Display impls of network requests + info!("got mempool request while mempool is disabled"); + // We can't return an error since that will cause a disconnection // by the peer connection handler. Therefore, return successful // empty responses. diff --git a/zebrad/src/components/mempool/downloads.rs b/zebrad/src/components/mempool/downloads.rs index 472d9b477d5..f938541967b 100644 --- a/zebrad/src/components/mempool/downloads.rs +++ b/zebrad/src/components/mempool/downloads.rs @@ -36,6 +36,7 @@ use futures::{ future::TryFutureExt, ready, stream::{FuturesUnordered, Stream}, + FutureExt, }; use pin_project::{pin_project, pinned_drop}; use thiserror::Error; @@ -233,7 +234,7 @@ where let txid = gossiped_tx.id(); if self.cancel_handles.contains_key(&txid) { - debug!( + info!( ?txid, queue_len = self.pending.len(), ?MAX_INBOUND_CONCURRENCY, @@ -248,7 +249,7 @@ where } if self.pending.len() >= MAX_INBOUND_CONCURRENCY { - debug!( + info!( ?txid, queue_len = self.pending.len(), ?MAX_INBOUND_CONCURRENCY, @@ -273,6 +274,8 @@ where // Don't download/verify if the transaction is already in the state. Self::transaction_in_state(&mut state, txid).await?; + info!(?txid, "transaction is not in state"); + let next_height = match state.oneshot(zs::Request::Tip).await { Ok(zs::Response::Tip(None)) => Ok(Height(0)), Ok(zs::Response::Tip(Some((height, _hash)))) => { @@ -284,6 +287,8 @@ where Err(e) => Err(TransactionDownloadVerifyError::StateError(e)), }?; + info!(?txid, ?next_height, "got next height"); + let tx = match gossiped_tx { Gossip::Id(txid) => { let req = zn::Request::TransactionsById(std::iter::once(txid).collect()); @@ -322,6 +327,8 @@ where } }; + info!(?txid, "got tx"); + let result = verifier .oneshot(tx::Request::Mempool { transaction: tx.clone(), @@ -333,7 +340,8 @@ where }) .await; - debug!(?txid, ?result, "verified transaction for the mempool"); + // Hide the transaction data to avoid filling the logs + info!(?txid, result = ?result.as_ref().map(|_tx| ()), "verified transaction for the mempool"); result.map_err(|e| TransactionDownloadVerifyError::Invalid(e.into())) } @@ -348,6 +356,11 @@ where // Tack the hash onto the error so we can remove the cancel handle // on failure as well as on success. .map_err(move |e| (e, txid)) + .inspect(move |result| { + // Hide the transaction data to avoid filling the logs + let result = result.as_ref().map(|_tx| txid); + info!("mempool transaction result: {result:?}"); + }) .in_current_span(); let task = tokio::spawn(async move { diff --git a/zebrad/src/components/mempool/storage.rs b/zebrad/src/components/mempool/storage.rs index 440bba0a7de..376b04b63c0 100644 --- a/zebrad/src/components/mempool/storage.rs +++ b/zebrad/src/components/mempool/storage.rs @@ -179,6 +179,13 @@ impl Storage { // First, check if we have a cached rejection for this transaction. if let Some(error) = self.rejection_error(&tx_id) { + tracing::info!( + ?tx_id, + ?error, + stored_transaction_count = ?self.verified.transaction_count(), + "returning cached error for transaction", + ); + return Err(error); } @@ -187,12 +194,25 @@ impl Storage { // Security: transactions must not get refreshed by new queries, // because that allows malicious peers to keep transactions live forever. if self.verified.contains(&tx_id) { + tracing::info!( + ?tx_id, + stored_transaction_count = ?self.verified.transaction_count(), + "returning InMempool error for transaction that is already in the mempool", + ); + return Err(MempoolError::InMempool); } // Then, we try to insert into the pool. If this fails the transaction is rejected. let mut result = Ok(tx_id); if let Err(rejection_error) = self.verified.insert(tx) { + tracing::info!( + ?tx_id, + ?rejection_error, + stored_transaction_count = ?self.verified.transaction_count(), + "insertion error for transaction", + ); + // We could return here, but we still want to check the mempool size self.reject(tx_id, rejection_error.clone().into()); result = Err(rejection_error.into()); diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index 4f48445498a..3735e4b6a48 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -72,7 +72,7 @@ impl VerifiedSet { } /// Returns `true` if the set of verified transactions contains the transaction with the - /// specified `id. + /// specified [`UnminedTxId`]. pub fn contains(&self, id: &UnminedTxId) -> bool { self.transactions.iter().any(|tx| &tx.transaction.id == id) } From 69cc0c2ff4e2e05ebe625384222ce47762f4a4fc Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 30 Aug 2022 12:56:00 +1000 Subject: [PATCH 08/25] Test fetching sent mempool transactions using gRPC --- .../lightwalletd/send_transaction_test.rs | 73 +++++++++++++++++-- 1 file changed, 66 insertions(+), 7 deletions(-) diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index 2821befecd9..a9fcf86d871 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -25,7 +25,7 @@ use tower::{Service, ServiceExt}; use zebra_chain::{ block, chain_tip::ChainTip, parameters::Network, serialization::ZcashSerialize, - transaction::Transaction, + transaction::{self, Transaction}, }; use zebra_rpc::queue::CHANNEL_AND_QUEUE_CAPACITY; use zebra_state::HashOrHeight; @@ -35,13 +35,22 @@ use crate::common::{ cached_state::{load_tip_height_from_state_directory, start_state_service_with_cache_dir}, launch::spawn_zebrad_for_rpc_without_initial_peers, lightwalletd::{ - wallet_grpc::{self, connect_to_lightwalletd, spawn_lightwalletd_with_rpc_server}, + wallet_grpc::{self, connect_to_lightwalletd, spawn_lightwalletd_with_rpc_server, Exclude, Empty}, zebra_skip_lightwalletd_tests, LightwalletdTestType::*, }, sync::copy_state_and_perform_full_sync, }; +/// The maximum number of transactions we want to send in the test. +/// This avoids filling the mempool queue and generating errors. +/// +/// TODO: replace with a const when `min()` stabilises as a const function: +/// https://github.com/rust-lang/rust/issues/92391 +fn max_sent_transactions() -> usize { + min(CHANNEL_AND_QUEUE_CAPACITY, MAX_INBOUND_CONCURRENCY) - 1 +} + /// The test entry point. pub async fn run() -> Result<()> { let _init_guard = zebra_test::init(); @@ -82,7 +91,7 @@ pub async fn run() -> Result<()> { ); let mut transactions = - load_transactions_from_a_future_block(network, zebrad_state_path.clone()).await?; + load_transactions_from_future_blocks(network, zebrad_state_path.clone()).await?; tracing::info!( transaction_count = ?transactions.len(), @@ -114,19 +123,29 @@ pub async fn run() -> Result<()> { let mut rpc_client = connect_to_lightwalletd(lightwalletd_rpc_port).await?; // To avoid filling the mempool queue, limit the transactions to be sent to the RPC and mempool queue limits - transactions.truncate(min(CHANNEL_AND_QUEUE_CAPACITY, MAX_INBOUND_CONCURRENCY) - 1); + transactions.truncate(max_sent_transactions()); + + let transaction_hashes: Vec = transactions.iter().map(|tx| tx.hash()).collect(); tracing::info!( transaction_count = ?transactions.len(), + ?transaction_hashes, "connected gRPC client to lightwalletd, sending transactions...", ); for transaction in transactions { + let transaction_hash = transaction.hash(); + let expected_response = wallet_grpc::SendResponse { error_code: 0, - error_message: format!("\"{}\"", transaction.hash()), + error_message: format!("\"{}\"", transaction_hash), }; + tracing::info!( + ?transaction_hash, + "sending transaction...", + ); + let request = prepare_send_transaction_request(transaction); let response = rpc_client.send_transaction(request).await?.into_inner(); @@ -134,6 +153,46 @@ pub async fn run() -> Result<()> { assert_eq!(response, expected_response); } + // Wait a bit to query the mempool. + tokio::time::sleep(std::time::Duration::from_secs(30)).await; + + // Call `GetMempoolTx` and get a stream of transactions. + let mut transactions_stream = rpc_client + .get_mempool_tx(Exclude { txid: vec![] }) + .await? + .into_inner(); + + // Make sure at least one of the transactions were inserted into the mempool. + let mut counter = 0; + while let Some(tx) = transactions_stream.message().await? { + let hash: [u8; 32] = tx.hash.clone().try_into().expect("hash is correct length"); + let hash = transaction::Hash::from_bytes_in_display_order(&hash); + + assert!( + transaction_hashes.contains(&hash), + "unexpected transaction {hash:?}\n\ + in isolated mempool: {tx:?}", + ); + + counter += 1; + } + + assert!(counter >= 1, "all transactions from future blocks failed to send to an isolated mempool"); + + // Get the mempool transactions by calling `GetMempoolStream`. + let mut transaction_stream = rpc_client.get_mempool_stream(Empty {}).await?.into_inner(); + + let mut counter = 0; + while let Some(_tx) = transaction_stream.message().await? { + counter += 1; + } + + // This RPC has temporarily been disabled in `lightwalletd`: + // https://github.com/adityapk00/lightwalletd/blob/b563f765f620e38f482954cd8ff3cc6d17cf2fa7/frontend/service.go#L515-L517 + // + // TODO: re-enable it when lightwalletd starts streaming transactions again. + assert_eq!(counter, 0); + Ok(()) } @@ -147,7 +206,7 @@ pub async fn run() -> Result<()> { /// Returns a list of valid transactions that are not in any of the blocks present in the /// original `zebrad_state_path`. #[tracing::instrument] -async fn load_transactions_from_a_future_block( +async fn load_transactions_from_future_blocks( network: Network, zebrad_state_path: PathBuf, ) -> Result>> { @@ -202,7 +261,7 @@ async fn load_transactions_from_block_after( let mut target_height = height.0; let mut transactions = Vec::new(); - while transactions.is_empty() { + while transactions.len() < max_sent_transactions() { transactions = load_transactions_from_block(block::Height(target_height), &mut state).await?; From 9052cfd06ff566ac5da5775e53b7ad0f0ae4d929 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 30 Aug 2022 16:23:06 +1000 Subject: [PATCH 09/25] Add extra log checks to the send transaction test --- zebrad/tests/common/lightwalletd/send_transaction_test.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index a9fcf86d871..532e8bf688a 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -153,10 +153,10 @@ pub async fn run() -> Result<()> { assert_eq!(response, expected_response); } - // Wait a bit to query the mempool. - tokio::time::sleep(std::time::Duration::from_secs(30)).await; + tracing::info!("waiting for mempool to verify some transactions..."); + tokio::time::sleep(std::time::Duration::from_secs(10)).await; - // Call `GetMempoolTx` and get a stream of transactions. + tracing::info!("calling GetMempoolTx gRPC to fetch transactions..."); let mut transactions_stream = rpc_client .get_mempool_tx(Exclude { txid: vec![] }) .await? @@ -179,7 +179,7 @@ pub async fn run() -> Result<()> { assert!(counter >= 1, "all transactions from future blocks failed to send to an isolated mempool"); - // Get the mempool transactions by calling `GetMempoolStream`. + tracing::info!("calling GetMempoolStream gRPC to fetch transactions..."); let mut transaction_stream = rpc_client.get_mempool_stream(Empty {}).await?.into_inner(); let mut counter = 0; From 1e78f91bd31b940abeca3ae44696944fdf553650 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 30 Aug 2022 16:36:51 +1000 Subject: [PATCH 10/25] Wait for zebrad mempool activation before running gRPC tests --- .../lightwalletd/send_transaction_test.rs | 23 ++++++++++++++++--- .../common/lightwalletd/wallet_grpc_test.rs | 22 +++++++++++++++--- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index 532e8bf688a..555b5d4e86a 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -52,6 +52,11 @@ fn max_sent_transactions() -> usize { } /// The test entry point. +// +// TODO: +// - check output of zebrad and lightwalletd in different threads, +// to avoid test hangs due to full output pipes +// (see lightwalletd_integration_test for an example) pub async fn run() -> Result<()> { let _init_guard = zebra_test::init(); @@ -100,14 +105,15 @@ pub async fn run() -> Result<()> { ); // TODO: change debug_skip_parameter_preload to true if we do the mempool test in the wallet gRPC test - let (_zebrad, zebra_rpc_address) = + let (mut zebrad, zebra_rpc_address) = spawn_zebrad_for_rpc_without_initial_peers(Network::Mainnet, zebrad_state_path, test_type, false)?; tracing::info!( ?zebra_rpc_address, - "spawned disconnected zebrad with shorter chain", + "spawned disconnected zebrad with shorter chain, waiting for mempool activation...", ); + let (_lightwalletd, lightwalletd_rpc_port) = spawn_lightwalletd_with_rpc_server( zebra_rpc_address, lightwalletd_state_path, @@ -117,7 +123,18 @@ pub async fn run() -> Result<()> { tracing::info!( ?lightwalletd_rpc_port, - "spawned lightwalletd connected to zebrad", + "spawned lightwalletd connected to zebrad, waiting for zebrad mempool activation...", + ); + + zebrad.expect_stdout_line_matches("activating mempool")?; + + // TODO: check that lightwalletd is at the tip using gRPC (#4894) + // + // If this takes a long time, we might need to check zebrad logs for failures in a separate thread. + + tracing::info!( + ?lightwalletd_rpc_port, + "connecting gRPC client to lightwalletd...", ); let mut rpc_client = connect_to_lightwalletd(lightwalletd_rpc_port).await?; diff --git a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs index cdc47245b13..b72f8498d21 100644 --- a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs +++ b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs @@ -59,6 +59,11 @@ use crate::common::{ }; /// The test entry point. +// +// TODO: +// - check output of zebrad and lightwalletd in different threads, +// to avoid test hangs due to full output pipes +// (see lightwalletd_integration_test for an example) pub async fn run() -> Result<()> { let _init_guard = zebra_test::init(); @@ -98,12 +103,12 @@ pub async fn run() -> Result<()> { // Launch zebra using a predefined zebrad state path // // TODO: change debug_skip_parameter_preload to true if we do the mempool test in the send transaction test - let (_zebrad, zebra_rpc_address) = + let (mut zebrad, zebra_rpc_address) = spawn_zebrad_for_rpc_without_initial_peers(network, zebrad_state_path.unwrap(), test_type, false)?; tracing::info!( ?zebra_rpc_address, - "launching lightwalletd connected to zebrad...", + "launching lightwalletd connected to zebrad, waiting for the mempool to activate...", ); // Launch lightwalletd @@ -114,7 +119,18 @@ pub async fn run() -> Result<()> { false, )?; - // Give lightwalletd a few seconds to open its grpc port before connecting to it + tracing::info!( + ?lightwalletd_rpc_port, + "spawned lightwalletd connected to zebrad, waiting for zebrad mempool activation...", + ); + + zebrad.expect_stdout_line_matches("activating mempool")?; + + // Give lightwalletd a few seconds to sync to the tip before connecting to it + // + // TODO: check that lightwalletd is at the tip using gRPC (#4894) + // + // If this takes a long time, we might need to check zebrad logs for failures in a separate thread. tokio::time::sleep(std::time::Duration::from_secs(60)).await; tracing::info!( From 4b71d56f5b73d894b746b0680bdcc283c94648f3 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 31 Aug 2022 14:15:43 +1000 Subject: [PATCH 11/25] Update send transaction test for lightwalletd not returning mempool transactions --- .../lightwalletd/send_transaction_test.rs | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index 555b5d4e86a..53b7c6ce629 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -179,7 +179,7 @@ pub async fn run() -> Result<()> { .await? .into_inner(); - // Make sure at least one of the transactions were inserted into the mempool. + // GetMempoolTx: make sure at least one of the transactions were inserted into the mempool. let mut counter = 0; while let Some(tx) = transactions_stream.message().await? { let hash: [u8; 32] = tx.hash.clone().try_into().expect("hash is correct length"); @@ -194,13 +194,28 @@ pub async fn run() -> Result<()> { counter += 1; } - assert!(counter >= 1, "all transactions from future blocks failed to send to an isolated mempool"); + // This RPC has temporarily been disabled in `lightwalletd`: + // https://github.com/adityapk00/lightwalletd/blob/b563f765f620e38f482954cd8ff3cc6d17cf2fa7/frontend/service.go#L529-L531 + // + // TODO: re-enable it when lightwalletd starts returning transactions again. + //assert!(counter >= 1, "all transactions from future blocks failed to send to an isolated mempool"); + assert_eq!(counter, 0, "developers: update this test for lightwalletd sending transactions"); + // GetMempoolTx: make sure at least one of the transactions were inserted into the mempool. tracing::info!("calling GetMempoolStream gRPC to fetch transactions..."); let mut transaction_stream = rpc_client.get_mempool_stream(Empty {}).await?.into_inner(); let mut counter = 0; - while let Some(_tx) = transaction_stream.message().await? { + while let Some(tx) = transaction_stream.message().await? { + let hash: [u8; 32] = tx.hash.clone().try_into().expect("hash is correct length"); + let hash = transaction::Hash::from_bytes_in_display_order(&hash); + + assert!( + transaction_hashes.contains(&hash), + "unexpected transaction {hash:?}\n\ + in isolated mempool: {tx:?}", + ); + counter += 1; } @@ -208,7 +223,8 @@ pub async fn run() -> Result<()> { // https://github.com/adityapk00/lightwalletd/blob/b563f765f620e38f482954cd8ff3cc6d17cf2fa7/frontend/service.go#L515-L517 // // TODO: re-enable it when lightwalletd starts streaming transactions again. - assert_eq!(counter, 0); + //assert!(counter >= 1, "all transactions from future blocks failed to send to an isolated mempool"); + assert_eq!(counter, 0, "developers: update this test for lightwalletd sending transactions"); Ok(()) } From f6edf3c98516a670518a43ea18251b84a008cad9 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 31 Aug 2022 14:31:30 +1000 Subject: [PATCH 12/25] Check zebrad logs instead of disabled lightwalletd gRPCs --- .../common/lightwalletd/send_transaction_test.rs | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index 53b7c6ce629..c4e33668613 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -171,7 +171,7 @@ pub async fn run() -> Result<()> { } tracing::info!("waiting for mempool to verify some transactions..."); - tokio::time::sleep(std::time::Duration::from_secs(10)).await; + zebrad.expect_stdout_line_matches("sending new transactions to peers")?; tracing::info!("calling GetMempoolTx gRPC to fetch transactions..."); let mut transactions_stream = rpc_client @@ -179,6 +179,8 @@ pub async fn run() -> Result<()> { .await? .into_inner(); + zebrad.expect_stdout_line_matches("answered mempool request req=TransactionIds")?; + // GetMempoolTx: make sure at least one of the transactions were inserted into the mempool. let mut counter = 0; while let Some(tx) = transactions_stream.message().await? { @@ -206,15 +208,8 @@ pub async fn run() -> Result<()> { let mut transaction_stream = rpc_client.get_mempool_stream(Empty {}).await?.into_inner(); let mut counter = 0; - while let Some(tx) = transaction_stream.message().await? { - let hash: [u8; 32] = tx.hash.clone().try_into().expect("hash is correct length"); - let hash = transaction::Hash::from_bytes_in_display_order(&hash); - - assert!( - transaction_hashes.contains(&hash), - "unexpected transaction {hash:?}\n\ - in isolated mempool: {tx:?}", - ); + while let Some(_tx) = transaction_stream.message().await? { + // TODO: check tx.data or tx.height here? counter += 1; } From 30c52ff614ff5d773d0c6af6a87603fc937b1050 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 31 Aug 2022 14:33:09 +1000 Subject: [PATCH 13/25] Add a debug option that makes RPCs pretend the sync is finished --- zebra-rpc/src/config.rs | 7 ++++++ zebra-rpc/src/methods.rs | 30 +++++++++++++++++++------ zebra-rpc/src/methods/tests/snapshot.rs | 3 ++- zebra-rpc/src/methods/tests/vectors.rs | 30 ++++++++++++++++--------- zebra-rpc/src/server.rs | 10 +++++++-- 5 files changed, 60 insertions(+), 20 deletions(-) diff --git a/zebra-rpc/src/config.rs b/zebra-rpc/src/config.rs index d8cb4127492..c08b7da9a2a 100644 --- a/zebra-rpc/src/config.rs +++ b/zebra-rpc/src/config.rs @@ -49,6 +49,10 @@ pub struct Config { /// /// If some of those instances are outdated or failed, RPC queries can be slow or inconsistent. pub parallel_cpu_threads: usize, + + /// Test-only option that makes Zebra say it is at the chain tip, + /// no matter what the estimated height or local clock is. + pub debug_force_finished_sync: bool, } impl Default for Config { @@ -59,6 +63,9 @@ impl Default for Config { // Use a single thread, so we can detect RPC port conflicts. parallel_cpu_threads: 1, + + // Debug options are always off by default. + debug_force_finished_sync: false, } } } diff --git a/zebra-rpc/src/methods.rs b/zebra-rpc/src/methods.rs index 6da888f16f2..4f74d2496a3 100644 --- a/zebra-rpc/src/methods.rs +++ b/zebra-rpc/src/methods.rs @@ -243,9 +243,20 @@ where >, Tip: ChainTip, { + // Configuration + // /// Zebra's application version. app_version: String, + /// The configured network for this RPC service. + network: Network, + + /// Test-only option that makes Zebra say it is at the chain tip, + /// no matter what the estimated height or local clock is. + debug_force_finished_sync: bool, + + // Services + // /// A handle to the mempool service. mempool: Buffer, @@ -255,10 +266,8 @@ where /// Allows efficient access to the best tip of the blockchain. latest_chain_tip: Tip, - /// The configured network for this RPC service. - #[allow(dead_code)] - network: Network, - + // Tasks + // /// A sender component of a channel used to send transactions to the queue. queue_sender: Sender>, } @@ -279,10 +288,11 @@ where /// Create a new instance of the RPC handler. pub fn new( app_version: Version, + network: Network, + debug_force_finished_sync: bool, mempool: Buffer, state: State, latest_chain_tip: Tip, - network: Network, ) -> (Self, JoinHandle<()>) where Version: ToString, @@ -300,10 +310,11 @@ where let rpc_impl = RpcImpl { app_version, + network, + debug_force_finished_sync, mempool: mempool.clone(), state: state.clone(), latest_chain_tip: latest_chain_tip.clone(), - network, queue_sender: runner.sender(), }; @@ -379,13 +390,18 @@ where data: None, })?; - let estimated_height = + let mut estimated_height = if current_block_time > Utc::now() || zebra_estimated_height < tip_height { tip_height } else { zebra_estimated_height }; + // If we're testing the mempool, force the estimated height to be the actual tip height. + if self.debug_force_finished_sync { + estimated_height = tip_height; + } + // `upgrades` object // // Get the network upgrades in height order, like `zcashd`. diff --git a/zebra-rpc/src/methods/tests/snapshot.rs b/zebra-rpc/src/methods/tests/snapshot.rs index 48e9d71a942..d9e1a8f4277 100644 --- a/zebra-rpc/src/methods/tests/snapshot.rs +++ b/zebra-rpc/src/methods/tests/snapshot.rs @@ -44,10 +44,11 @@ async fn test_rpc_response_data_for_network(network: Network) { // Init RPC let (rpc, _rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", + network, + false, Buffer::new(mempool.clone(), 1), read_state, latest_chain_tip, - network, ); // Start snapshots of RPC responses. diff --git a/zebra-rpc/src/methods/tests/vectors.rs b/zebra-rpc/src/methods/tests/vectors.rs index 2e276aa6d6e..114602ca7d9 100644 --- a/zebra-rpc/src/methods/tests/vectors.rs +++ b/zebra-rpc/src/methods/tests/vectors.rs @@ -29,10 +29,11 @@ async fn rpc_getinfo() { let (rpc, rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", + Mainnet, + false, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, - Mainnet, ); let get_info = rpc.get_info().expect("We should have a GetInfo struct"); @@ -71,10 +72,11 @@ async fn rpc_getblock() { // Init RPC let (rpc, rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", + Mainnet, + false, Buffer::new(mempool.clone(), 1), read_state, latest_chain_tip, - Mainnet, ); // Make calls with verbosity=0 and check response @@ -123,10 +125,11 @@ async fn rpc_getblock_parse_error() { // Init RPC let (rpc, rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", + Mainnet, + false, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, - Mainnet, ); // Make sure we get an error if Zebra can't parse the block height. @@ -153,10 +156,11 @@ async fn rpc_getblock_missing_error() { // Init RPC let (rpc, rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", + Mainnet, + false, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, - Mainnet, ); // Make sure Zebra returns the correct error code `-8` for missing blocks @@ -219,10 +223,11 @@ async fn rpc_getbestblockhash() { // Init RPC let (rpc, rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", + Mainnet, + false, Buffer::new(mempool.clone(), 1), read_state, latest_chain_tip, - Mainnet, ); // Get the tip hash using RPC method `get_best_block_hash` @@ -259,10 +264,11 @@ async fn rpc_getrawtransaction() { // Init RPC let (rpc, rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", + Mainnet, + false, Buffer::new(mempool.clone(), 1), read_state, latest_chain_tip, - Mainnet, ); // Test case where transaction is in mempool. @@ -344,10 +350,11 @@ async fn rpc_getaddresstxids_invalid_arguments() { let (rpc, rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", + Mainnet, + false, Buffer::new(mempool.clone(), 1), Buffer::new(read_state.clone(), 1), latest_chain_tip, - Mainnet, ); // call the method with an invalid address string @@ -485,10 +492,11 @@ async fn rpc_getaddresstxids_response_with( let (rpc, rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", + network, + false, Buffer::new(mempool.clone(), 1), Buffer::new(read_state.clone(), 1), latest_chain_tip, - network, ); // call the method with valid arguments @@ -534,10 +542,11 @@ async fn rpc_getaddressutxos_invalid_arguments() { let rpc = RpcImpl::new( "RPC test", + Mainnet, + false, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, - Mainnet, ); // call the method with an invalid address string @@ -583,10 +592,11 @@ async fn rpc_getaddressutxos_response() { let rpc = RpcImpl::new( "RPC test", + Mainnet, + false, Buffer::new(mempool.clone(), 1), Buffer::new(read_state.clone(), 1), latest_chain_tip, - Mainnet, ); // call the method with a valid address diff --git a/zebra-rpc/src/server.rs b/zebra-rpc/src/server.rs index b8cd316a352..b49460502a9 100644 --- a/zebra-rpc/src/server.rs +++ b/zebra-rpc/src/server.rs @@ -65,8 +65,14 @@ impl RpcServer { info!("Trying to open RPC endpoint at {}...", listen_addr,); // Initialize the rpc methods with the zebra version - let (rpc_impl, rpc_tx_queue_task_handle) = - RpcImpl::new(app_version, mempool, state, latest_chain_tip, network); + let (rpc_impl, rpc_tx_queue_task_handle) = RpcImpl::new( + app_version, + network, + config.debug_force_finished_sync, + mempool, + state, + latest_chain_tip, + ); // Create handler compatible with V1 and V2 RPC protocols let mut io: MetaIoHandler<(), _> = From f8b63e164160f9a896bed4d62216f0a0c940b8d6 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 31 Aug 2022 14:39:04 +1000 Subject: [PATCH 14/25] Remove an unused debug option --- zebrad/src/components/mempool.rs | 20 +++----------------- zebrad/src/components/mempool/config.rs | 4 ---- zebrad/tests/common/launch.rs | 2 -- 3 files changed, 3 insertions(+), 23 deletions(-) diff --git a/zebrad/src/components/mempool.rs b/zebrad/src/components/mempool.rs index 8279050ef28..0a490c5d251 100644 --- a/zebrad/src/components/mempool.rs +++ b/zebrad/src/components/mempool.rs @@ -280,11 +280,6 @@ impl Mempool { .copied() .collect() } - - /// TBA - pub fn skip_full_validation(&self) -> bool { - self.config.skip_full_validation - } } impl Service for Mempool { @@ -410,7 +405,7 @@ impl Service for Mempool { /// and will cause callers to disconnect from the remote peer. #[instrument(name = "mempool", skip(self, req))] fn call(&mut self, req: Request) -> Self::Future { - let skip_full_validation = self.skip_full_validation(); + //let skip_full_validation = self.skip_full_validation(); match &mut self.active_state { ActiveState::Enabled { @@ -466,17 +461,8 @@ impl Service for Mempool { .into_iter() .map(|gossiped_tx| -> Result<(), MempoolError> { storage.should_download_or_verify(gossiped_tx.id())?; - tx_downloads.download_if_needed_and_verify(gossiped_tx.clone())?; - if skip_full_validation { - if let Some(tx) = gossiped_tx.tx() { - storage.insert( - zebra_chain::transaction::VerifiedUnminedTx::new( - tx, - zebra_chain::amount::Amount::zero(), - ), - )?; - } - } + tx_downloads.download_if_needed_and_verify(gossiped_tx)?; + Ok(()) }) .map(|result| result.map_err(BoxError::from)) diff --git a/zebrad/src/components/mempool/config.rs b/zebrad/src/components/mempool/config.rs index 25d4a7fce97..2b4f20f6a1b 100644 --- a/zebrad/src/components/mempool/config.rs +++ b/zebrad/src/components/mempool/config.rs @@ -41,9 +41,6 @@ pub struct Config { // - allow the mempool to be enabled before the genesis block is committed? // we could replace `Option` with an enum that has an `AlwaysEnable` variant pub debug_enable_at_height: Option, - - /// TBA - pub skip_full_validation: bool, } impl Default for Config { @@ -62,7 +59,6 @@ impl Default for Config { eviction_memory_time: Duration::from_secs(60 * 60), debug_enable_at_height: None, - skip_full_validation: false, } } } diff --git a/zebrad/tests/common/launch.rs b/zebrad/tests/common/launch.rs index bb044f74269..2932de95004 100644 --- a/zebrad/tests/common/launch.rs +++ b/zebrad/tests/common/launch.rs @@ -227,8 +227,6 @@ pub fn spawn_zebrad_for_rpc_without_initial_peers Date: Wed, 31 Aug 2022 14:43:16 +1000 Subject: [PATCH 15/25] Remove unused test code and downgrade some logs --- zebra-consensus/src/transaction.rs | 4 +- zebra-rpc/src/methods.rs | 2 +- zebra-test/src/vectors.rs | 13 ----- zebrad/src/components/mempool.rs | 32 +++++------ zebrad/src/components/mempool/downloads.rs | 14 ++--- zebrad/src/components/mempool/storage.rs | 6 +- .../lightwalletd/send_transaction_test.rs | 36 ++++++++---- .../common/lightwalletd/wallet_grpc_test.rs | 57 ++++--------------- 8 files changed, 62 insertions(+), 102 deletions(-) diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index 7f13bf3ee80..f992bdda3d8 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -304,8 +304,6 @@ where let tx_id = req.tx_id(); let span = tracing::debug_span!("tx", ?tx_id); - let is_mempool = req.is_mempool(); - async move { tracing::trace!(?tx_id, ?req, "got tx verify request"); @@ -317,7 +315,7 @@ where check::has_inputs_and_outputs(&tx)?; check::has_enough_orchard_flags(&tx)?; - if is_mempool && tx.is_coinbase() { + if req.is_mempool() && tx.is_coinbase() { return Err(TransactionError::CoinbaseInMempool); } if tx.is_coinbase() { diff --git a/zebra-rpc/src/methods.rs b/zebra-rpc/src/methods.rs index 4f74d2496a3..dcd8c91a835 100644 --- a/zebra-rpc/src/methods.rs +++ b/zebra-rpc/src/methods.rs @@ -522,7 +522,7 @@ where "mempool service returned more results than expected" ); - tracing::info!("sent transaction to mempool: {:?}", &queue_results[0]); + tracing::debug!("sent transaction to mempool: {:?}", &queue_results[0]); match &queue_results[0] { Ok(()) => Ok(SentTransactionHash(transaction_hash)), diff --git a/zebra-test/src/vectors.rs b/zebra-test/src/vectors.rs index 3973028b2a1..cfcd48e811b 100644 --- a/zebra-test/src/vectors.rs +++ b/zebra-test/src/vectors.rs @@ -188,17 +188,4 @@ lazy_static! { /// pub static ref ZIP243_3: Vec = >::from_hex("0400008085202f8901a8c685478265f4c14dada651969c45a65e1aeb8cd6791f2f5bb6a1d9952104d9010000006b483045022100a61e5d557568c2ddc1d9b03a7173c6ce7c996c4daecab007ac8f34bee01e6b9702204d38fdc0bcf2728a69fde78462a10fb45a9baa27873e6a5fc45fb5c76764202a01210365ffea3efa3908918a8b8627724af852fc9b86d7375b103ab0543cf418bcaa7ffeffffff02005a6202000000001976a9148132712c3ff19f3a151234616777420a6d7ef22688ac8b959800000000001976a9145453e4698f02a38abdaa521cd1ff2dee6fac187188ac29b0040048b004000000000000000000000000") .expect("Transaction bytes are in valid hex representation"); - - /// https://explorer.zcha.in/transactions/91c61ef8770986f27651a73fe8ea35607c6ebe91201894f3cd7c6a97359a2751 but with the - /// expiration height modified to hex 00093d (4000000 decimal in little endian). - /// v5 transaction with transparent inputs and outputs, no shielded. - pub static ref DUMMY_TX2: Vec = >::from_hex("050000800a27a726b4d0d6c23fbe190000093d0001ce6336775066a26ef5b38e91c73c907d255da7488d40cb0682284aeac84c8af4000000006b483045022100eaf7fdcaff8f0716385cefb997dd1b64a6d2cf4cdfc25b04352376ea610e937702202c17f82f3ce506bdcc3e5d74a103305cfdd838f75095495dc5e4995ecab7ae37012103f68dd6a595fa8381470462eef99803eccbc24aea0cdda5a88c2010ab10943db8feffffff0258252800000000001976a9141cb70d902612db048aa9b0462cea068f81aed9cc88ac491a2400000000001976a914fe4637b0103af2339fb69105e2f17a6d37c282a788ac000000") - .expect("Transaction bytes are in valid hex representation"); - - /// https://explorer.zcha.in/transactions/babaf52aaafbee607bce231178be756121e839c4c99e2a15eedad84b1288a396 but with the - /// expiration height modified to hex 00093d (4000000 decimal in little endian). - /// v5 transaction, no transparent data, only shielded sapling. - pub static ref DUMMY_TX3: Vec = >::from_hex("050000800a27a726b4d0d6c20000000000093d00000001918291c84ae675708527da9b7f5e836df9df29b126f9f0e0afa01c7b794e01267f0d85b3a51a242883f4b3786b6ecddf67d9c935c4cce59f038ce13451bbc259cd14a2edc26a42478e15680dbafc9bb5ea6f05af911150d72c8abf3b973658ed02dbdf3941aae0b511102a718851429fffab6b761b2ff03c5182ff091b9363d1c4b2eae6fd2e4ade1b7f2126a4ec6167c76828fc1006bedeafcf6ba18eebdaf166b8a50eb85ed95abae3f8425e8c4859e57d2335e66be322046705f7db6fe01669fc6cd051be33ff2dbcd6ac1ff3791f474380dec36b56f9364b1e748bde301b2c1824841e3516636a9986e8c832a084b363ed92de75858f9014b7f7b987449bfea635007e949a9f2d74f64d393e302876764172ee90862df1fe743dbe5242b8d6c34882dbc240b95586bc79001356fb08952d3d336355dbbc10a92bd189181f469aae920269c6efaa7b2a1ecdcfd9efe978ed50af0b26c130436469cf13812071c557ef140b3d1028e200662f74a4e9ece1510282a473eb7820cbdd2af8a10dfca03c567d093d209334b935c35958bc17bce51dfda9c6231fea4fcac2720b8096b263c40e0fbe9e4c30abe0e814307a3c40dec9b8b508ccbf907579fbf9da953ab2ea634918ee350905bed666b67bfb74a808a927b74c5ccddaa08d4215e501278b0deb1955dd073bc9fe4688567ed7880b00322f6092e8cb794cefafd572c8b61fa89bee8ead90aa02b667c9b29bfd0a4111db62518795c7f6f2f940f5f6e2610c50d0ced13a070ed08eb781fe9102572aa46bd0fc2ff8b09a5bc29b8cf755f8ab62ea09bdc23dcb6bf08fe99b4031aeaec61bbc55add8cac4d9e8f35395ae0043199831a4bffac91988b711a9abb5c135c907c40fcd36917eceeedfcc4cd172bce8bbe5e05496cb8c2d45c5e779d20f2ff93a7deffcb8f8fac15e83e02924dee7a19b0e5ea9cc3a7397b3e53572d3d55b73d8029651f30a118ac722669db7e8d1effd70bf5acf69bca10af12f7c1b44cdabf7c013bfc8bc16b891ce94548335258bad5d2ad189d413141dff8342639cd9b7f199cb7c81e7c4900c35e75f934fa407a7c3e2c4cf5d6bb0eee3449ece95efce0a4fbb4c1b97f209398ba4c32092ade4f435b82e50038b25c503afc00f9f9a772bf7f04cb8aba32704efe0a10ce1bd2c7a669da3dec4cee5aa7f4a7dcdc367b4f62275eb08bd6a32457e5f28ddd56d6518049f60754b0b37fc6e3d714dfbc63cadceb643705be09893af0c26ce2dd742fab7a4c462500a6aa3bd82b9b01cd2c2935c31b16b195adcebc981b6413009450db84005b49a4da62893ee3a36cce886eee2796da3f8767760abf40d5f93bc560f06696e670b6edaa93b1e25c228684baebfaf538d1da43a180cd491d330a4046d7b6a51dd15c20cf5a414aa59829288003741af558be0079740a94059ed6cdce6c133bfbaa700751176e71743657d0b077f6ca12e3cc062e5d613ecb9c36d63129e7c4a6a1188c3dbb9e954126b15592b06d0d3c1c7b232866a4cb3398fc11496180df5a928091d056a216fa577e230824de875ad77902d10eebd6c6e755f59fc32b3ee892cb213e4c81acd4312e7f0e486b1446cea00b2caa87565334d638267c2810f02bca09b799d6a01940d17c48fc2afff18e25984d8f7ee9b102c88e610e61f9695c2475d3c97a7f9a32f0af730f4848ea855fa2f963dd633d4c0edecfde477503940d86caf449185b15efb278e08c428135848fd2bc312c8eb51f001b5e5e3f61fb5ac8d57d92e3c604697beb53368641101b382fdb0edf1cac0053c5ea1f5732e557060f74c5d14ef8e2c906938d15bbf5423e4226f510a938d7be7769275f6461c2ef65db98b18cc62a38be82ce997265e1b585099903be8308ac0d5bd66ff210658263dc5fb2267b7a8d2b3dd2fc25c25d18a2b5ab9b5e9a43ab7c900f2a63e2950c711b33eedfcb725d133e67259a5f867a6f69eed537e48edae77cf3d0b0c7e60f61f6c1c418318d2508fdce67039892db855561506ec534a2120de338f6d165dcf8f0e8e28c1a529395c2b068129c4b49b31f0328de95475c83bb625193203d20c9df9a762f3e7229c0a6a31ec703dc772729248cd9fd979839d1e67b41ed06083574dca9070aedf183da902a71c64021af018a596be977f3fb55d797b80a9a4ef37f9446aa522db786db700162085ad4200a4275382301c7fb1882159bdec2867942951fde2549f0e8d2bf2f70461ec7c6de9be52938e4bebca393bedddf9e8030000000000009f600fe22c07fd60d49559791edcf74ecef542f800ab0adcfeda49684655044c990dc78dab58805cbe2d2b088c3946383a5f145a42501fde7c42b44b14d0126547e8375a067c6c5877f8faebb131e5dead4946c3200080706698f5f1695b986a77e7e5ba44bf64e8cbff66cd63449f67304542c403c367940d6e8c81f8e977990ec7f30ce82b238ee24e4a0e2e5472a0a4342d8d3227fd47af48c5bfb0f28dc97fb9baa88fbbbb8630773fa023f4db2f92f413a8babd112e3a7d11321c3135296f7851f3be628a183d42627f859dfe2d9ccf66a6f5749fbce5bb9c87353950ae4031115744c2393f38f88349c668028bfaf0b7bd3a5088f48b500cc905bb443cc60d80a1eb21b6da0202f673a570ec4940b162d3c67c04a4e796fe6aef0ca4089502f0d129707ada789abeeba5c09f572f32489498d588a5f2a02c03b64bc9ab9558c38236b32f7312977530547d03e5a2f6374cabe7809aece49bd3b5c40890a7e276e66d7d3672ca3d610f83e1be09ffde1a9debabfdecb1a5dec7b5d9d3a505949ba33e5c5e7c06fd32c162bc82a6398e1adbfeff7f549d7dfc8f510a21fcfd7baa2e4a295afcc064c5189ebeef37b580d01ae34d1fcd571f2c89c8f566c4cc5abedb4653ba52edc6a9d480092f6765bbcba9acebc9f11e3bfbe2afd22fd9a9d1dd1afec6a0777324f325b64fe34cbf4f57444d70cd20e53466aab657dcbfc37309ce0ff4df6cd120ed9ea564be50b94355ee97bfd403aba15466d2475df2827947185dfb1c7c2eacbbf11622c81e86400f89bced44ffd6f84f3e18645aab168a3b5920ace730cfb0a1b939b00815c2a97b78317fd22e7e17b5630282b84bebe72d23b634c7d1501dde3a6d92147e94d5227937b5975341d2f9f4ae2b15ffcf6690a8bef76afe0ff4a157a08adc727fd69b78f896b2011607cd8229e38d3e813ed4c5728d8c3b3431738d5860b3692def17096122f7771b545764a83ea2512973187b565e4ee96838b3295dd33d1cf67bb759510e8fffdbdb73cde9c15b0900") - .expect("Transaction bytes are in valid hex representation"); - } diff --git a/zebrad/src/components/mempool.rs b/zebrad/src/components/mempool.rs index 0a490c5d251..208c980ed09 100644 --- a/zebrad/src/components/mempool.rs +++ b/zebrad/src/components/mempool.rs @@ -291,7 +291,7 @@ impl Service for Mempool { fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { let is_state_changed = self.update_state(); - tracing::info!(is_enabled = ?self.is_enabled(), ?is_state_changed, "started polling the mempool..."); + tracing::trace!(is_enabled = ?self.is_enabled(), ?is_state_changed, "started polling the mempool..."); // When the mempool is disabled we still return that the service is ready. // Otherwise, callers could block waiting for the mempool to be enabled. @@ -342,7 +342,7 @@ impl Service for Mempool { Ok(tx) => { let insert_result = storage.insert(tx.clone()); - tracing::info!( + tracing::trace!( ?insert_result, "got Ok(_) transaction verify, tried to store", ); @@ -353,7 +353,7 @@ impl Service for Mempool { } } Err((txid, error)) => { - tracing::info!(?txid, ?error, "mempool transaction failed to verify"); + tracing::debug!(?txid, ?error, "mempool transaction failed to verify"); metrics::counter!("mempool.failed.verify.tasks.total", 1, "reason" => error.to_string()); storage.reject_if_needed(txid, error); @@ -363,7 +363,7 @@ impl Service for Mempool { // Handle best chain tip changes if let Some(TipAction::Grow { block }) = tip_action { - tracing::info!(block_height = ?block.height, "handling blocks added to tip"); + tracing::trace!(block_height = ?block.height, "handling blocks added to tip"); // Cancel downloads/verifications/storage of transactions // with the same mined IDs as recently mined transactions. @@ -381,7 +381,7 @@ impl Service for Mempool { Self::remove_expired_from_peer_list(&send_to_peers_ids, &expired_transactions); if !expired_transactions.is_empty() { - tracing::info!( + tracing::debug!( ?expired_transactions, "removed expired transactions from the mempool", ); @@ -414,48 +414,48 @@ impl Service for Mempool { } => match req { // Queries Request::TransactionIds => { - info!(?req, "got mempool request"); + trace!(?req, "got mempool request"); let res = storage.tx_ids().collect(); - info!(?req, ?res, "answered mempool request"); + trace!(?req, ?res, "answered mempool request"); async move { Ok(Response::TransactionIds(res)) }.boxed() } Request::TransactionsById(ref ids) => { - info!(?req, "got mempool request"); + trace!(?req, "got mempool request"); let res: Vec<_> = storage.transactions_exact(ids.clone()).cloned().collect(); - info!(?req, res_count = ?res.len(), "answered mempool request"); + trace!(?req, res_count = ?res.len(), "answered mempool request"); async move { Ok(Response::Transactions(res)) }.boxed() } Request::TransactionsByMinedId(ref ids) => { - info!(?req, "got mempool request"); + trace!(?req, "got mempool request"); let res: Vec<_> = storage .transactions_same_effects(ids.clone()) .cloned() .collect(); - info!(?req, res_count = ?res.len(), "answered mempool request"); + trace!(?req, res_count = ?res.len(), "answered mempool request"); async move { Ok(Response::Transactions(res)) }.boxed() } Request::RejectedTransactionIds(ref ids) => { - info!(?req, "got mempool request"); + trace!(?req, "got mempool request"); let res = storage.rejected_transactions(ids.clone()).collect(); - info!(?req, ?res, "answered mempool request"); + trace!(?req, ?res, "answered mempool request"); async move { Ok(Response::RejectedTransactionIds(res)) }.boxed() } // Queue mempool candidates Request::Queue(gossiped_txs) => { - info!(req_count = ?gossiped_txs.len(), "got mempool Queue request"); + trace!(req_count = ?gossiped_txs.len(), "got mempool Queue request"); let rsp: Vec> = gossiped_txs .into_iter() @@ -472,7 +472,7 @@ impl Service for Mempool { // Store successfully downloaded and verified transactions in the mempool Request::CheckForVerifiedTransactions => { - info!(?req, "got mempool request"); + trace!(?req, "got mempool request"); // all the work for this request is done in poll_ready async move { Ok(Response::CheckedForVerifiedTransactions) }.boxed() @@ -481,7 +481,7 @@ impl Service for Mempool { ActiveState::Disabled => { // TODO: add the name of the request, but not the content, // like the command() or Display impls of network requests - info!("got mempool request while mempool is disabled"); + trace!("got mempool request while mempool is disabled"); // We can't return an error since that will cause a disconnection // by the peer connection handler. Therefore, return successful diff --git a/zebrad/src/components/mempool/downloads.rs b/zebrad/src/components/mempool/downloads.rs index f938541967b..6cffca9f73b 100644 --- a/zebrad/src/components/mempool/downloads.rs +++ b/zebrad/src/components/mempool/downloads.rs @@ -234,7 +234,7 @@ where let txid = gossiped_tx.id(); if self.cancel_handles.contains_key(&txid) { - info!( + debug!( ?txid, queue_len = self.pending.len(), ?MAX_INBOUND_CONCURRENCY, @@ -249,7 +249,7 @@ where } if self.pending.len() >= MAX_INBOUND_CONCURRENCY { - info!( + debug!( ?txid, queue_len = self.pending.len(), ?MAX_INBOUND_CONCURRENCY, @@ -274,7 +274,7 @@ where // Don't download/verify if the transaction is already in the state. Self::transaction_in_state(&mut state, txid).await?; - info!(?txid, "transaction is not in state"); + trace!(?txid, "transaction is not in state"); let next_height = match state.oneshot(zs::Request::Tip).await { Ok(zs::Response::Tip(None)) => Ok(Height(0)), @@ -287,7 +287,7 @@ where Err(e) => Err(TransactionDownloadVerifyError::StateError(e)), }?; - info!(?txid, ?next_height, "got next height"); + trace!(?txid, ?next_height, "got next height"); let tx = match gossiped_tx { Gossip::Id(txid) => { @@ -327,7 +327,7 @@ where } }; - info!(?txid, "got tx"); + trace!(?txid, "got tx"); let result = verifier .oneshot(tx::Request::Mempool { @@ -341,7 +341,7 @@ where .await; // Hide the transaction data to avoid filling the logs - info!(?txid, result = ?result.as_ref().map(|_tx| ()), "verified transaction for the mempool"); + trace!(?txid, result = ?result.as_ref().map(|_tx| ()), "verified transaction for the mempool"); result.map_err(|e| TransactionDownloadVerifyError::Invalid(e.into())) } @@ -359,7 +359,7 @@ where .inspect(move |result| { // Hide the transaction data to avoid filling the logs let result = result.as_ref().map(|_tx| txid); - info!("mempool transaction result: {result:?}"); + debug!("mempool transaction result: {result:?}"); }) .in_current_span(); diff --git a/zebrad/src/components/mempool/storage.rs b/zebrad/src/components/mempool/storage.rs index 376b04b63c0..d4acc47f33a 100644 --- a/zebrad/src/components/mempool/storage.rs +++ b/zebrad/src/components/mempool/storage.rs @@ -179,7 +179,7 @@ impl Storage { // First, check if we have a cached rejection for this transaction. if let Some(error) = self.rejection_error(&tx_id) { - tracing::info!( + tracing::trace!( ?tx_id, ?error, stored_transaction_count = ?self.verified.transaction_count(), @@ -194,7 +194,7 @@ impl Storage { // Security: transactions must not get refreshed by new queries, // because that allows malicious peers to keep transactions live forever. if self.verified.contains(&tx_id) { - tracing::info!( + tracing::trace!( ?tx_id, stored_transaction_count = ?self.verified.transaction_count(), "returning InMempool error for transaction that is already in the mempool", @@ -206,7 +206,7 @@ impl Storage { // Then, we try to insert into the pool. If this fails the transaction is rejected. let mut result = Ok(tx_id); if let Err(rejection_error) = self.verified.insert(tx) { - tracing::info!( + tracing::debug!( ?tx_id, ?rejection_error, stored_transaction_count = ?self.verified.transaction_count(), diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index c4e33668613..b4ee30d40f0 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -24,7 +24,10 @@ use futures::TryFutureExt; use tower::{Service, ServiceExt}; use zebra_chain::{ - block, chain_tip::ChainTip, parameters::Network, serialization::ZcashSerialize, + block, + chain_tip::ChainTip, + parameters::Network, + serialization::ZcashSerialize, transaction::{self, Transaction}, }; use zebra_rpc::queue::CHANNEL_AND_QUEUE_CAPACITY; @@ -35,7 +38,9 @@ use crate::common::{ cached_state::{load_tip_height_from_state_directory, start_state_service_with_cache_dir}, launch::spawn_zebrad_for_rpc_without_initial_peers, lightwalletd::{ - wallet_grpc::{self, connect_to_lightwalletd, spawn_lightwalletd_with_rpc_server, Exclude, Empty}, + wallet_grpc::{ + self, connect_to_lightwalletd, spawn_lightwalletd_with_rpc_server, Empty, Exclude, + }, zebra_skip_lightwalletd_tests, LightwalletdTestType::*, }, @@ -105,15 +110,18 @@ pub async fn run() -> Result<()> { ); // TODO: change debug_skip_parameter_preload to true if we do the mempool test in the wallet gRPC test - let (mut zebrad, zebra_rpc_address) = - spawn_zebrad_for_rpc_without_initial_peers(Network::Mainnet, zebrad_state_path, test_type, false)?; + let (mut zebrad, zebra_rpc_address) = spawn_zebrad_for_rpc_without_initial_peers( + Network::Mainnet, + zebrad_state_path, + test_type, + false, + )?; tracing::info!( ?zebra_rpc_address, "spawned disconnected zebrad with shorter chain, waiting for mempool activation...", ); - let (_lightwalletd, lightwalletd_rpc_port) = spawn_lightwalletd_with_rpc_server( zebra_rpc_address, lightwalletd_state_path, @@ -142,7 +150,8 @@ pub async fn run() -> Result<()> { // To avoid filling the mempool queue, limit the transactions to be sent to the RPC and mempool queue limits transactions.truncate(max_sent_transactions()); - let transaction_hashes: Vec = transactions.iter().map(|tx| tx.hash()).collect(); + let transaction_hashes: Vec = + transactions.iter().map(|tx| tx.hash()).collect(); tracing::info!( transaction_count = ?transactions.len(), @@ -158,10 +167,7 @@ pub async fn run() -> Result<()> { error_message: format!("\"{}\"", transaction_hash), }; - tracing::info!( - ?transaction_hash, - "sending transaction...", - ); + tracing::info!(?transaction_hash, "sending transaction..."); let request = prepare_send_transaction_request(transaction); @@ -201,7 +207,10 @@ pub async fn run() -> Result<()> { // // TODO: re-enable it when lightwalletd starts returning transactions again. //assert!(counter >= 1, "all transactions from future blocks failed to send to an isolated mempool"); - assert_eq!(counter, 0, "developers: update this test for lightwalletd sending transactions"); + assert_eq!( + counter, 0, + "developers: update this test for lightwalletd sending transactions" + ); // GetMempoolTx: make sure at least one of the transactions were inserted into the mempool. tracing::info!("calling GetMempoolStream gRPC to fetch transactions..."); @@ -219,7 +228,10 @@ pub async fn run() -> Result<()> { // // TODO: re-enable it when lightwalletd starts streaming transactions again. //assert!(counter >= 1, "all transactions from future blocks failed to send to an isolated mempool"); - assert_eq!(counter, 0, "developers: update this test for lightwalletd sending transactions"); + assert_eq!( + counter, 0, + "developers: update this test for lightwalletd sending transactions" + ); Ok(()) } diff --git a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs index b72f8498d21..8b4bade02e4 100644 --- a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs +++ b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs @@ -19,7 +19,8 @@ //! - `GetTaddressBalance`: Covered. //! - `GetTaddressBalanceStream`: Covered. //! -//! - `GetMempoolTx`: Covered by the send_transaction_test. +//! - `GetMempoolTx`: Covered by the send_transaction_test, +//! currently disabled by `lightwalletd`. //! - `GetMempoolStream`: Covered by the send_transaction_test, //! currently disabled by `lightwalletd`. //! @@ -39,8 +40,7 @@ use zebra_chain::{ block::Block, parameters::Network, parameters::NetworkUpgrade::{self, Canopy}, - serialization::{ZcashDeserialize, ZcashDeserializeInto, ZcashSerialize}, - transaction::Transaction, + serialization::ZcashDeserializeInto, }; use zebra_network::constants::USER_AGENT; @@ -50,7 +50,7 @@ use crate::common::{ lightwalletd::{ wallet_grpc::{ connect_to_lightwalletd, spawn_lightwalletd_with_rpc_server, Address, AddressList, - BlockId, BlockRange, ChainSpec, Empty, Exclude, GetAddressUtxosArg, RawTransaction, + BlockId, BlockRange, ChainSpec, Empty, GetAddressUtxosArg, TransparentAddressBlockFilter, TxFilter, }, zebra_skip_lightwalletd_tests, @@ -103,8 +103,12 @@ pub async fn run() -> Result<()> { // Launch zebra using a predefined zebrad state path // // TODO: change debug_skip_parameter_preload to true if we do the mempool test in the send transaction test - let (mut zebrad, zebra_rpc_address) = - spawn_zebrad_for_rpc_without_initial_peers(network, zebrad_state_path.unwrap(), test_type, false)?; + let (mut zebrad, zebra_rpc_address) = spawn_zebrad_for_rpc_without_initial_peers( + network, + zebrad_state_path.unwrap(), + test_type, + false, + )?; tracing::info!( ?zebra_rpc_address, @@ -363,46 +367,5 @@ pub async fn run() -> Result<()> { // Make sure the subversion field is zebra the user agent assert_eq!(lightd_info.zcashd_subversion, USER_AGENT); - // Transactions in mempool - - let transaction = Transaction::zcash_deserialize(&zebra_test::vectors::DUMMY_TX3[..]).unwrap(); - let transaction_bytes = transaction.zcash_serialize_to_vec().unwrap(); - - // Send transaction by calling `SendTransaction` - let request1 = RawTransaction { - data: transaction_bytes, - height: -1, - }; - let _ = rpc_client.send_transaction(request1).await?.into_inner(); - - // Wait a bit to query the mempool - tokio::time::sleep(std::time::Duration::from_secs(10)).await; - - // Call `GetMempoolTx` and get a stream of transactions - let mut transactions_stream = rpc_client - .get_mempool_tx(Exclude { txid: vec![] }) - .await? - .into_inner(); - - // Make sure our transaction, and only our transaction was inserted to the mempool. - let mut counter = 0; - while let Some(tx) = transactions_stream.message().await? { - assert_eq!(tx.hash, transaction.hash().bytes_in_display_order()); - counter += 1; - } - - assert_eq!(counter, 1); - - // Get the mempool by calling `GetMempoolStream`. - let mut transaction_stream = rpc_client.get_mempool_stream(Empty {}).await?.into_inner(); - - let mut counter = 0; - while let Some(_tx) = transaction_stream.message().await? { - counter += 1; - } - - // This one is not working. - assert_eq!(counter, 0); - Ok(()) } From 09254728c1bb64093f865b29b512c6ef5ae432c6 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 31 Aug 2022 15:29:05 +1000 Subject: [PATCH 16/25] Fix test log checks --- zebrad/src/components/mempool.rs | 10 +++++++--- .../tests/common/lightwalletd/send_transaction_test.rs | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/zebrad/src/components/mempool.rs b/zebrad/src/components/mempool.rs index 208c980ed09..166efed35f0 100644 --- a/zebrad/src/components/mempool.rs +++ b/zebrad/src/components/mempool.rs @@ -390,7 +390,7 @@ impl Service for Mempool { // Send transactions that were not rejected nor expired to peers if !send_to_peers_ids.is_empty() { - tracing::info!(?send_to_peers_ids, "sending new transactions to peers"); + tracing::trace!(?send_to_peers_ids, "sending new transactions to peers"); self.transaction_sender.send(send_to_peers_ids)?; } @@ -416,9 +416,13 @@ impl Service for Mempool { Request::TransactionIds => { trace!(?req, "got mempool request"); - let res = storage.tx_ids().collect(); + let res: HashSet<_> = storage.tx_ids().collect(); - trace!(?req, ?res, "answered mempool request"); + // This log line is checked by tests, + // because lightwalletd doesn't return mempool transactions at the moment. + // + // TODO: downgrade to trace level when we can check transactions via gRPC + info!(?req, res_count = ?res.len(), "answered mempool request"); async move { Ok(Response::TransactionIds(res)) }.boxed() } diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index b4ee30d40f0..6c8f8465049 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -177,7 +177,7 @@ pub async fn run() -> Result<()> { } tracing::info!("waiting for mempool to verify some transactions..."); - zebrad.expect_stdout_line_matches("sending new transactions to peers")?; + zebrad.expect_stdout_line_matches("sending mempool transaction broadcast")?; tracing::info!("calling GetMempoolTx gRPC to fetch transactions..."); let mut transactions_stream = rpc_client From defdd8340f139fde5ca1204b0be48764925cd5dc Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 1 Sep 2022 06:25:13 +1000 Subject: [PATCH 17/25] Fix some rustdoc warnings --- zebra-chain/src/block/hash.rs | 2 +- zebra-chain/src/transaction/hash.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zebra-chain/src/block/hash.rs b/zebra-chain/src/block/hash.rs index 741ccc724f1..045265ad69c 100644 --- a/zebra-chain/src/block/hash.rs +++ b/zebra-chain/src/block/hash.rs @@ -35,7 +35,7 @@ impl Hash { reversed_bytes } - /// Convert bytes in big-endian byte-order into a [`Hash`]. + /// Convert bytes in big-endian byte-order into a [`block::Hash`](crate::block::Hash). /// /// Zebra displays transaction and block hashes in big-endian byte-order, /// following the u256 convention set by Bitcoin and zcashd. diff --git a/zebra-chain/src/transaction/hash.rs b/zebra-chain/src/transaction/hash.rs index b6ad5efd9cf..2525c1d955c 100644 --- a/zebra-chain/src/transaction/hash.rs +++ b/zebra-chain/src/transaction/hash.rs @@ -115,7 +115,7 @@ impl Hash { reversed_bytes } - /// Convert bytes in big-endian byte-order into a [`Hash`]. + /// Convert bytes in big-endian byte-order into a [`transaction::Hash`](crate::transaction::Hash). /// /// Zebra displays transaction and block hashes in big-endian byte-order, /// following the u256 convention set by Bitcoin and zcashd. From 234067a5d32c7ca7c4d089b6fa20cea5c5a98b84 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 1 Sep 2022 06:28:58 +1000 Subject: [PATCH 18/25] Fix a compilation error due to new function arguments --- zebra-rpc/src/methods/tests/prop.rs | 42 +++++++++++++++++++---------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/zebra-rpc/src/methods/tests/prop.rs b/zebra-rpc/src/methods/tests/prop.rs index f4152658bed..01692c343d0 100644 --- a/zebra-rpc/src/methods/tests/prop.rs +++ b/zebra-rpc/src/methods/tests/prop.rs @@ -41,10 +41,11 @@ proptest! { let mut state: MockService<_, _, _, BoxError> = MockService::build().for_prop_tests(); let (rpc, rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", + Mainnet, + false, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, - Mainnet, ); let hash = SentTransactionHash(transaction.hash()); @@ -93,10 +94,11 @@ proptest! { let (rpc, rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", + Mainnet, + false, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, - Mainnet, ); let transaction_bytes = transaction @@ -150,10 +152,11 @@ proptest! { let (rpc, rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", + Mainnet, + false, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, - Mainnet, ); let transaction_bytes = transaction @@ -215,10 +218,11 @@ proptest! { let (rpc, rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", + Mainnet, + false, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, - Mainnet, ); let send_task = tokio::spawn(rpc.send_raw_transaction(non_hex_string)); @@ -269,10 +273,11 @@ proptest! { let (rpc, rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", + Mainnet, + false, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, - Mainnet, ); let send_task = tokio::spawn(rpc.send_raw_transaction(hex::encode(random_bytes))); @@ -321,10 +326,11 @@ proptest! { let (rpc, rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", + Mainnet, + false, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, - Mainnet, ); let call_task = tokio::spawn(rpc.get_raw_mempool()); @@ -376,10 +382,11 @@ proptest! { let (rpc, rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", + Mainnet, + false, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, - Mainnet, ); let send_task = tokio::spawn(rpc.get_raw_transaction(non_hex_string, 0)); @@ -432,10 +439,11 @@ proptest! { let (rpc, rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", + Mainnet, + false, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, - Mainnet, ); let send_task = tokio::spawn(rpc.get_raw_transaction(hex::encode(random_bytes), 0)); @@ -477,10 +485,11 @@ proptest! { // look for an error with a `NoChainTip` let (rpc, rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", + network, + false, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, - network, ); let response = rpc.get_blockchain_info(); @@ -525,10 +534,11 @@ proptest! { // Start RPC with the mocked `ChainTip` let (rpc, rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", + network, + false, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), chain_tip, - network, ); let response = rpc.get_blockchain_info(); @@ -609,10 +619,11 @@ proptest! { runtime.block_on(async move { let (rpc, _rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", + network, + false, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), chain_tip, - network, ); // Build the future to call the RPC @@ -670,10 +681,11 @@ proptest! { runtime.block_on(async move { let (rpc, _rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", + network, + false, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), chain_tip, - network, ); let address_strings = AddressStrings { @@ -719,10 +731,11 @@ proptest! { let (rpc, rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", + Mainnet, + false, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, - Mainnet, ); // send a transaction @@ -806,10 +819,11 @@ proptest! { let (rpc, rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", + Mainnet, + false, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, - Mainnet, ); let mut transactions_hash_set = HashSet::new(); From 7bd93a7a18d10381c4d3fdc72e2e6e20e49bf0dc Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 1 Sep 2022 06:43:23 +1000 Subject: [PATCH 19/25] Make zebrad sync timeouts consistent and remove outdated code --- zebrad/tests/common/launch.rs | 26 ++++++++------------------ zebrad/tests/common/lightwalletd.rs | 11 ++++------- zebrad/tests/common/sync.rs | 3 +-- 3 files changed, 13 insertions(+), 27 deletions(-) diff --git a/zebrad/tests/common/launch.rs b/zebrad/tests/common/launch.rs index 2932de95004..881405fd918 100644 --- a/zebrad/tests/common/launch.rs +++ b/zebrad/tests/common/launch.rs @@ -23,7 +23,10 @@ use zebra_test::{ }; use zebrad::config::ZebradConfig; -use crate::common::lightwalletd::{random_known_rpc_port_config, LightwalletdTestType}; +use crate::common::{ + lightwalletd::{random_known_rpc_port_config, LightwalletdTestType}, + sync::FINISH_PARTIAL_SYNC_TIMEOUT, +}; /// After we launch `zebrad`, wait this long for the command to start up, /// take the actions expected by the tests, and log the expected logs. @@ -47,27 +50,14 @@ pub const BETWEEN_NODES_DELAY: Duration = Duration::from_secs(5); /// The amount of time we wait for lightwalletd to update to the tip. /// -/// `lightwalletd` takes about 90 minutes to fully sync, -/// and `zebrad` takes about 30 minutes to update to the tip. -/// -/// TODO: reduce to 20 minutes when `zebrad` sync performance improves -pub const LIGHTWALLETD_UPDATE_TIP_DELAY: Duration = Duration::from_secs(11 * 60 * 60); +/// `lightwalletd` takes about 30-60 minutes to fully sync, +/// and `zebrad` can take hours to update to the tip under load. +pub const LIGHTWALLETD_UPDATE_TIP_DELAY: Duration = FINISH_PARTIAL_SYNC_TIMEOUT; /// The amount of time we wait for lightwalletd to do a full sync to the tip. /// /// See [`LIGHTWALLETD_UPDATE_TIP_DELAY`] for details. -pub const LIGHTWALLETD_FULL_SYNC_TIP_DELAY: Duration = Duration::from_secs(11 * 60 * 60); - -/// The amount of extra time we wait for Zebra to sync to the tip, -/// after we ignore a lightwalletd failure. -/// -/// Since we restart `lightwalletd` after a hang, we allow time for another full `lightwalletd` sync. -/// -/// See [`LIGHTWALLETD_UPDATE_TIP_DELAY`] for details. -/// -/// TODO: remove this extra time when lightwalletd hangs are fixed -pub const ZEBRAD_EXTRA_DELAY_FOR_LIGHTWALLETD_WORKAROUND: Duration = - LIGHTWALLETD_FULL_SYNC_TIP_DELAY; +pub const LIGHTWALLETD_FULL_SYNC_TIP_DELAY: Duration = FINISH_PARTIAL_SYNC_TIMEOUT; /// Extension trait for methods on `tempfile::TempDir` for using it as a test /// directory for `zebrad`. diff --git a/zebrad/tests/common/lightwalletd.rs b/zebrad/tests/common/lightwalletd.rs index d740d397d91..1b71d8d681d 100644 --- a/zebrad/tests/common/lightwalletd.rs +++ b/zebrad/tests/common/lightwalletd.rs @@ -28,7 +28,7 @@ use super::{ }, launch::{ ZebradTestDirExt, LIGHTWALLETD_DELAY, LIGHTWALLETD_FULL_SYNC_TIP_DELAY, - LIGHTWALLETD_UPDATE_TIP_DELAY, ZEBRAD_EXTRA_DELAY_FOR_LIGHTWALLETD_WORKAROUND, + LIGHTWALLETD_UPDATE_TIP_DELAY, }, }; @@ -378,14 +378,11 @@ impl LightwalletdTestType { /// Returns the `zebrad` timeout for this test type. pub fn zebrad_timeout(&self) -> Duration { - let base_timeout = match self { + match self { LaunchWithEmptyState => LIGHTWALLETD_DELAY, FullSyncFromGenesis { .. } => LIGHTWALLETD_FULL_SYNC_TIP_DELAY, UpdateCachedState | UpdateZebraCachedStateNoRpc => LIGHTWALLETD_UPDATE_TIP_DELAY, - }; - - // If lightwalletd hangs and times out, Zebra needs a bit of extra time to finish - base_timeout + ZEBRAD_EXTRA_DELAY_FOR_LIGHTWALLETD_WORKAROUND + } } /// Returns the `lightwalletd` timeout for this test type. @@ -396,7 +393,7 @@ impl LightwalletdTestType { } // We use the same timeouts for zebrad and lightwalletd, - // because the tests swap between checking zebrad and lightwalletd. + // because the tests check zebrad and lightwalletd concurrently. match self { LaunchWithEmptyState => LIGHTWALLETD_DELAY, FullSyncFromGenesis { .. } => LIGHTWALLETD_FULL_SYNC_TIP_DELAY, diff --git a/zebrad/tests/common/sync.rs b/zebrad/tests/common/sync.rs index 871a1815585..0a130054cb4 100644 --- a/zebrad/tests/common/sync.rs +++ b/zebrad/tests/common/sync.rs @@ -72,8 +72,7 @@ pub const LARGE_CHECKPOINT_TIMEOUT: Duration = Duration::from_secs(180); /// The partially synchronized state is expected to be close to the tip, so this timeout can be /// lower than what's expected for a full synchronization. However, a value that's too short may /// cause the test to fail. -#[allow(dead_code)] -pub const FINISH_PARTIAL_SYNC_TIMEOUT: Duration = Duration::from_secs(60 * 60); +pub const FINISH_PARTIAL_SYNC_TIMEOUT: Duration = Duration::from_secs(11 * 60 * 60); /// The test sync height where we switch to using the default lookahead limit. /// From 7bae8657659e970678e8f0a62fd3d9a7b3bd1aef Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 1 Sep 2022 06:51:43 +1000 Subject: [PATCH 20/25] Document how to increase temporary directory space for tests --- zebrad/tests/acceptance.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index 54ebc22f491..50584b849b8 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -102,6 +102,16 @@ //! ``` //! //! Please refer to the documentation of each test for more information. +//! +//! ## Disk Space for Testing +//! +//! The full sync and lightwalletd tests with cached state expect a temporary directory with +//! at least 300 GB of disk space (2 copies of the full chain). To use another disk for the +//! temporary test files: +//! +//! ```sh +//! export TMPDIR=/path/to/disk/directory +//! ``` use std::{collections::HashSet, env, fs, panic, path::PathBuf, time::Duration}; From 94eb3b6647cabf708ab35a1243890277f6699422 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 2 Sep 2022 12:33:39 +1000 Subject: [PATCH 21/25] Stop checking for a log that doesn't always happen --- zebrad/tests/common/lightwalletd/send_transaction_test.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index 6c8f8465049..57738b189f6 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -185,7 +185,8 @@ pub async fn run() -> Result<()> { .await? .into_inner(); - zebrad.expect_stdout_line_matches("answered mempool request req=TransactionIds")?; + // We'd like to check that lightwalletd queries the mempool, but it looks like it doesn't do it after each GetMempoolTx request. + //zebrad.expect_stdout_line_matches("answered mempool request req=TransactionIds")?; // GetMempoolTx: make sure at least one of the transactions were inserted into the mempool. let mut counter = 0; From af4993135eb6327c1b316b9e85ea68bdbb654572 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 5 Sep 2022 10:17:14 +1000 Subject: [PATCH 22/25] Remove some commented-out code Co-authored-by: Alfredo Garcia --- zebrad/src/components/mempool.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/zebrad/src/components/mempool.rs b/zebrad/src/components/mempool.rs index 166efed35f0..6202aadd532 100644 --- a/zebrad/src/components/mempool.rs +++ b/zebrad/src/components/mempool.rs @@ -405,8 +405,6 @@ impl Service for Mempool { /// and will cause callers to disconnect from the remote peer. #[instrument(name = "mempool", skip(self, req))] fn call(&mut self, req: Request) -> Self::Future { - //let skip_full_validation = self.skip_full_validation(); - match &mut self.active_state { ActiveState::Enabled { storage, From f78dbfcc19f32f678d7778a519f4bb9ab8bb2a71 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 5 Sep 2022 10:18:51 +1000 Subject: [PATCH 23/25] Update a comment about run time Co-authored-by: Alfredo Garcia --- zebrad/tests/common/launch.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebrad/tests/common/launch.rs b/zebrad/tests/common/launch.rs index 881405fd918..dcbe2e4df19 100644 --- a/zebrad/tests/common/launch.rs +++ b/zebrad/tests/common/launch.rs @@ -50,7 +50,7 @@ pub const BETWEEN_NODES_DELAY: Duration = Duration::from_secs(5); /// The amount of time we wait for lightwalletd to update to the tip. /// -/// `lightwalletd` takes about 30-60 minutes to fully sync, +/// `lightwalletd` takes about 60-120 minutes to fully sync, /// and `zebrad` can take hours to update to the tip under load. pub const LIGHTWALLETD_UPDATE_TIP_DELAY: Duration = FINISH_PARTIAL_SYNC_TIMEOUT; From 46d9fc0aef7ad4bae3dd84c6de454115deb19cf2 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 5 Sep 2022 11:14:40 +1000 Subject: [PATCH 24/25] Add new config to new tests from the `main` branch --- zebra-rpc/src/server/tests/vectors.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zebra-rpc/src/server/tests/vectors.rs b/zebra-rpc/src/server/tests/vectors.rs index bb9ce71ba3b..fe1d90bb722 100644 --- a/zebra-rpc/src/server/tests/vectors.rs +++ b/zebra-rpc/src/server/tests/vectors.rs @@ -38,6 +38,7 @@ fn rpc_server_spawn(parallel_cpu_threads: bool) { let config = Config { listen_addr: Some(SocketAddrV4::new(Ipv4Addr::LOCALHOST, port).into()), parallel_cpu_threads: if parallel_cpu_threads { 2 } else { 1 }, + debug_force_finished_sync: false, }; let rt = tokio::runtime::Runtime::new().unwrap(); @@ -102,6 +103,7 @@ fn rpc_server_spawn_unallocated_port(parallel_cpu_threads: bool) { let config = Config { listen_addr: Some(SocketAddrV4::new(Ipv4Addr::LOCALHOST, port).into()), parallel_cpu_threads: if parallel_cpu_threads { 0 } else { 1 }, + debug_force_finished_sync: false, }; let rt = tokio::runtime::Runtime::new().unwrap(); @@ -151,6 +153,7 @@ fn rpc_server_spawn_port_conflict() { let config = Config { listen_addr: Some(SocketAddrV4::new(Ipv4Addr::LOCALHOST, port).into()), parallel_cpu_threads: 1, + debug_force_finished_sync: false, }; let rt = tokio::runtime::Runtime::new().unwrap(); @@ -242,6 +245,7 @@ fn rpc_server_spawn_port_conflict_parallel_auto() { let config = Config { listen_addr: Some(SocketAddrV4::new(Ipv4Addr::LOCALHOST, port).into()), parallel_cpu_threads: 2, + debug_force_finished_sync: false, }; let rt = tokio::runtime::Runtime::new().unwrap(); From 225e6ee1bd6b771e47822f4eebb317ad8edd85a9 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 5 Sep 2022 13:54:14 +1000 Subject: [PATCH 25/25] Add transactions to the list, rather than replacing the list with each new block --- .../lightwalletd/send_transaction_test.rs | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index 57738b189f6..35f20db4c77 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -296,21 +296,38 @@ async fn load_transactions_from_block_after( assert!( tip_height > height, - "Chain not synchronized to a block after the specified height" + "Chain not synchronized to a block after the specified height", ); let mut target_height = height.0; let mut transactions = Vec::new(); while transactions.len() < max_sent_transactions() { - transactions = + let new_transactions = load_transactions_from_block(block::Height(target_height), &mut state).await?; - transactions.retain(|transaction| !transaction.is_coinbase()); + if let Some(mut new_transactions) = new_transactions { + new_transactions.retain(|transaction| !transaction.is_coinbase()); + transactions.append(&mut new_transactions); + } else { + tracing::info!( + "Reached the end of the finalized chain\n\ + collected {} transactions from {} blocks before {target_height:?}", + transactions.len(), + target_height - height.0 - 1, + ); + break; + } target_height += 1; } + tracing::info!( + "Collected {} transactions from {} blocks before {target_height:?}", + transactions.len(), + target_height - height.0 - 1, + ); + Ok(transactions) } @@ -320,7 +337,7 @@ async fn load_transactions_from_block_after( async fn load_transactions_from_block( height: block::Height, state: &mut ReadStateService, -) -> Result>> +) -> Result>>> where ReadStateService: Service< zebra_state::ReadRequest, @@ -339,12 +356,15 @@ where let block = match response { zebra_state::ReadResponse::Block(Some(block)) => block, zebra_state::ReadResponse::Block(None) => { - panic!("Missing block at {height:?} from state") + tracing::info!( + "Reached the end of the finalized chain, state is missing block at {height:?}", + ); + return Ok(None); } _ => unreachable!("Incorrect response from state service: {response:?}"), }; - Ok(block.transactions.to_vec()) + Ok(Some(block.transactions.to_vec())) } /// Prepare a request to send to lightwalletd that contains a transaction to be sent.