From 45fcb58372b7cc5154743ec09e8e6044cd0cf9c8 Mon Sep 17 00:00:00 2001 From: coderofstuff <114628839+coderofstuff@users.noreply.github.com> Date: Thu, 4 Jan 2024 00:13:30 +0800 Subject: [PATCH] Cherry-picked Optimized Rothschild performance on high TPS (#371) --- Cargo.lock | 5 + consensus/core/src/tx.rs | 25 +- rothschild/Cargo.toml | 10 + rothschild/benches/bench.rs | 50 ++++ rothschild/src/main.rs | 240 ++++++++++++++---- rpc/grpc/client/Cargo.toml | 8 +- .../grpc/client/src}/client_pool.rs | 2 +- rpc/grpc/client/src/lib.rs | 3 + simpa/src/simulator/miner.rs | 8 +- testing/integration/src/common/daemon.rs | 2 +- testing/integration/src/common/mod.rs | 1 - 11 files changed, 284 insertions(+), 70 deletions(-) create mode 100644 rothschild/benches/bench.rs rename {testing/integration/src/common => rpc/grpc/client/src}/client_pool.rs (98%) diff --git a/Cargo.lock b/Cargo.lock index 9b08c3f4ff..e4210f28df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2385,6 +2385,7 @@ dependencies = [ "async-trait", "faster-hex 0.6.1", "futures", + "futures-util", "h2", "kash-core", "kash-grpc-core", @@ -4494,7 +4495,9 @@ dependencies = [ name = "rothschild" version = "0.13.1" dependencies = [ + "async-channel 2.1.1", "clap 4.4.11", + "criterion", "faster-hex 0.6.1", "itertools 0.11.0", "kash-addresses", @@ -4505,6 +4508,8 @@ dependencies = [ "kash-txscript", "kash-utils", "log", + "parking_lot", + "rayon", "secp256k1", "tokio", ] diff --git a/consensus/core/src/tx.rs b/consensus/core/src/tx.rs index 154d01d987..adf8b7e630 100644 --- a/consensus/core/src/tx.rs +++ b/consensus/core/src/tx.rs @@ -275,20 +275,23 @@ impl Transaction { gas: u64, payload: Vec, ) -> Self { - let mut tx = Self { - version, - inputs, - outputs, - action, - lock_time, - subnetwork_id, - gas, - payload, - id: Default::default(), // Temp init before the finalize below - }; + let mut tx = Self::new_non_finalized(version, inputs, outputs, action, lock_time, subnetwork_id, gas, payload); tx.finalize(); tx } + + pub fn new_non_finalized( + version: u16, + inputs: Vec, + outputs: Vec, + action: TransactionAction, + lock_time: u64, + subnetwork_id: SubnetworkId, + gas: u64, + payload: Vec, + ) -> Self { + Self { version, inputs, outputs, action, lock_time, subnetwork_id, gas, payload, id: Default::default() } + } } impl Transaction { diff --git a/rothschild/Cargo.toml b/rothschild/Cargo.toml index 8b4b439d2c..70fa1d6566 100644 --- a/rothschild/Cargo.toml +++ b/rothschild/Cargo.toml @@ -15,10 +15,20 @@ kash-rpc-core.workspace = true kash-addresses.workspace = true kash-txscript.workspace = true kash-utils.workspace = true +async-channel.workspace = true +parking_lot.workspace = true clap.workspace = true faster-hex.workspace = true itertools.workspace = true log.workspace = true +rayon.workspace = true secp256k1 = { workspace = true, features = ["global-context", "rand-std"] } tokio = { workspace = true, features = ["rt", "macros", "rt-multi-thread"] } + +[dev-dependencies] +criterion.workspace = true + +[[bench]] +name = "bench" +harness = false diff --git a/rothschild/benches/bench.rs b/rothschild/benches/bench.rs new file mode 100644 index 0000000000..d9a3ea5fd7 --- /dev/null +++ b/rothschild/benches/bench.rs @@ -0,0 +1,50 @@ +use criterion::{criterion_group, criterion_main, Criterion}; +use rayon::prelude::*; + +use kash_consensus_core::asset_type::AssetType::KSH; +use kash_consensus_core::tx::TransactionAction::TransferKSH; +use kash_consensus_core::{ + constants::TX_VERSION, + subnets::SUBNETWORK_ID_NATIVE, + tx::{ScriptPublicKey, Transaction, TransactionInput, TransactionOutpoint, TransactionOutput}, + Hash, +}; + +fn constuct_tx() -> Transaction { + let inputs = vec![TransactionInput { + previous_outpoint: TransactionOutpoint { transaction_id: Hash::from_bytes([0xFF; 32]), index: 0 }, + signature_script: vec![], + sequence: 0, + sig_op_count: 1, + }]; + let outputs = + vec![TransactionOutput { value: 10000, asset_type: KSH, script_public_key: ScriptPublicKey::from_vec(0, vec![0xff; 35]) }]; + Transaction::new(TX_VERSION, inputs, outputs, TransferKSH, 0, SUBNETWORK_ID_NATIVE, 0, vec![]) +} + +fn construct_txs_serially() { + let _ = (0..10000) + .map(|_| { + constuct_tx(); + }) + .collect::>(); +} + +fn construct_txs_parallel() { + let _ = (0..10000) + .into_par_iter() + .map(|_| { + constuct_tx(); + }) + .collect::>(); +} + +pub fn bench_compare_tx_generation(c: &mut Criterion) { + let mut group = c.benchmark_group("compare txs"); + group.bench_function("Transaction::SerialCreation", |b| b.iter(construct_txs_serially)); + group.bench_function("Transaction::ParallelCreation", |b| b.iter(construct_txs_parallel)); + group.finish(); +} + +criterion_group!(benches, bench_compare_tx_generation); +criterion_main!(benches); diff --git a/rothschild/src/main.rs b/rothschild/src/main.rs index e05c238ba4..e3a1c1b94a 100644 --- a/rothschild/src/main.rs +++ b/rothschild/src/main.rs @@ -1,6 +1,6 @@ -use std::{collections::HashMap, time::Duration}; +use std::{collections::HashMap, sync::Arc, time::Duration}; -use clap::{Arg, Command}; +use clap::{Arg, ArgAction, Command}; use itertools::Itertools; use kash_addresses::Address; use kash_consensus_core::asset_type::AssetType::KSH; @@ -13,15 +13,17 @@ use kash_consensus_core::{ tx::{MutableTransaction, Transaction, TransactionInput, TransactionOutpoint, TransactionOutput, UtxoEntry}, }; use kash_core::{info, kashd_env::version, time::unix_now, warn}; -use kash_grpc_client::GrpcClient; +use kash_grpc_client::{ClientPool, GrpcClient}; use kash_rpc_core::{api::rpc::RpcApi, notify::mode::NotificationMode}; use kash_txscript::pay_to_address_script; +use parking_lot::Mutex; +use rayon::prelude::*; use secp256k1::{rand::thread_rng, KeyPair}; use tokio::time::{interval, MissedTickBehavior}; const DEFAULT_SEND_AMOUNT: u64 = 10_000; - const FEE_PER_MASS: u64 = 10; +const MILLIS_PER_TICK: u64 = 10; struct Stats { num_txs: usize, @@ -35,6 +37,8 @@ pub struct Args { pub private_key: Option, pub tps: u64, pub rpc_server: String, + pub threads: u8, + pub unleashed: bool, } impl Args { @@ -44,6 +48,8 @@ impl Args { private_key: m.get_one::("private-key").cloned(), tps: m.get_one::("tps").cloned().unwrap(), rpc_server: m.get_one::("rpcserver").cloned().unwrap_or("localhost:16210".to_owned()), + threads: m.get_one::("threads").cloned().unwrap(), + unleashed: m.get_one::("unleashed").cloned().unwrap_or(false), } } } @@ -70,13 +76,36 @@ pub fn cli() -> Command { .default_value("localhost:16210") .help("RPC server"), ) + .arg( + Arg::new("threads") + .long("threads") + .default_value("2") + .value_parser(clap::value_parser!(u8)) + .help("The number of threads to use for TX generation. Set to 0 to use 1 thread per core. Default is 2."), + ) + .arg(Arg::new("unleashed").long("unleashed").action(ArgAction::SetTrue).hide(true).help("Allow higher TPS")) +} + +async fn new_rpc_client(address: &str) -> GrpcClient { + GrpcClient::connect(NotificationMode::Direct, format!("grpc://{}", address), true, None, false, Some(500_000), Default::default()) + .await + .unwrap() +} + +struct ClientPoolArg { + tx: Transaction, + stats: Arc>, + selected_utxos_len: usize, + selected_utxos_amount: u64, + pending_len: usize, + utxos_len: usize, } #[tokio::main] async fn main() { kash_core::log::init_logger(None, ""); let args = Args::parse(); - let mut stats = Stats { num_txs: 0, since: unix_now(), num_utxos: 0, utxos_amount: 0, num_outs: 0 }; + let stats = Arc::new(Mutex::new(Stats { num_txs: 0, since: unix_now(), num_utxos: 0, utxos_amount: 0, num_outs: 0 })); let rpc_client = GrpcClient::connect( NotificationMode::Direct, format!("grpc://{}", args.rpc_server), @@ -111,6 +140,8 @@ async fn main() { let kash_addr = Address::new(kash_addresses::Prefix::Testnet, kash_addresses::Version::PubKey, &schnorr_key.x_only_public_key().0.serialize()); + rayon::ThreadPoolBuilder::new().num_threads(args.threads as usize).build_global().unwrap(); + info!("Using Rothschild with private key {} and address {}", schnorr_key.display_secret(), String::from(&kash_addr)); let info = rpc_client.get_block_dag_info().await.unwrap(); let coinbase_maturity = match info.network.suffix { @@ -133,18 +164,89 @@ async fn main() { coinbase_maturity, ); + const CLIENT_POOL_SIZE: usize = 8; + let mut rpc_clients = Vec::with_capacity(CLIENT_POOL_SIZE); + for _ in 0..CLIENT_POOL_SIZE { + rpc_clients.push(Arc::new(new_rpc_client(&args.rpc_server).await)); + } + + let submit_tx_pool = ClientPool::new(rpc_clients, 1000, |c, arg: ClientPoolArg| async move { + let ClientPoolArg { tx, stats, selected_utxos_len, selected_utxos_amount, pending_len, utxos_len } = arg; + match c.submit_transaction(tx.as_ref().into(), false).await { + Ok(_) => { + let mut stats = stats.lock(); + stats.num_txs += 1; + stats.num_utxos += selected_utxos_len; + stats.utxos_amount += selected_utxos_amount; + stats.num_outs += tx.outputs.len(); + let now = unix_now(); + let time_past = now - stats.since; + if time_past > 10_000 { + info!( + "Tx rate: {:.1}/sec, avg UTXO amount: {}, avg UTXOs per tx: {}, avg outs per tx: {}, estimated available UTXOs: {}", + 1000f64 * (stats.num_txs as f64) / (time_past as f64), + (stats.utxos_amount / stats.num_utxos as u64), + stats.num_utxos / stats.num_txs, + stats.num_outs / stats.num_txs, + if utxos_len > pending_len { utxos_len - pending_len } else { 0 }, + ); + stats.since = now; + stats.num_txs = 0; + stats.num_utxos = 0; + stats.utxos_amount = 0; + stats.num_outs = 0; + } + } + Err(e) => { + let mut tx = tx; + tx.finalize(); + warn!("RPC error when submitting {}: {}", tx.id(), e); + } + } + false + }); + let tx_sender = submit_tx_pool.sender(); + + let target_tps = args.tps.min(if args.unleashed { u64::MAX } else { 100 }); + let should_tick_per_second = target_tps * MILLIS_PER_TICK / 1000 == 0; + let avg_txs_per_tick = if should_tick_per_second { target_tps } else { target_tps * MILLIS_PER_TICK / 1000 }; let mut utxos = refresh_utxos(&rpc_client, kash_addr.clone(), &mut pending, coinbase_maturity).await; - let mut ticker = interval(Duration::from_secs_f64(1.0 / (args.tps.min(100) as f64))); + let mut ticker = interval(Duration::from_millis(if should_tick_per_second { 1000 } else { MILLIS_PER_TICK })); ticker.set_missed_tick_behavior(MissedTickBehavior::Delay); let mut maximize_inputs = false; let mut last_refresh = unix_now(); + // This allows us to keep track of the UTXOs we already tried to use for this period + // until the UTXOs are refreshed. At that point, this will be reset as well. + let mut next_available_utxo_index = 0; + // Tracker so we can try to send as close as possible to the target TPS + let mut remaining_txs_in_interval = target_tps; + loop { ticker.tick().await; maximize_inputs = should_maximize_inputs(maximize_inputs, &utxos, &pending); + let txs_to_send = if remaining_txs_in_interval > avg_txs_per_tick * 2 { + remaining_txs_in_interval -= avg_txs_per_tick; + avg_txs_per_tick + } else { + let count = remaining_txs_in_interval; + remaining_txs_in_interval = target_tps; + count + }; + let now = unix_now(); - let has_funds = - maybe_send_tx(&rpc_client, kash_addr.clone(), &mut utxos, &mut pending, schnorr_key, &mut stats, maximize_inputs).await; + let has_funds = maybe_send_tx( + txs_to_send, + &tx_sender, + kash_addr.clone(), + &mut utxos, + &mut pending, + schnorr_key, + stats.clone(), + maximize_inputs, + &mut next_available_utxo_index, + ) + .await; if !has_funds { info!("Has not enough funds"); } @@ -153,6 +255,7 @@ async fn main() { tokio::time::sleep(Duration::from_millis(100)).await; // We don't want this operation to be too frequent since its heavy on the node, so we wait some time before executing it. utxos = refresh_utxos(&rpc_client, kash_addr.clone(), &mut pending, coinbase_maturity).await; last_refresh = unix_now(); + next_available_utxo_index = 0; pause_if_mempool_is_full(&rpc_client).await; } clean_old_pending_outpoints(&mut pending); @@ -179,7 +282,7 @@ fn should_maximize_inputs( async fn pause_if_mempool_is_full(rpc_client: &GrpcClient) { loop { let mempool_size = rpc_client.get_info().await.unwrap().mempool_size; - if mempool_size < 10_000 { + if mempool_size < 200_000 { break; } @@ -196,7 +299,7 @@ async fn refresh_utxos( coinbase_maturity: u64, ) -> Vec<(TransactionOutpoint, UtxoEntry)> { populate_pending_outpoints_from_mempool(rpc_client, kash_addr.clone(), pending).await; - fetch_spendable_utxos(rpc_client, kash_addr, coinbase_maturity).await + fetch_spendable_utxos(rpc_client, kash_addr, coinbase_maturity, pending).await } async fn populate_pending_outpoints_from_mempool( @@ -219,12 +322,16 @@ async fn fetch_spendable_utxos( rpc_client: &GrpcClient, kash_addr: Address, coinbase_maturity: u64, + pending: &mut HashMap, ) -> Vec<(TransactionOutpoint, UtxoEntry)> { let resp = rpc_client.get_utxos_by_addresses(vec![kash_addr]).await.unwrap(); let dag_info = rpc_client.get_block_dag_info().await.unwrap(); let mut utxos = Vec::with_capacity(resp.len()); - for resp_entry in - resp.into_iter().filter(|resp_entry| is_utxo_spendable(&resp_entry.utxo_entry, dag_info.virtual_daa_score, coinbase_maturity)) + for resp_entry in resp + .into_iter() + .filter(|resp_entry| is_utxo_spendable(&resp_entry.utxo_entry, dag_info.virtual_daa_score, coinbase_maturity)) + // Eliminates UTXOs we already tried to spend so we don't try to spend them again in this period + .filter(|utxo| !pending.contains_key(&utxo.outpoint)) { utxos.push((resp_entry.outpoint, resp_entry.utxo_entry)); } @@ -242,55 +349,70 @@ fn is_utxo_spendable(entry: &UtxoEntry, virtual_daa_score: u64, coinbase_maturit } async fn maybe_send_tx( - rpc_client: &GrpcClient, + txs_to_send: u64, + tx_sender: &async_channel::Sender, kash_addr: Address, utxos: &mut Vec<(TransactionOutpoint, UtxoEntry)>, pending: &mut HashMap, schnorr_key: KeyPair, - stats: &mut Stats, + stats: Arc>, maximize_inputs: bool, + next_available_utxo_index: &mut usize, ) -> bool { let num_outs = if maximize_inputs { 1 } else { 2 }; - let (selected_utxos, selected_amount) = select_utxos(utxos, DEFAULT_SEND_AMOUNT, num_outs, maximize_inputs, pending); - if selected_amount == 0 { - return false; - } - let tx = generate_tx(schnorr_key, &selected_utxos, selected_amount, num_outs, &kash_addr); + let mut has_fund = false; - let now = unix_now(); - for input in tx.inputs.iter() { - pending.insert(input.previous_outpoint, now); - } + let selected_utxos_groups = (0..txs_to_send) + .map(|_| { + let (selected_utxos, selected_amount) = + select_utxos(utxos, DEFAULT_SEND_AMOUNT, num_outs, maximize_inputs, next_available_utxo_index); + if selected_amount == 0 { + return None; + } - match rpc_client.submit_transaction((&tx).into(), false).await { - Ok(_) => {} - Err(e) => { - warn!("RPC error: {}", e); - return true; - } + // If any iteration successfully selected UTXOs, we assume to still + // have funds in this tick + has_fund = true; + + let now = unix_now(); + for input in selected_utxos.iter() { + pending.insert(input.0, now); + } + + Some((selected_utxos, selected_amount)) + }) + .collect::>(); + + if !has_fund { + return false; } - stats.num_txs += 1; - stats.num_utxos += selected_utxos.len(); - stats.utxos_amount += selected_utxos.into_iter().map(|(_, entry)| entry.amount).sum::(); - stats.num_outs += tx.outputs.len(); - let now = unix_now(); - let time_past = now - stats.since; - if time_past > 10_000 { - info!( - "Tx rate: {:.1}/sec, avg UTXO amount: {}, avg UTXOs per tx: {}, avg outs per tx: {}, estimated available UTXOs: {}", - 1000f64 * (stats.num_txs as f64) / (time_past as f64), - (stats.utxos_amount / stats.num_utxos as u64), - stats.num_utxos / stats.num_txs, - stats.num_outs / stats.num_txs, - if utxos.len() > pending.len() { utxos.len() - pending.len() } else { 0 }, - ); - stats.since = now; - stats.num_txs = 0; - stats.num_utxos = 0; - stats.utxos_amount = 0; - stats.num_outs = 0; + let txs = selected_utxos_groups + .into_par_iter() + .map(|utxo_option| { + if let Some((selected_utxos, selected_amount)) = utxo_option { + let tx = generate_tx(schnorr_key, &selected_utxos, selected_amount, num_outs, &kash_addr); + + return Some((tx, selected_utxos.len(), selected_utxos.into_iter().map(|(_, entry)| entry.amount).sum::())); + } + + None + }) + .collect::>(); + + for (tx, selected_utxos_len, selected_utxos_amount) in txs.into_iter().flatten() { + tx_sender + .send(ClientPoolArg { + tx, + stats: stats.clone(), + selected_utxos_len, + selected_utxos_amount, + pending_len: pending.len(), + utxos_len: utxos.len(), + }) + .await + .unwrap(); } true @@ -328,8 +450,16 @@ fn generate_tx( let outputs = (0..num_outs) .map(|_| TransactionOutput { value: send_amount / num_outs, script_public_key: script_public_key.clone(), asset_type: KSH }) .collect_vec(); - let unsigned_tx = - Transaction::new(TX_VERSION, inputs, outputs, TransactionAction::TransferKSH, 0, SUBNETWORK_ID_NATIVE, 0, vec![]); + let unsigned_tx = Transaction::new_non_finalized( + TX_VERSION, + inputs, + outputs, + TransactionAction::TransferKSH, + 0, + SUBNETWORK_ID_NATIVE, + 0, + vec![], + ); let signed_tx = sign(MutableTransaction::with_entries(unsigned_tx, utxos.iter().map(|(_, entry)| entry.clone()).collect_vec()), schnorr_key); signed_tx.tx @@ -340,17 +470,21 @@ fn select_utxos( min_amount: u64, num_outs: u64, maximize_utxos: bool, - pending: &HashMap, + next_available_utxo_index: &mut usize, ) -> (Vec<(TransactionOutpoint, UtxoEntry)>, u64) { const MAX_UTXOS: usize = 84; let mut selected_amount: u64 = 0; let mut selected = Vec::new(); - for (outpoint, entry) in utxos.iter().filter(|(op, _)| !pending.contains_key(op)).cloned() { + + while next_available_utxo_index < &mut utxos.len() { + let (outpoint, entry) = utxos[*next_available_utxo_index].clone(); selected_amount += entry.amount; selected.push((outpoint, entry)); let fee = required_fee(selected.len(), num_outs); + *next_available_utxo_index += 1; + if selected_amount >= min_amount + fee && (!maximize_utxos || selected.len() == MAX_UTXOS) { return (selected, selected_amount - fee); } diff --git a/rpc/grpc/client/Cargo.toml b/rpc/grpc/client/Cargo.toml index 4b766df053..7adcec727b 100644 --- a/rpc/grpc/client/Cargo.toml +++ b/rpc/grpc/client/Cargo.toml @@ -27,7 +27,13 @@ prost.workspace = true rand.workspace = true regex.workspace = true thiserror.workspace = true -tokio = { workspace = true, features = ["rt-multi-thread", "macros", "sync", "time"] } +tokio = { workspace = true, features = [ + "rt-multi-thread", + "macros", + "sync", + "time", +] } tokio-stream.workspace = true tonic = { workspace = true, features = ["gzip"] } triggered.workspace = true +futures-util.workspace = true diff --git a/testing/integration/src/common/client_pool.rs b/rpc/grpc/client/src/client_pool.rs similarity index 98% rename from testing/integration/src/common/client_pool.rs rename to rpc/grpc/client/src/client_pool.rs index d0fc7d2bcf..4f5ab4a148 100644 --- a/testing/integration/src/common/client_pool.rs +++ b/rpc/grpc/client/src/client_pool.rs @@ -1,7 +1,7 @@ +use super::GrpcClient; use async_channel::{SendError, Sender}; use futures_util::Future; use kash_core::trace; -use kash_grpc_client::GrpcClient; use kash_utils::{any::type_name_short, channel::Channel}; use std::sync::Arc; use tokio::task::JoinHandle; diff --git a/rpc/grpc/client/src/lib.rs b/rpc/grpc/client/src/lib.rs index f8d9666170..adf5b4e78d 100644 --- a/rpc/grpc/client/src/lib.rs +++ b/rpc/grpc/client/src/lib.rs @@ -4,6 +4,7 @@ use self::{ }; use async_channel::{Receiver, Sender}; use async_trait::async_trait; +pub use client_pool::ClientPool; use connection_event::ConnectionEvent; use futures::{future::FutureExt, pin_mut, select}; use kash_core::{debug, trace}; @@ -55,6 +56,8 @@ mod resolver; #[macro_use] mod route; +mod client_pool; + pub type GrpcClientCollector = CollectorFrom; pub type GrpcClientNotify = DynNotify; pub type GrpcClientNotifier = Notifier; diff --git a/simpa/src/simulator/miner.rs b/simpa/src/simulator/miner.rs index 156b70142c..ce8f0b2b75 100644 --- a/simpa/src/simulator/miner.rs +++ b/simpa/src/simulator/miner.rs @@ -143,7 +143,11 @@ impl Miner { .take(self.target_txs_per_block as usize) .collect::>() .into_par_iter() - .map(|mutable_tx| sign(mutable_tx, schnorr_key).tx) + .map(|mutable_tx| { + let mut signed_tx = sign(mutable_tx, schnorr_key).tx; + signed_tx.finalize(); + signed_tx + }) .collect::>(); for outpoint in txs.iter().flat_map(|t| t.inputs.iter().map(|i| i.previous_outpoint)) { @@ -171,7 +175,7 @@ impl Miner { // Creates an unsigned transaction for simulation purposes. fn create_unsigned_tx(&self, outpoint: TransactionOutpoint, input_amount: u64, multiple_outputs: bool) -> Transaction { - Transaction::new( + Transaction::new_non_finalized( 0, vec![TransactionInput::new(outpoint, vec![], 0, 0)], if multiple_outputs && input_amount > 4 { diff --git a/testing/integration/src/common/daemon.rs b/testing/integration/src/common/daemon.rs index d506c1e060..f6fc6b979a 100644 --- a/testing/integration/src/common/daemon.rs +++ b/testing/integration/src/common/daemon.rs @@ -8,7 +8,7 @@ use kashd_lib::{args::Args, daemon::create_core_with_runtime}; use std::{sync::Arc, time::Duration}; use tempfile::TempDir; -use super::client_pool::ClientPool; +use kash_grpc_client::ClientPool; pub struct Daemon { // Type and suffix of the daemon network diff --git a/testing/integration/src/common/mod.rs b/testing/integration/src/common/mod.rs index 0767fb7286..663390eb90 100644 --- a/testing/integration/src/common/mod.rs +++ b/testing/integration/src/common/mod.rs @@ -5,7 +5,6 @@ use std::{ }; pub mod client_notify; -pub mod client_pool; pub mod daemon; pub mod utils;