From b7627c4cf8e109dfc80095c5c58f4cf082b56e4d Mon Sep 17 00:00:00 2001 From: CrocdileChan <1576710154@qq.com> Date: Mon, 21 Oct 2019 15:25:50 +0800 Subject: [PATCH] use ThreadPool to execute spawn_worker(fn) (#3836) * use ThreadPool to spawn_worker() * use ThreadPool to implement spawn_worker(fn) * use ThreadPool to implement spawn_worker(f) * update [dependencies] threadpool and num_cpus version * rm 'extern crate num_cpus' * cargo.lock update * merge the newest cargo.lock * Update Cargo.lock * use Mutex to wrap OffchainWorkers.thread_pool * format use crate * use parking_lot::Mutex instead of std::sync::Mutex --- Cargo.lock | 11 +++++++++++ core/offchain/Cargo.toml | 2 ++ core/offchain/src/lib.rs | 29 ++++++++++++++++------------- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aab1866999df0..2e30793e47b7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5316,6 +5316,7 @@ dependencies = [ "hyper 0.12.35 (registry+https://github.com/rust-lang/crates.io-index)", "hyper-tls 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -5328,6 +5329,7 @@ dependencies = [ "substrate-primitives 2.0.0", "substrate-test-runtime-client 2.0.0", "substrate-transaction-pool 2.0.0", + "threadpool 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -5906,6 +5908,14 @@ dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "threadpool" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "time" version = "0.1.42" @@ -7176,6 +7186,7 @@ dependencies = [ "checksum termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "96d6098003bde162e4277c70665bd87c326f5a0c3f3fbfb285787fa482d54e6e" "checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" +"checksum threadpool 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e2f0c90a5f3459330ac8bc0d2f879c693bb7a2f59689c1083fc4ef83834da865" "checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" "checksum tiny-bip39 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c1c5676413eaeb1ea35300a0224416f57abc3bd251657e0fafc12c47ff98c060" "checksum tiny-keccak 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d8a021c69bb74a44ccedb824a046447e2c84a01df9e5c20779750acb38e11b2" diff --git a/core/offchain/Cargo.toml b/core/offchain/Cargo.toml index ca9b27eee2fc7..678d943c597ce 100644 --- a/core/offchain/Cargo.toml +++ b/core/offchain/Cargo.toml @@ -16,6 +16,8 @@ futures-timer = "0.4.0" hyper = "0.12.35" hyper-tls = "0.3.2" log = "0.4.8" +threadpool = "1.7" +num_cpus = "1.10" offchain-primitives = { package = "substrate-offchain-primitives", path = "./primitives" } codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] } parking_lot = "0.9.0" diff --git a/core/offchain/src/lib.rs b/core/offchain/src/lib.rs index 79c6df04ea109..a335ca53807ab 100644 --- a/core/offchain/src/lib.rs +++ b/core/offchain/src/lib.rs @@ -39,6 +39,8 @@ use std::{ sync::Arc, }; +use parking_lot::Mutex; +use threadpool::ThreadPool; use client::runtime_api::ApiExt; use futures::future::Future; use log::{debug, warn}; @@ -58,6 +60,7 @@ pub struct OffchainWorkers { client: Arc, db: Storage, _block: PhantomData, + thread_pool: Mutex, } impl OffchainWorkers { @@ -67,6 +70,7 @@ impl OffchainWorkers OffchainWorkers< debug!("Spawning offchain workers at {:?}", at); let number = *number; let client = self.client.clone(); - spawn_worker(move || { + self.spawn_worker(move || { let runtime = client.runtime_api(); let api = Box::new(api); debug!("Running offchain workers at {:?}", at); @@ -134,19 +138,18 @@ impl OffchainWorkers< futures::future::Either::Right(futures::future::ready(())) } } -} -/// Spawns a new offchain worker. -/// -/// We spawn offchain workers for each block in a separate thread, -/// since they can run for a significant amount of time -/// in a blocking fashion and we don't want to block the runtime. -/// -/// Note that we should avoid that if we switch to future-based runtime in the future, -/// alternatively: -/// TODO [ToDr] (#1458) we can consider using a thread pool instead. -fn spawn_worker(f: impl FnOnce() -> () + Send + 'static) { - std::thread::spawn(f); + /// Spawns a new offchain worker. + /// + /// We spawn offchain workers for each block in a separate thread, + /// since they can run for a significant amount of time + /// in a blocking fashion and we don't want to block the runtime. + /// + /// Note that we should avoid that if we switch to future-based runtime in the future, + /// alternatively: + fn spawn_worker(&self, f: impl FnOnce() -> () + Send + 'static) { + self.thread_pool.lock().execute(f); + } } #[cfg(test)]