WASM Local-blob override

client/cli/src/config.rs

@@ -42,6 +42,8 @@ pub(crate) const NODE_NAME_MAX_LENGTH: usize = 64;
 /// Default sub directory to store network config.
 pub(crate) const DEFAULT_NETWORK_CONFIG_PATH: &'static str = "network";
 
+pub(crate) const DEFAULT_WASM_OVERWRITE_PATH: &'static str = "wasm_runtime_overwrites";
+
 /// The recommended open file descriptor limit to be configured for the process.
 const RECOMMENDED_OPEN_FILE_DESCRIPTOR_LIMIT: u64 = 10_000;
 
@@ -277,6 +279,24 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
 			.map(|x| x.wasm_method())
 			.unwrap_or_default())
 	}
+
+	/// Check whether overwriting on-chain WASM is enabled.
+	///
+	/// By default overwriting on-chain WASM is disabled.
+	fn wasm_overwrite(&self) -> bool {
+		self.import_params()
+			.map(|x| x.wasm_overwrite())
+			.unwrap_or_default()
+	}
+
+	/// Get the path where WASM overwrites live.
+	///
+	/// By default this is set to ${config_dir}/chain/wasm_runtime_overwrites/
+	fn wasm_overwrite_path(&self) -> Option<PathBuf> {
+		self.import_params()
+			.map(|x| x.wasm_overwrite_path())
+			.unwrap_or_default()
+	}
 
 	/// Get the execution strategies.
 	///
@@ -459,6 +479,9 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
 			.join("chains")
 			.join(chain_spec.id());
 		let net_config_dir = config_dir.join(DEFAULT_NETWORK_CONFIG_PATH);
+		let wasm_overwrite_dir: PathBuf = self
+			.wasm_overwrite_path()
+			.unwrap_or(config_dir.join(DEFAULT_WASM_OVERWRITE_PATH));
 		let client_id = C::client_id();
 		let database_cache_size = self.database_cache_size()?.unwrap_or(128);
 		let database = self.database()?.unwrap_or(Database::RocksDb);
@@ -492,6 +515,8 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
 			state_cache_child_ratio: self.state_cache_child_ratio()?,
 			pruning: self.pruning(unsafe_pruning, &role)?,
 			wasm_method: self.wasm_method()?,
+			wasm_overwrite: self.wasm_overwrite(),
+			wasm_overwrite_path: wasm_overwrite_dir,
 			execution_strategies: self.execution_strategies(is_dev, is_validator)?,
 			rpc_http: self.rpc_http(DCV::rpc_http_listen_port())?,
 			rpc_ws: self.rpc_ws(DCV::rpc_ws_listen_port())?,

client/cli/src/params/import_params.rs

@@ -25,6 +25,7 @@ use crate::params::DatabaseParams;
 use crate::params::PruningParams;
 use sc_client_api::execution_extensions::ExecutionStrategies;
 use structopt::StructOpt;
+use std::path::PathBuf;
 
 /// Parameters for block import.
 #[derive(Debug, StructOpt)]
@@ -54,6 +55,15 @@ pub struct ImportParams {
 		default_value = "Interpreted"
 	)]
 	pub wasm_method: WasmExecutionMethod,
+
+	/// Enable overriding the on-chain WASM with local wasm on-disk.
+	#[structopt(long)]
+	pub wasm_overwrite: bool, 
+
+	/// Specify the path where local WASM runtimes are stored.
+	/// These runtimes will override on-chain runtimes when the version matches.
+	#[structopt(long, value_name = "PATH", parse(from_os_str))]
+	pub wasm_overwrite_path: Option<PathBuf>,
 
 	#[allow(missing_docs)]
 	#[structopt(flatten)]
@@ -102,6 +112,16 @@ impl ImportParams {
 	pub fn wasm_method(&self) -> sc_service::config::WasmExecutionMethod {
 		self.wasm_method.into()
 	}
+
+	/// Enable overwriting on-chain WASM with locally-stored WASM.
+	pub fn wasm_overwrite(&self) -> bool {
+		self.wasm_overwrite
+	}
+
+	/// Path where local WASM is stored.
+	pub fn wasm_overwrite_path(&self) -> Option<PathBuf> {
+		self.wasm_overwrite_path.clone()
+	}
 
 	/// Get execution strategies for the parameters
 	pub fn execution_strategies(&self, is_dev: bool, is_validator: bool) -> ExecutionStrategies {

client/service/src/builder.rs

@@ -303,8 +303,10 @@ pub fn new_full_parts<TBl, TRtApi, TExecDisp>(
 			Box::new(task_manager.spawn_handle()),
 			config.prometheus_config.as_ref().map(|config| config.registry.clone()),
 			ClientConfig {
-				offchain_worker_enabled : config.offchain_worker.enabled ,
+				offchain_worker_enabled : config.offchain_worker.enabled,
 				offchain_indexing_api: config.offchain_worker.indexing_enabled,
+				wasm_overwrite_enabled: config.wasm_overwrite,
+				wasm_overwrite_path: config.wasm_overwrite_path.clone(),
 			},
 		)?
 	};
@@ -396,7 +398,7 @@ pub fn new_client<E, Block, RA>(
 	const CANONICALIZATION_DELAY: u64 = 4096;
 
 	let backend = Arc::new(Backend::new(settings, CANONICALIZATION_DELAY)?);
-	let executor = crate::client::LocalCallExecutor::new(backend.clone(), executor, spawn_handle, config.clone());
+	let executor = crate::client::LocalCallExecutor::new(backend.clone(), executor, spawn_handle, config.clone())?;
 	Ok((
 		crate::client::Client::new(
 			backend.clone(),

client/service/src/client/call_executor.rs

@@ -33,31 +33,41 @@ use sp_core::{
 };
 use sp_api::{ProofRecorder, InitializeBlock, StorageTransactionCache};
 use sc_client_api::{backend, call_executor::CallExecutor};
-use super::client::ClientConfig;
+use super::{client::ClientConfig, wasm_overwrite::WasmOverwrite};
 
 /// Call executor that executes methods locally, querying all required
 /// data from local backend.
 pub struct LocalCallExecutor<B, E> {
 	backend: Arc<B>,
 	executor: E,
+	wasm_overwrite: WasmOverwrite<E>,
 	spawn_handle: Box<dyn SpawnNamed>,
 	client_config: ClientConfig,
 }
 
-impl<B, E> LocalCallExecutor<B, E> {
+impl<B, E> LocalCallExecutor<B, E> 
+where
+	E: RuntimeInfo + Clone + 'static 
+{
 	/// Creates new instance of local call executor.
 	pub fn new(
 		backend: Arc<B>,
 		executor: E,
 		spawn_handle: Box<dyn SpawnNamed>,
 		client_config: ClientConfig,
-	) -> Self {
-		LocalCallExecutor {
+	) -> sp_blockchain::Result<Self> {
+		let wasm_overwrite = WasmOverwrite::new(
+			client_config.wasm_overwrite_path.clone(), 
+			client_config.wasm_overwrite_enabled, 
+			executor.clone()
+		)?;
+		Ok(LocalCallExecutor {
 			backend,
 			executor,
+			wasm_overwrite,
 			spawn_handle,
 			client_config,
-		}
+		})
 	}
 }
 
@@ -66,6 +76,7 @@ impl<B, E> Clone for LocalCallExecutor<B, E> where E: Clone {
 		LocalCallExecutor {
 			backend: self.backend.clone(),
 			executor: self.executor.clone(),
+			wasm_overwrite: self.wasm_overwrite.clone(),
 			spawn_handle: self.spawn_handle.clone(),
 			client_config: self.client_config.clone(),
 		}
@@ -101,6 +112,8 @@ where
 		)?;
 		let state = self.backend.state_at(*id)?;
 		let state_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&state);
+		let runtime_code = state_runtime_code.runtime_code()?;
+		let runtime_code = self.wasm_overwrite.try_replace(runtime_code, &state)?;
 		let return_data = StateMachine::new(
 			&state,
 			changes_trie,
@@ -110,7 +123,7 @@ where
 			method,
 			call_data,
 			extensions.unwrap_or_default(),
-			&state_runtime_code.runtime_code()?,
+			&runtime_code,
 			self.spawn_handle.clone(),
 		).execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
 			strategy.get_manager(),
@@ -174,6 +187,7 @@ where
 				// It is important to extract the runtime code here before we create the proof
 				// recorder.
 				let runtime_code = state_runtime_code.runtime_code()?;
+				let runtime_code = self.wasm_overwrite.try_replace(runtime_code, &trie_state)?;
 
 				let backend = sp_state_machine::ProvingBackend::new_with_recorder(
 					trie_state,
@@ -199,6 +213,8 @@ where
 			None => {
 				let state_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&state);
 				let runtime_code = state_runtime_code.runtime_code()?;
+				let runtime_code = self.wasm_overwrite.try_replace(runtime_code, &state)?;
+
 				let mut state_machine = StateMachine::new(
 					&state,
 					changes_trie_state,

client/service/src/client/client.rs

@@ -22,6 +22,7 @@ use std::{
 	marker::PhantomData,
 	collections::{HashSet, BTreeMap, HashMap},
 	sync::Arc, panic::UnwindSafe, result,
+	path::PathBuf
 };
 use log::{info, trace, warn};
 use parking_lot::{Mutex, RwLock};
@@ -181,6 +182,10 @@ pub struct ClientConfig {
 	pub offchain_worker_enabled: bool,
 	/// If true, allows access from the runtime to write into offchain worker db.
 	pub offchain_indexing_api: bool,
+	/// Enable overwriting on-chain WASM with local WASM
+	pub wasm_overwrite_enabled: bool,
+	/// Path where WASM exists to overwrite the on-chain WASM
+	pub wasm_overwrite_path: PathBuf,
 }
 
 /// Create a client with the explicitly provided backend.
@@ -201,7 +206,7 @@ pub fn new_with_backend<B, E, Block, S, RA>(
 		Block: BlockT,
 		B: backend::LocalBackend<Block> + 'static,
 {
-	let call_executor = LocalCallExecutor::new(backend.clone(), executor, spawn_handle, config.clone());
+	let call_executor = LocalCallExecutor::new(backend.clone(), executor, spawn_handle, config.clone())?;
 	let extensions = ExecutionExtensions::new(Default::default(), keystore);
 	Client::new(
 		backend,

client/service/src/client/light.rs

@@ -60,7 +60,7 @@ pub fn new_light<B, S, RA, E>(
 		code_executor,
 		spawn_handle.clone(),
 		ClientConfig::default()
-	);
+	)?;
 	let executor = GenesisCallExecutor::new(backend.clone(), local_executor);
 	Client::new(
 		backend,

client/service/src/client/mod.rs

@@ -49,6 +49,7 @@ pub mod light;
 mod call_executor;
 mod client;
 mod block_rules;
+mod wasm_overwrite;
 
 pub use self::{
 	call_executor::LocalCallExecutor,

client/service/src/client/wasm_overwrite.rs

@@ -0,0 +1,144 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use std::{ 
+	fs, collections::{HashMap, hash_map::DefaultHasher}, path::Path,
+	hash::Hasher as _, 
+};
+use codec::{Encode, Decode};
+use sp_core::{
+	traits::FetchRuntimeCode,
+};
+use sp_state_machine::{BasicExternalities, backend};
+use sp_blockchain::Result;
+use sc_executor::RuntimeInfo;
+use sp_version::RuntimeVersion;
+use sp_core::traits::RuntimeCode;
+use hash_db::Hasher;
+
+#[derive(Clone, Debug)]
+struct WasmBlob {
+	code: Vec<u8>,
+}
+
+impl WasmBlob {
+	fn new(code: Vec<u8>) -> Self {
+		Self { code }
+	}
+
+	fn runtime_code(&self, heap_pages: Option<u64>) -> RuntimeCode {
+		RuntimeCode {
+			code_fetcher: self,
+			hash: make_hash(self.code.as_slice()).encode(),
+			heap_pages,
+		}
+	}
+}
+
+/// Make a hash out of a byte string using the default rust hasher
+pub fn make_hash<K: std::hash::Hash + ?Sized>(val: &K) -> Vec<u8> {
+	let mut state = DefaultHasher::new();
+	val.hash(&mut state);
+	state.finish().to_le_bytes().to_vec()
+}
+
+impl FetchRuntimeCode for WasmBlob {
+	fn fetch_runtime_code<'a>(&'a self) -> Option<std::borrow::Cow<'a, [u8]>> {
+		Some(self.code.as_slice().into())
+	}
+}
+
+#[derive(Clone, Debug)]
+pub struct WasmOverwrite<E> {
+	// Map of runtime spec version -> Wasm Blob
+	overwrites: HashMap<u32, WasmBlob>,
+	executor: E,
+	enabled: bool,
+}
+
+impl<E> WasmOverwrite<E> 
+where 
+	E: RuntimeInfo + Clone + 'static 
+{
+	pub fn new<P>(path: P, enabled: bool, executor: E) -> Result<Self> 
+	where
+		P: AsRef<Path>,
+	{
+		let overwrites = Self::scrape_overwrites(path.as_ref(), &executor)?;
+		Ok(Self { overwrites, executor, enabled })
+	}
+
+	/// Tries to replace the given `code` with an overwrite, if it exists.
+	/// If the overwrite does not exist, or overwrites are not enabled,
+	/// this function returns the original runtime code.
+	pub fn try_replace<'a, 'b: 'a, B, H>(
+		&'b self, 
+		code: RuntimeCode<'a>, 
+		state: &'a B,
+	) -> Result<RuntimeCode<'a>>
+	where
+		B: backend::Backend<H>,
+		H: Hasher
+	{
+		if !self.enabled {
+			return Ok(code);
+		}
+
+		let backend_code = code.fetch_runtime_code()
+			.ok_or(sp_blockchain::Error::Msg(format!("Runtime code could not be found in the backend")))?;
+		let heap_pages = state.storage(sp_core::storage::well_known_keys::HEAP_PAGES)
+			.ok()
+			.flatten()
+			.and_then(|d| Decode::decode(&mut &d[..]).ok()); 
+		let version = Self::runtime_version(&self.executor, &WasmBlob::new(backend_code.to_vec()), heap_pages)?;
+
+		if let Some(runtime_code) = self.overwrites
+			.get(&version.spec_version)
+			.map(|w| w.runtime_code(heap_pages)) 
+		{
+			Ok(runtime_code)
+		} else {
+			Ok(code)
+		}
+	}
+
+	/// Scrapes a folder for WASM runtimes.
+    /// Returns a hashmap of the runtime version and wasm runtime code.
+	fn scrape_overwrites(dir: &Path, executor: &E) -> Result<HashMap<u32, WasmBlob>> {
+		let handle_err = |e: std::io::Error | -> sp_blockchain::Error {
+			sp_blockchain::Error::Msg(format!("{}", e.to_string()))
+		};
+
+		let mut overwrites = HashMap::new(); 
+		if dir.is_dir() {
+			for entry in fs::read_dir(dir).map_err(handle_err)? {
+				let entry = entry.map_err(handle_err)?;
+				let path = entry.path();
+				let wasm = WasmBlob::new(fs::read(path).map_err(handle_err)?);
+				let version = Self::runtime_version(executor, &wasm, Some(128))?;
+				overwrites.insert(version.spec_version, wasm);
+			}
+		}
+		Ok(overwrites)
+	}
+
+	fn runtime_version(executor: &E, code: &WasmBlob, heap_pages: Option<u64>) -> Result<RuntimeVersion> {
+		let mut ext = BasicExternalities::default(); 
+		executor.runtime_version(&mut ext, &code.runtime_code(heap_pages))
+			.map_err(|e| sp_blockchain::Error::VersionInvalid(format!("{:?}", e)).into())
+	}
+}