From ea0356758ed5823fd6396035c6a0ae55aa0d9f14 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 21 Aug 2020 11:56:12 +0200 Subject: [PATCH 01/21] checkpoint before removing CT from change trie --- Cargo.lock | 27 +- primitives/arithmetic/src/biguint.rs | 2 +- primitives/externalities/Cargo.toml | 18 +- primitives/externalities/src/extensions.rs | 9 +- primitives/externalities/src/lib.rs | 8 +- primitives/state-machine/Cargo.toml | 44 +- primitives/state-machine/src/backend.rs | 9 +- .../state-machine/src/changes_trie/build.rs | 89 +- .../state-machine/src/changes_trie/mod.rs | 11 +- primitives/state-machine/src/error.rs | 4 +- primitives/state-machine/src/lib.rs | 1380 +++++++++-------- .../src/overlayed_changes/changeset.rs | 63 +- .../src/overlayed_changes/mod.rs | 110 +- primitives/state-machine/src/stats.rs | 19 +- primitives/state-machine/src/trie_backend.rs | 10 +- .../state-machine/src/trie_backend_essence.rs | 43 +- .../state-machine/src/witness_backend.rs | 404 +++++ test-utils/runtime/Cargo.toml | 3 +- test-utils/runtime/src/lib.rs | 3 + test-utils/runtime/src/system.rs | 10 + 20 files changed, 1453 insertions(+), 813 deletions(-) create mode 100644 primitives/state-machine/src/witness_backend.rs diff --git a/Cargo.lock b/Cargo.lock index c80c0557443a2..1796a4ebcbf22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -930,7 +930,7 @@ dependencies = [ "clap", "criterion-plot 0.3.1", "csv", - "itertools 0.8.2", + "itertools", "lazy_static", "libc", "num-traits", @@ -957,7 +957,7 @@ dependencies = [ "clap", "criterion-plot 0.4.1", "csv", - "itertools 0.8.2", + "itertools", "lazy_static", "num-traits", "oorandom", @@ -979,7 +979,7 @@ checksum = "76f9212ddf2f4a9eb2d401635190600656a1f88a932ef53d06e7fa4c7e02fb8e" dependencies = [ "byteorder", "cast", - "itertools 0.8.2", + "itertools", ] [[package]] @@ -989,7 +989,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a01e15e0ea58e8234f96146b1f91fa9d0e4dd7a38da93ff7a75d42c0b9d3a545" dependencies = [ "cast", - "itertools 0.8.2", + "itertools", ] [[package]] @@ -2494,15 +2494,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "0.4.5" @@ -5433,7 +5424,7 @@ checksum = "02b10678c913ecbd69350e8535c3aef91a8676c0773fc1d7b95cdd196d7f2f26" dependencies = [ "bytes 0.5.4", "heck", - "itertools 0.8.2", + "itertools", "log", "multimap", "petgraph", @@ -5450,7 +5441,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "537aa19b95acde10a12fec4301466386f757403de4cd4e5b4fa78fb5ecb18f72" dependencies = [ "anyhow", - "itertools 0.8.2", + "itertools", "proc-macro2", "quote", "syn", @@ -8153,7 +8144,6 @@ version = "0.8.0-rc5" dependencies = [ "hash-db", "hex-literal", - "itertools 0.9.0", "log", "num-traits", "parity-scale-codec", @@ -8165,6 +8155,7 @@ dependencies = [ "sp-externalities", "sp-panic-handler", "sp-runtime", + "sp-std", "sp-trie", "trie-db", "trie-root", @@ -8400,7 +8391,7 @@ dependencies = [ "hex", "hex-literal", "hyper 0.12.35", - "itertools 0.8.2", + "itertools", "jsonrpc-core-client", "libp2p", "node-primitives", @@ -8657,7 +8648,7 @@ dependencies = [ "build-helper", "cargo_metadata", "fs2", - "itertools 0.8.2", + "itertools", "tempfile", "toml", "walkdir", diff --git a/primitives/arithmetic/src/biguint.rs b/primitives/arithmetic/src/biguint.rs index 41e2c759a5967..32edbfcd235d6 100644 --- a/primitives/arithmetic/src/biguint.rs +++ b/primitives/arithmetic/src/biguint.rs @@ -18,7 +18,7 @@ //! Infinite precision unsigned integer for substrate runtime. use num_traits::Zero; -use sp_std::{cmp::Ordering, ops, prelude::*, cell::RefCell, convert::TryFrom}; +use sp_std::{cmp::Ordering, ops, prelude::*, cell::RefCell, convert::TryFrom, vec}; // A sensible value for this would be half of the dword size of the host machine. Since the // runtime is compiled to 32bit webassembly, using 32 and 64 for single and double respectively diff --git a/primitives/externalities/Cargo.toml b/primitives/externalities/Cargo.toml index 62a2413f3355d..9ebd2f233188e 100644 --- a/primitives/externalities/Cargo.toml +++ b/primitives/externalities/Cargo.toml @@ -13,7 +13,17 @@ documentation = "https://docs.rs/sp-externalities" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-storage = { version = "2.0.0-rc5", path = "../storage" } -sp-std = { version = "2.0.0-rc5", path = "../std" } -environmental = { version = "1.1.1" } -codec = { package = "parity-scale-codec", version = "1.3.1" } +sp-storage = { version = "2.0.0-rc5", path = "../storage", default-features = false } +sp-std = { version = "2.0.0-rc5", path = "../std", default-features = false } +environmental = { version = "1.1.1", optional = true } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } + +[features] +default = ["std"] +std = [ + "codec/std", + "sp-std/std", + "sp-storage/std", + + "environmental", +] diff --git a/primitives/externalities/src/extensions.rs b/primitives/externalities/src/extensions.rs index 08d81e46c88fc..8eb3c9a568324 100644 --- a/primitives/externalities/src/extensions.rs +++ b/primitives/externalities/src/extensions.rs @@ -22,7 +22,11 @@ //! //! It is required that each extension implements the [`Extension`] trait. -use std::{collections::HashMap, collections::hash_map::Entry, any::{Any, TypeId}, ops::DerefMut}; +#[cfg(feature = "std")] +use std::{collections::HashMap as Map, collections::hash_map::Entry}; +#[cfg(not(feature = "std"))] +use sp_std::collections::btree_map::{BTreeMap as Map, Entry}; +use sp_std::{any::{Any, TypeId}, ops::DerefMut, boxed::Box}; use crate::Error; /// Marker trait for types that should be registered as [`Externalities`](crate::Externalities) extension. @@ -104,9 +108,10 @@ pub trait ExtensionStore { /// Stores extensions that should be made available through the externalities. #[derive(Default)] pub struct Extensions { - extensions: HashMap>, + extensions: Map>, } +#[cfg(feature = "std")] impl std::fmt::Debug for Extensions { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "Extensions: ({})", self.extensions.len()) diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index 01570e0bfadd3..e0df7ea8cbb67 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -15,6 +15,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![cfg_attr(not(feature = "std"), no_std)] + //! Substrate externalities abstraction //! //! The externalities mainly provide access to storage and to registered extensions. Extensions @@ -23,14 +25,18 @@ //! //! This crate exposes the main [`Externalities`] trait. -use std::any::{Any, TypeId}; +use sp_std::any::{Any, TypeId}; +use sp_std::vec::Vec; +use sp_std::boxed::Box; use sp_storage::{ChildInfo, TrackedStorageKey}; +#[cfg(feature = "std")] pub use scope_limited::{set_and_run_with_externalities, with_externalities}; pub use extensions::{Extension, Extensions, ExtensionStore}; mod extensions; +#[cfg(feature = "std")] mod scope_limited; /// Externalities error. diff --git a/primitives/state-machine/Cargo.toml b/primitives/state-machine/Cargo.toml index 781d3b068a857..43f2c8e2aae75 100644 --- a/primitives/state-machine/Cargo.toml +++ b/primitives/state-machine/Cargo.toml @@ -13,20 +13,20 @@ documentation = "https://docs.rs/sp-state-machine" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -log = "0.4.8" -parking_lot = "0.10.0" -hash-db = "0.15.2" -trie-db = "0.22.0" -trie-root = "0.16.0" -sp-trie = { version = "2.0.0-rc5", path = "../trie" } -sp-core = { version = "2.0.0-rc5", path = "../core" } -sp-panic-handler = { version = "2.0.0-rc5", path = "../panic-handler" } -codec = { package = "parity-scale-codec", version = "1.3.1" } -num-traits = "0.2.8" -rand = "0.7.2" -sp-externalities = { version = "0.8.0-rc5", path = "../externalities" } -itertools = "0.9" +log = { version = "0.4.8", optional = true } +parking_lot = { version = "0.10.0", optional = true } +hash-db = { version = "0.15.2", default-features = false } +trie-db = { version = "0.22.0", default-features = false } +trie-root = { version = "0.16.0", default-features = false } +sp-trie = { version = "2.0.0-rc5", path = "../trie", default-features = false } +sp-core = { version = "2.0.0-rc5", path = "../core", default-features = false } +sp-panic-handler = { version = "2.0.0-rc5", path = "../panic-handler", optional = true } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } +num-traits = { version = "0.2.8", default-features = false } +rand = { version = "0.7.2", optional = true } +sp-externalities = { version = "0.8.0-rc5", path = "../externalities", default-features = false } smallvec = "1.4.1" +sp-std = { version = "2.0.0-rc5", default-features = false, path = "../std" } [dev-dependencies] hex-literal = "0.2.1" @@ -34,4 +34,20 @@ sp-runtime = { version = "2.0.0-rc5", path = "../runtime" } pretty_assertions = "0.6.1" [features] -default = [] +default = ["std"] +std = [ + "codec/std", + "hash-db/std", + "num-traits/std", + "sp-core/std", + "sp-externalities/std", + "sp-std/std", + "sp-trie/std", + "trie-db/std", + "trie-root/std", + + "log", + "parking_lot", + "rand", + "sp-panic-handler", +] diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index cfff2c6fc6967..c1ace54e388e5 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -20,7 +20,6 @@ use hash_db::Hasher; use codec::{Decode, Encode}; use sp_core::{ - traits::RuntimeCode, storage::{ChildInfo, well_known_keys, TrackedStorageKey} }; use crate::{ @@ -28,12 +27,15 @@ use crate::{ trie_backend_essence::TrieBackendStorage, UsageInfo, StorageKey, StorageValue, StorageCollection, }; +use sp_std::vec::Vec; +#[cfg(feature = "std")] +use sp_core::traits::RuntimeCode; /// A state backend is used to read state data and can have changes committed /// to it. /// /// The clone operation (if implemented) should be cheap. -pub trait Backend: std::fmt::Debug { +pub trait Backend: sp_std::fmt::Debug { /// An error type when fetching data is not possible. type Error: super::Error; @@ -369,11 +371,13 @@ pub(crate) fn insert_into_memory_db(mdb: &mut sp_trie::MemoryDB, input: } /// Wrapper to create a [`RuntimeCode`] from a type that implements [`Backend`]. +#[cfg(feature = "std")] pub struct BackendRuntimeCode<'a, B, H> { backend: &'a B, _marker: std::marker::PhantomData, } +#[cfg(feature = "std")] impl<'a, B: Backend, H: Hasher> sp_core::traits::FetchRuntimeCode for BackendRuntimeCode<'a, B, H> { @@ -382,6 +386,7 @@ impl<'a, B: Backend, H: Hasher> sp_core::traits::FetchRuntimeCode for } } +#[cfg(feature = "std")] impl<'a, B: Backend, H: Hasher> BackendRuntimeCode<'a, B, H> where H::Out: Encode { /// Create a new instance. pub fn new(backend: &'a B) -> Self { diff --git a/primitives/state-machine/src/changes_trie/build.rs b/primitives/state-machine/src/changes_trie/build.rs index 675904578be97..80c3048053ecd 100644 --- a/primitives/state-machine/src/changes_trie/build.rs +++ b/primitives/state-machine/src/changes_trie/build.rs @@ -25,7 +25,7 @@ use num_traits::One; use crate::{ StorageKey, backend::Backend, - overlayed_changes::{OverlayedChanges, OverlayedValue}, + overlayed_changes::{OverlayedChanges, OverlayedValue as OverlayedValueInner, ChangeTrieOverlay}, trie_backend_essence::TrieBackendEssence, changes_trie::{ AnchorBlockId, ConfigurationRange, Storage, BlockNumber, @@ -35,15 +35,17 @@ use crate::{ }; use sp_core::storage::{ChildInfo, PrefixedStorageKey}; +type OverlayedValue = OverlayedValueInner; + /// Prepare input pairs for building a changes trie of given block. /// /// Returns Err if storage error has occurred OR if storage haven't returned /// required data. -pub(crate) fn prepare_input<'a, B, H, Number>( +pub(crate) fn prepare_input<'a, B, H, Number, CT>( backend: &'a B, storage: &'a dyn Storage, config: ConfigurationRange<'a, Number>, - overlay: &'a OverlayedChanges, + overlay: &'a OverlayedChanges, parent: &'a AnchorBlockId, ) -> Result<( impl Iterator> + 'a, @@ -55,6 +57,7 @@ pub(crate) fn prepare_input<'a, B, H, Number>( H: Hasher + 'a, H::Out: Encode, Number: BlockNumber, + CT: ChangeTrieOverlay, { let number = parent.number.clone() + One::one(); let (extrinsics_input, children_extrinsics_input) = prepare_extrinsics_input( @@ -93,10 +96,10 @@ pub(crate) fn prepare_input<'a, B, H, Number>( )) } /// Prepare ExtrinsicIndex input pairs. -fn prepare_extrinsics_input<'a, B, H, Number>( +fn prepare_extrinsics_input<'a, B, H, Number, CT>( backend: &'a B, block: &Number, - overlay: &'a OverlayedChanges, + overlay: &'a OverlayedChanges, ) -> Result<( impl Iterator> + 'a, BTreeMap, impl Iterator> + 'a>, @@ -105,6 +108,7 @@ fn prepare_extrinsics_input<'a, B, H, Number>( B: Backend, H: Hasher + 'a, Number: BlockNumber, + CT: ChangeTrieOverlay, { let mut children_result = BTreeMap::new(); @@ -127,55 +131,58 @@ fn prepare_extrinsics_input<'a, B, H, Number>( Ok((top, children_result)) } -fn prepare_extrinsics_input_inner<'a, B, H, Number>( +fn prepare_extrinsics_input_inner<'a, B, H, Number, CT>( backend: &'a B, block: &Number, - overlay: &'a OverlayedChanges, + overlay: &'a OverlayedChanges, child_info: Option, - changes: impl Iterator + mut changes: impl Iterator)> ) -> Result> + 'a, String> where B: Backend, H: Hasher, Number: BlockNumber, + CT: ChangeTrieOverlay, { changes - .filter(|( _, v)| v.extrinsics().next().is_some()) .try_fold(BTreeMap::new(), |mut map: BTreeMap<&[u8], (ExtrinsicIndex, Vec)>, (k, v)| { - match map.entry(k) { - Entry::Vacant(entry) => { - // ignore temporary values (values that have null value at the end of operation - // AND are not in storage at the beginning of operation - if let Some(child_info) = child_info.as_ref() { - if !overlay.child_storage(child_info, k).map(|v| v.is_some()).unwrap_or_default() { - if !backend.exists_child_storage(&child_info, k) - .map_err(|e| format!("{}", e))? { - return Ok(map); + let extrinsics = v.extrinsics(); + if !extrinsics.is_empty() { + match map.entry(k) { + Entry::Vacant(entry) => { + // ignore temporary values (values that have null value at the end of operation + // AND are not in storage at the beginning of operation + if let Some(child_info) = child_info.as_ref() { + if !overlay.child_storage(child_info, k).map(|v| v.is_some()).unwrap_or_default() { + if !backend.exists_child_storage(&child_info, k) + .map_err(|e| format!("{}", e))? { + return Ok(map); + } } - } - } else { - if !overlay.storage(k).map(|v| v.is_some()).unwrap_or_default() { - if !backend.exists_storage(k).map_err(|e| format!("{}", e))? { - return Ok(map); + } else { + if !overlay.storage(k).map(|v| v.is_some()).unwrap_or_default() { + if !backend.exists_storage(k).map_err(|e| format!("{}", e))? { + return Ok(map); + } } - } - }; - - let extrinsics = v.extrinsics().cloned().collect(); - entry.insert((ExtrinsicIndex { - block: block.clone(), - key: k.to_vec(), - }, extrinsics)); - }, - Entry::Occupied(mut entry) => { - // we do not need to check for temporary values here, because entry is Occupied - // AND we are checking it before insertion - let extrinsics = &mut entry.get_mut().1; - extrinsics.extend( - v.extrinsics().cloned() - ); - extrinsics.sort(); - }, + }; + + let extrinsics = extrinsics.into_iter().collect(); + entry.insert((ExtrinsicIndex { + block: block.clone(), + key: k.to_vec(), + }, extrinsics)); + }, + Entry::Occupied(mut entry) => { + // we do not need to check for temporary values here, because entry is Occupied + // AND we are checking it before insertion + let entry_extrinsics = &mut entry.get_mut().1; + entry_extrinsics.extend( + extrinsics.into_iter() + ); + entry_extrinsics.sort(); + }, + } } Ok(map) diff --git a/primitives/state-machine/src/changes_trie/mod.rs b/primitives/state-machine/src/changes_trie/mod.rs index 04322f1d5930c..4452cbbb727b0 100644 --- a/primitives/state-machine/src/changes_trie/mod.rs +++ b/primitives/state-machine/src/changes_trie/mod.rs @@ -78,7 +78,7 @@ use sp_trie::trie_types::TrieDBMut; use crate::{ StorageKey, backend::Backend, - overlayed_changes::OverlayedChanges, + overlayed_changes::{OverlayedChanges, ChangeTrieOverlay}, changes_trie::{ build::prepare_input, build_cache::{IncompleteCachedBuildData, IncompleteCacheAction}, @@ -223,16 +223,19 @@ pub fn disabled_state<'a, H, Number>() -> Option> { /// Returns Err(()) if unknown `parent_hash` has been passed. /// Returns Ok(None) if there's no data to perform computation. /// Panics if background storage returns an error OR if insert to MemoryDB fails. -pub fn build_changes_trie<'a, B: Backend, H: Hasher, Number: BlockNumber>( +pub fn build_changes_trie<'a, B: Backend, H: Hasher, Number: BlockNumber, CT: ChangeTrieOverlay>( backend: &B, state: Option<&'a State<'a, H, Number>>, - changes: &OverlayedChanges, + changes: &OverlayedChanges, parent_hash: H::Out, panic_on_storage_error: bool, ) -> Result, H::Out, CacheAction)>, ()> where H::Out: Ord + 'static + Encode, { + if !CT::CHANGE_TRIE_CAPABLE { + return Ok(None); + } /// Panics when `res.is_err() && panic`, otherwise it returns `Err(())` on an error. fn maybe_panic( res: std::result::Result, @@ -271,7 +274,7 @@ pub fn build_changes_trie<'a, B: Backend, H: Hasher, Number: BlockNumber>( // storage errors are considered fatal (similar to situations when runtime fetches values from storage) let (input_pairs, child_input_pairs, digest_input_blocks) = maybe_panic( - prepare_input::( + prepare_input::( backend, state.storage, config_range.clone(), diff --git a/primitives/state-machine/src/error.rs b/primitives/state-machine/src/error.rs index 5468262f54a2c..489f6e6666001 100644 --- a/primitives/state-machine/src/error.rs +++ b/primitives/state-machine/src/error.rs @@ -17,7 +17,7 @@ /// State Machine Errors -use std::fmt; +use sp_std::fmt; /// State Machine Error bound. /// @@ -34,7 +34,7 @@ impl Error for T {} #[derive(Debug, Eq, PartialEq)] pub enum ExecutionError { /// Backend error. - Backend(String), + Backend(crate::DefaultError), /// The entry `:code` doesn't exist in storage so there's no way we can execute anything. CodeEntryDoesNotExist, /// Backend is incompatible with execution proof generation process. diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index ee0980f59b926..c66aa3d7fd948 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -18,747 +18,806 @@ //! Substrate state machine implementation. #![warn(missing_docs)] - -use std::{fmt, result, collections::HashMap, panic::UnwindSafe}; -use log::{warn, trace}; -use hash_db::Hasher; -use codec::{Decode, Encode, Codec}; -use sp_core::{ - offchain::storage::OffchainOverlayedChanges, - storage::ChildInfo, NativeOrEncoded, NeverNativeValue, hexdisplay::HexDisplay, - traits::{CodeExecutor, CallInWasmExt, RuntimeCode, SpawnNamed}, -}; -use sp_externalities::Extensions; +#![cfg_attr(not(feature = "std"), no_std)] pub mod backend; +pub mod witness_backend; +#[cfg(feature = "std")] mod in_memory_backend; +#[cfg(feature = "std")] mod changes_trie; mod error; +#[cfg(feature = "std")] mod ext; +#[cfg(feature = "std")] mod testing; +#[cfg(feature = "std")] mod basic; mod overlayed_changes; +#[cfg(feature = "std")] mod proving_backend; mod trie_backend; mod trie_backend_essence; mod stats; +#[cfg(feature = "std")] mod read_only; -pub use sp_trie::{trie_types::{Layout, TrieDBMut}, StorageProof, TrieMut, DBValue, MemoryDB}; -pub use testing::TestExternalities; -pub use basic::BasicExternalities; -pub use read_only::{ReadOnlyExternalities, InspectState}; -pub use ext::Ext; -pub use backend::Backend; -pub use changes_trie::{ - AnchorBlockId as ChangesTrieAnchorBlockId, - State as ChangesTrieState, - Storage as ChangesTrieStorage, - RootsStorage as ChangesTrieRootsStorage, - InMemoryStorage as InMemoryChangesTrieStorage, - BuildCache as ChangesTrieBuildCache, - CacheAction as ChangesTrieCacheAction, - ConfigurationRange as ChangesTrieConfigurationRange, - key_changes, key_changes_proof, - key_changes_proof_check, key_changes_proof_check_with_db, - prune as prune_changes_tries, - disabled_state as disabled_changes_trie_state, - BlockNumber as ChangesTrieBlockNumber, -}; -pub use overlayed_changes::{ - OverlayedChanges, StorageChanges, StorageTransactionCache, StorageKey, StorageValue, +#[cfg(feature = "std")] +pub use std_reexport::*; + +#[cfg(feature = "std")] +pub use execution::*; + + +#[cfg(feature = "std")] +pub use log::{debug, warn}; + +#[cfg(not(feature = "std"))] +#[macro_export] // TODO try remove export +macro_rules! warn { + (target: $target:expr, $($arg:tt)+) => ( + () + ); + ($($arg:tt)+) => ( + () + ); +} +#[cfg(not(feature = "std"))] +#[macro_export] +macro_rules! debug { + (target: $target:expr, $($arg:tt)+) => ( + () + ); + ($($arg:tt)+) => ( + () + ); +} + +/// Default rror type to use with state machine trie backend. +#[cfg(feature = "std")] +pub type DefaultError = String; +/// Error type to use with state machine trie backend. +#[cfg(not(feature = "std"))] +pub type DefaultError = (); + + +pub use crate::overlayed_changes::{ + OverlayedChanges, StorageTransactionCache, StorageKey, StorageValue, StorageCollection, ChildStorageCollection, }; -pub use proving_backend::{ - create_proof_check_backend, ProofRecorder, ProvingBackend, ProvingBackendRecorder, -}; -pub use trie_backend_essence::{TrieBackendStorage, Storage}; -pub use trie_backend::TrieBackend; +pub use crate::backend::Backend; +pub use crate::trie_backend_essence::{TrieBackendStorage, Storage}; +pub use crate::trie_backend::TrieBackend; +pub use crate::stats::{UsageInfo, UsageUnit, StateMachineStats}; pub use error::{Error, ExecutionError}; -pub use in_memory_backend::new_in_mem; -pub use stats::{UsageInfo, UsageUnit, StateMachineStats}; - -const PROOF_CLOSE_TRANSACTION: &str = "\ - Closing a transaction that was started in this function. Client initiated transactions - are protected from being closed by the runtime. qed"; - -type CallResult = Result, E>; - -/// Default handler of the execution manager. -pub type DefaultHandler = fn(CallResult, CallResult) -> CallResult; - -/// Type of changes trie transaction. -pub type ChangesTrieTransaction = ( - MemoryDB, - ChangesTrieCacheAction<::Out, N>, -); - -/// Trie backend with in-memory storage. -pub type InMemoryBackend = TrieBackend, H>; - -/// Strategy for executing a call into the runtime. -#[derive(Copy, Clone, Eq, PartialEq, Debug)] -pub enum ExecutionStrategy { - /// Execute with the native equivalent if it is compatible with the given wasm module; otherwise fall back to the wasm. - NativeWhenPossible, - /// Use the given wasm module. - AlwaysWasm, - /// Run with both the wasm and the native variant (if compatible). Report any discrepancy as an error. - Both, - /// First native, then if that fails or is not possible, wasm. - NativeElseWasm, -} -/// Storage backend trust level. -#[derive(Debug, Clone)] -pub enum BackendTrustLevel { - /// Panics from trusted backends are considered justified, and never caught. - Trusted, - /// Panics from untrusted backend are caught and interpreted as runtime error. - /// Untrusted backend may be missing some parts of the trie, so panics are not considered - /// fatal. - Untrusted, +#[cfg(feature = "std")] +mod std_reexport { + pub use crate::overlayed_changes::StorageChanges; + pub use sp_trie::{trie_types::{Layout, TrieDBMut}, StorageProof, TrieMut, DBValue, MemoryDB}; + pub use crate::testing::TestExternalities; + pub use crate::basic::BasicExternalities; + pub use crate::read_only::{ReadOnlyExternalities, InspectState}; + pub use crate::ext::Ext; + pub use crate::changes_trie::{ + AnchorBlockId as ChangesTrieAnchorBlockId, + State as ChangesTrieState, + Storage as ChangesTrieStorage, + RootsStorage as ChangesTrieRootsStorage, + InMemoryStorage as InMemoryChangesTrieStorage, + BuildCache as ChangesTrieBuildCache, + CacheAction as ChangesTrieCacheAction, + ConfigurationRange as ChangesTrieConfigurationRange, + key_changes, key_changes_proof, + key_changes_proof_check, key_changes_proof_check_with_db, + prune as prune_changes_tries, + disabled_state as disabled_changes_trie_state, + BlockNumber as ChangesTrieBlockNumber, + }; + pub use crate::proving_backend::{ + create_proof_check_backend, ProofRecorder, ProvingBackend, ProvingBackendRecorder, + }; + pub use crate::error::{Error, ExecutionError}; + pub use crate::in_memory_backend::new_in_mem; } -/// Like `ExecutionStrategy` only it also stores a handler in case of consensus failure. -#[derive(Clone)] -pub enum ExecutionManager { - /// Execute with the native equivalent if it is compatible with the given wasm module; otherwise fall back to the wasm. - NativeWhenPossible, - /// Use the given wasm module. The backend on which code is executed code could be - /// trusted to provide all storage or not (i.e. the light client cannot be trusted to provide - /// for all storage queries since the storage entries it has come from an external node). - AlwaysWasm(BackendTrustLevel), - /// Run with both the wasm and the native variant (if compatible). Call `F` in the case of any discrepancy. - Both(F), - /// First native, then if that fails or is not possible, wasm. - NativeElseWasm, -} +#[cfg(feature = "std")] +mod execution { + use super::*; + use std::{fmt, result, collections::HashMap, panic::UnwindSafe}; + use log::{warn, trace}; + use hash_db::Hasher; + use codec::{Decode, Encode, Codec}; + use sp_core::{ + offchain::storage::OffchainOverlayedChanges, + storage::ChildInfo, NativeOrEncoded, NeverNativeValue, hexdisplay::HexDisplay, + traits::{CodeExecutor, CallInWasmExt, RuntimeCode, SpawnNamed}, + }; + use sp_externalities::Extensions; -impl<'a, F> From<&'a ExecutionManager> for ExecutionStrategy { - fn from(s: &'a ExecutionManager) -> Self { - match *s { - ExecutionManager::NativeWhenPossible => ExecutionStrategy::NativeWhenPossible, - ExecutionManager::AlwaysWasm(_) => ExecutionStrategy::AlwaysWasm, - ExecutionManager::NativeElseWasm => ExecutionStrategy::NativeElseWasm, - ExecutionManager::Both(_) => ExecutionStrategy::Both, - } + + const PROOF_CLOSE_TRANSACTION: &str = "\ + Closing a transaction that was started in this function. Client initiated transactions + are protected from being closed by the runtime. qed"; + + pub(crate) type CallResult = Result, E>; + + /// Default handler of the execution manager. + pub type DefaultHandler = fn(CallResult, CallResult) -> CallResult; + + /// Type of changes trie transaction. + pub type ChangesTrieTransaction = ( + MemoryDB, + ChangesTrieCacheAction<::Out, N>, + ); + + /// Trie backend with in-memory storage. + pub type InMemoryBackend = TrieBackend, H>; + + /// Strategy for executing a call into the runtime. + #[derive(Copy, Clone, Eq, PartialEq, Debug)] + pub enum ExecutionStrategy { + /// Execute with the native equivalent if it is compatible with the given wasm module; otherwise fall back to the wasm. + NativeWhenPossible, + /// Use the given wasm module. + AlwaysWasm, + /// Run with both the wasm and the native variant (if compatible). Report any discrepancy as an error. + Both, + /// First native, then if that fails or is not possible, wasm. + NativeElseWasm, } -} -impl ExecutionStrategy { - /// Gets the corresponding manager for the execution strategy. - pub fn get_manager( - self, - ) -> ExecutionManager> { - match self { - ExecutionStrategy::AlwaysWasm => ExecutionManager::AlwaysWasm(BackendTrustLevel::Trusted), - ExecutionStrategy::NativeWhenPossible => ExecutionManager::NativeWhenPossible, - ExecutionStrategy::NativeElseWasm => ExecutionManager::NativeElseWasm, - ExecutionStrategy::Both => ExecutionManager::Both(|wasm_result, native_result| { - warn!( - "Consensus error between wasm {:?} and native {:?}. Using wasm.", - wasm_result, - native_result, - ); - warn!(" Native result {:?}", native_result); - warn!(" Wasm result {:?}", wasm_result); - wasm_result - }), - } + /// Storage backend trust level. + #[derive(Debug, Clone)] + pub enum BackendTrustLevel { + /// Panics from trusted backends are considered justified, and never caught. + Trusted, + /// Panics from untrusted backend are caught and interpreted as runtime error. + /// Untrusted backend may be missing some parts of the trie, so panics are not considered + /// fatal. + Untrusted, } -} -/// Evaluate to ExecutionManager::NativeElseWasm, without having to figure out the type. -pub fn native_else_wasm() -> ExecutionManager> { - ExecutionManager::NativeElseWasm -} + /// Like `ExecutionStrategy` only it also stores a handler in case of consensus failure. + #[derive(Clone)] + pub enum ExecutionManager { + /// Execute with the native equivalent if it is compatible with the given wasm module; otherwise fall back to the wasm. + NativeWhenPossible, + /// Use the given wasm module. The backend on which code is executed code could be + /// trusted to provide all storage or not (i.e. the light client cannot be trusted to provide + /// for all storage queries since the storage entries it has come from an external node). + AlwaysWasm(BackendTrustLevel), + /// Run with both the wasm and the native variant (if compatible). Call `F` in the case of any discrepancy. + Both(F), + /// First native, then if that fails or is not possible, wasm. + NativeElseWasm, + } -/// Evaluate to ExecutionManager::AlwaysWasm with trusted backend, without having to figure out the type. -fn always_wasm() -> ExecutionManager> { - ExecutionManager::AlwaysWasm(BackendTrustLevel::Trusted) -} + impl<'a, F> From<&'a ExecutionManager> for ExecutionStrategy { + fn from(s: &'a ExecutionManager) -> Self { + match *s { + ExecutionManager::NativeWhenPossible => ExecutionStrategy::NativeWhenPossible, + ExecutionManager::AlwaysWasm(_) => ExecutionStrategy::AlwaysWasm, + ExecutionManager::NativeElseWasm => ExecutionStrategy::NativeElseWasm, + ExecutionManager::Both(_) => ExecutionStrategy::Both, + } + } + } -/// Evaluate ExecutionManager::AlwaysWasm with untrusted backend, without having to figure out the type. -fn always_untrusted_wasm() -> ExecutionManager> { - ExecutionManager::AlwaysWasm(BackendTrustLevel::Untrusted) -} + impl ExecutionStrategy { + /// Gets the corresponding manager for the execution strategy. + pub fn get_manager( + self, + ) -> ExecutionManager> { + match self { + ExecutionStrategy::AlwaysWasm => ExecutionManager::AlwaysWasm(BackendTrustLevel::Trusted), + ExecutionStrategy::NativeWhenPossible => ExecutionManager::NativeWhenPossible, + ExecutionStrategy::NativeElseWasm => ExecutionManager::NativeElseWasm, + ExecutionStrategy::Both => ExecutionManager::Both(|wasm_result, native_result| { + warn!( + "Consensus error between wasm {:?} and native {:?}. Using wasm.", + wasm_result, + native_result, + ); + warn!(" Native result {:?}", native_result); + warn!(" Wasm result {:?}", wasm_result); + wasm_result + }), + } + } + } -/// The substrate state machine. -pub struct StateMachine<'a, B, H, N, Exec> - where - H: Hasher, - B: Backend, - N: ChangesTrieBlockNumber, -{ - backend: &'a B, - exec: &'a Exec, - method: &'a str, - call_data: &'a [u8], - overlay: &'a mut OverlayedChanges, - offchain_overlay: &'a mut OffchainOverlayedChanges, - extensions: Extensions, - changes_trie_state: Option>, - storage_transaction_cache: Option<&'a mut StorageTransactionCache>, - runtime_code: &'a RuntimeCode<'a>, - stats: StateMachineStats, -} + /// Evaluate to ExecutionManager::NativeElseWasm, without having to figure out the type. + pub fn native_else_wasm() -> ExecutionManager> { + ExecutionManager::NativeElseWasm + } -impl<'a, B, H, N, Exec> Drop for StateMachine<'a, B, H, N, Exec> where - H: Hasher, - B: Backend, - N: ChangesTrieBlockNumber, -{ - fn drop(&mut self) { - self.backend.register_overlay_stats(&self.stats); + /// Evaluate to ExecutionManager::AlwaysWasm with trusted backend, without having to figure out the type. + fn always_wasm() -> ExecutionManager> { + ExecutionManager::AlwaysWasm(BackendTrustLevel::Trusted) + } + + /// Evaluate ExecutionManager::AlwaysWasm with untrusted backend, without having to figure out the type. + fn always_untrusted_wasm() -> ExecutionManager> { + ExecutionManager::AlwaysWasm(BackendTrustLevel::Untrusted) } -} -impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where - H: Hasher, - H::Out: Ord + 'static + codec::Codec, - Exec: CodeExecutor + Clone + 'static, - B: Backend, - N: crate::changes_trie::BlockNumber, -{ - /// Creates new substrate state machine. - pub fn new( + /// The substrate state machine. + pub struct StateMachine<'a, B, H, N, Exec> + where + H: Hasher, + B: Backend, + N: ChangesTrieBlockNumber, + { backend: &'a B, - changes_trie_state: Option>, - overlay: &'a mut OverlayedChanges, - offchain_overlay: &'a mut OffchainOverlayedChanges, exec: &'a Exec, method: &'a str, call_data: &'a [u8], - mut extensions: Extensions, - runtime_code: &'a RuntimeCode, - spawn_handle: impl SpawnNamed + Send + 'static, - ) -> Self { - extensions.register(CallInWasmExt::new(exec.clone())); - extensions.register(sp_core::traits::TaskExecutorExt::new(spawn_handle)); - - Self { - backend, - exec, - method, - call_data, - extensions, - overlay, - offchain_overlay, - changes_trie_state, - storage_transaction_cache: None, - runtime_code, - stats: StateMachineStats::default(), - } - } - - /// Use given `cache` as storage transaction cache. - /// - /// The cache will be used to cache storage transactions that can be build while executing a - /// function in the runtime. For example, when calculating the storage root a transaction is - /// build that will be cached. - pub fn with_storage_transaction_cache( - mut self, - cache: Option<&'a mut StorageTransactionCache>, - ) -> Self { - self.storage_transaction_cache = cache; - self + overlay: &'a mut OverlayedChanges, + offchain_overlay: &'a mut OffchainOverlayedChanges, + extensions: Extensions, + changes_trie_state: Option>, + storage_transaction_cache: Option<&'a mut StorageTransactionCache>, + runtime_code: &'a RuntimeCode<'a>, + stats: StateMachineStats, } - /// Execute a call using the given state backend, overlayed changes, and call executor. - /// - /// On an error, no prospective changes are written to the overlay. - /// - /// Note: changes to code will be in place if this call is made again. For running partial - /// blocks (e.g. a transaction at a time), ensure a different method is used. - /// - /// Returns the SCALE encoded result of the executed function. - pub fn execute(&mut self, strategy: ExecutionStrategy) -> Result, Box> { - // We are not giving a native call and thus we are sure that the result can never be a native - // value. - self.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>( - strategy.get_manager(), - None, - ).map(NativeOrEncoded::into_encoded) + impl<'a, B, H, N, Exec> Drop for StateMachine<'a, B, H, N, Exec> where + H: Hasher, + B: Backend, + N: ChangesTrieBlockNumber, + { + fn drop(&mut self) { + self.backend.register_overlay_stats(&self.stats); + } } - fn execute_aux( - &mut self, - use_native: bool, - native_call: Option, - ) -> ( - CallResult, - bool, - ) where - R: Decode + Encode + PartialEq, - NC: FnOnce() -> result::Result + UnwindSafe, + impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where + H: Hasher, + H::Out: Ord + 'static + codec::Codec, + Exec: CodeExecutor + Clone + 'static, + B: Backend, + N: crate::changes_trie::BlockNumber, { - let mut cache = StorageTransactionCache::default(); + /// Creates new substrate state machine. + pub fn new( + backend: &'a B, + changes_trie_state: Option>, + overlay: &'a mut OverlayedChanges, + offchain_overlay: &'a mut OffchainOverlayedChanges, + exec: &'a Exec, + method: &'a str, + call_data: &'a [u8], + mut extensions: Extensions, + runtime_code: &'a RuntimeCode, + spawn_handle: impl SpawnNamed + Send + 'static, + ) -> Self { + extensions.register(CallInWasmExt::new(exec.clone())); + extensions.register(sp_core::traits::TaskExecutorExt::new(spawn_handle)); + + Self { + backend, + exec, + method, + call_data, + extensions, + overlay, + offchain_overlay, + changes_trie_state, + storage_transaction_cache: None, + runtime_code, + stats: StateMachineStats::default(), + } + } - let cache = match self.storage_transaction_cache.as_mut() { - Some(cache) => cache, - None => &mut cache, - }; + /// Use given `cache` as storage transaction cache. + /// + /// The cache will be used to cache storage transactions that can be build while executing a + /// function in the runtime. For example, when calculating the storage root a transaction is + /// build that will be cached. + pub fn with_storage_transaction_cache( + mut self, + cache: Option<&'a mut StorageTransactionCache>, + ) -> Self { + self.storage_transaction_cache = cache; + self + } - self.overlay.enter_runtime().expect("StateMachine is never called from the runtime; qed"); + /// Execute a call using the given state backend, overlayed changes, and call executor. + /// + /// On an error, no prospective changes are written to the overlay. + /// + /// Note: changes to code will be in place if this call is made again. For running partial + /// blocks (e.g. a transaction at a time), ensure a different method is used. + /// + /// Returns the SCALE encoded result of the executed function. + pub fn execute(&mut self, strategy: ExecutionStrategy) -> Result, Box> { + // We are not giving a native call and thus we are sure that the result can never be a native + // value. + self.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>( + strategy.get_manager(), + None, + ).map(NativeOrEncoded::into_encoded) + } - let mut ext = Ext::new( - self.overlay, - self.offchain_overlay, - cache, - self.backend, - self.changes_trie_state.clone(), - Some(&mut self.extensions), - ); + fn execute_aux( + &mut self, + use_native: bool, + native_call: Option, + ) -> ( + CallResult, + bool, + ) where + R: Decode + Encode + PartialEq, + NC: FnOnce() -> result::Result + UnwindSafe, + { + let mut cache = StorageTransactionCache::default(); - let id = ext.id; - trace!( - target: "state", "{:04x}: Call {} at {:?}. Input={:?}", - id, - self.method, - self.backend, - HexDisplay::from(&self.call_data), - ); + let cache = match self.storage_transaction_cache.as_mut() { + Some(cache) => cache, + None => &mut cache, + }; - let (result, was_native) = self.exec.call( - &mut ext, - self.runtime_code, - self.method, - self.call_data, - use_native, - native_call, - ); + self.overlay.enter_runtime().expect("StateMachine is never called from the runtime; qed"); - self.overlay.exit_runtime() - .expect("Runtime is not able to call this function in the overlay; qed"); + let mut ext = Ext::new( + self.overlay, + self.offchain_overlay, + cache, + self.backend, + self.changes_trie_state.clone(), + Some(&mut self.extensions), + ); - trace!( - target: "state", "{:04x}: Return. Native={:?}, Result={:?}", - id, - was_native, - result, - ); + let id = ext.id; + trace!( + target: "state", "{:04x}: Call {} at {:?}. Input={:?}", + id, + self.method, + self.backend, + HexDisplay::from(&self.call_data), + ); - (result, was_native) - } + let (result, was_native) = self.exec.call( + &mut ext, + self.runtime_code, + self.method, + self.call_data, + use_native, + native_call, + ); - fn execute_call_with_both_strategy( - &mut self, - mut native_call: Option, - on_consensus_failure: Handler, - ) -> CallResult - where - R: Decode + Encode + PartialEq, - NC: FnOnce() -> result::Result + UnwindSafe, - Handler: FnOnce( - CallResult, - CallResult, - ) -> CallResult - { - self.overlay.start_transaction(); - let (result, was_native) = self.execute_aux(true, native_call.take()); + self.overlay.exit_runtime() + .expect("Runtime is not able to call this function in the overlay; qed"); - if was_native { - self.overlay.rollback_transaction().expect(PROOF_CLOSE_TRANSACTION); - let (wasm_result, _) = self.execute_aux( - false, - native_call, + trace!( + target: "state", "{:04x}: Return. Native={:?}, Result={:?}", + id, + was_native, + result, + ); + + (result, was_native) + } + + fn execute_call_with_both_strategy( + &mut self, + mut native_call: Option, + on_consensus_failure: Handler, + ) -> CallResult + where + R: Decode + Encode + PartialEq, + NC: FnOnce() -> result::Result + UnwindSafe, + Handler: FnOnce( + CallResult, + CallResult, + ) -> CallResult + { + self.overlay.start_transaction(); + let (result, was_native) = self.execute_aux(true, native_call.take()); + + if was_native { + self.overlay.rollback_transaction().expect(PROOF_CLOSE_TRANSACTION); + let (wasm_result, _) = self.execute_aux( + false, + native_call, + ); + + if (result.is_ok() && wasm_result.is_ok() + && result.as_ref().ok() == wasm_result.as_ref().ok()) + || result.is_err() && wasm_result.is_err() + { + result + } else { + on_consensus_failure(wasm_result, result) + } + } else { + self.overlay.commit_transaction().expect(PROOF_CLOSE_TRANSACTION); + result + } + } + + fn execute_call_with_native_else_wasm_strategy( + &mut self, + mut native_call: Option, + ) -> CallResult + where + R: Decode + Encode + PartialEq, + NC: FnOnce() -> result::Result + UnwindSafe, + { + self.overlay.start_transaction(); + let (result, was_native) = self.execute_aux( + true, + native_call.take(), ); - if (result.is_ok() && wasm_result.is_ok() - && result.as_ref().ok() == wasm_result.as_ref().ok()) - || result.is_err() && wasm_result.is_err() - { + if !was_native || result.is_ok() { + self.overlay.commit_transaction().expect(PROOF_CLOSE_TRANSACTION); result } else { - on_consensus_failure(wasm_result, result) + self.overlay.rollback_transaction().expect(PROOF_CLOSE_TRANSACTION); + let (wasm_result, _) = self.execute_aux( + false, + native_call, + ); + wasm_result } - } else { - self.overlay.commit_transaction().expect(PROOF_CLOSE_TRANSACTION); - result + } + + /// Execute a call using the given state backend, overlayed changes, and call executor. + /// + /// On an error, no prospective changes are written to the overlay. + /// + /// Note: changes to code will be in place if this call is made again. For running partial + /// blocks (e.g. a transaction at a time), ensure a different method is used. + /// + /// Returns the result of the executed function either in native representation `R` or + /// in SCALE encoded representation. + pub fn execute_using_consensus_failure_handler( + &mut self, + manager: ExecutionManager, + mut native_call: Option, + ) -> Result, Box> + where + R: Decode + Encode + PartialEq, + NC: FnOnce() -> result::Result + UnwindSafe, + Handler: FnOnce( + CallResult, + CallResult, + ) -> CallResult + { + let changes_tries_enabled = self.changes_trie_state.is_some(); + self.overlay.set_collect_extrinsics(changes_tries_enabled); + + let result = { + match manager { + ExecutionManager::Both(on_consensus_failure) => { + self.execute_call_with_both_strategy( + native_call.take(), + on_consensus_failure, + ) + }, + ExecutionManager::NativeElseWasm => { + self.execute_call_with_native_else_wasm_strategy( + native_call.take(), + ) + }, + ExecutionManager::AlwaysWasm(trust_level) => { + let _abort_guard = match trust_level { + BackendTrustLevel::Trusted => None, + BackendTrustLevel::Untrusted => Some(sp_panic_handler::AbortGuard::never_abort()), + }; + self.execute_aux(false, native_call).0 + }, + ExecutionManager::NativeWhenPossible => { + self.execute_aux(true, native_call).0 + }, + } + }; + + result.map_err(|e| Box::new(e) as _) } } - fn execute_call_with_native_else_wasm_strategy( - &mut self, - mut native_call: Option, - ) -> CallResult - where - R: Decode + Encode + PartialEq, - NC: FnOnce() -> result::Result + UnwindSafe, + /// Prove execution using the given state backend, overlayed changes, and call executor. + pub fn prove_execution( + mut backend: B, + overlay: &mut OverlayedChanges, + exec: &Exec, + spawn_handle: Spawn, + method: &str, + call_data: &[u8], + runtime_code: &RuntimeCode, + ) -> Result<(Vec, StorageProof), Box> + where + B: Backend, + H: Hasher, + H::Out: Ord + 'static + codec::Codec, + Exec: CodeExecutor + Clone + 'static, + N: crate::changes_trie::BlockNumber, + Spawn: SpawnNamed + Send + 'static, { - self.overlay.start_transaction(); - let (result, was_native) = self.execute_aux( - true, - native_call.take(), - ); - - if !was_native || result.is_ok() { - self.overlay.commit_transaction().expect(PROOF_CLOSE_TRANSACTION); - result - } else { - self.overlay.rollback_transaction().expect(PROOF_CLOSE_TRANSACTION); - let (wasm_result, _) = self.execute_aux( - false, - native_call, - ); - wasm_result - } + let trie_backend = backend.as_trie_backend() + .ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box)?; + prove_execution_on_trie_backend::<_, _, N, _, _>( + trie_backend, + overlay, + exec, + spawn_handle, + method, + call_data, + runtime_code, + ) } - /// Execute a call using the given state backend, overlayed changes, and call executor. + /// Prove execution using the given trie backend, overlayed changes, and call executor. + /// Produces a state-backend-specific "transaction" which can be used to apply the changes + /// to the backing store, such as the disk. + /// Execution proof is the set of all 'touched' storage DBValues from the backend. /// /// On an error, no prospective changes are written to the overlay. /// /// Note: changes to code will be in place if this call is made again. For running partial /// blocks (e.g. a transaction at a time), ensure a different method is used. - /// - /// Returns the result of the executed function either in native representation `R` or - /// in SCALE encoded representation. - pub fn execute_using_consensus_failure_handler( - &mut self, - manager: ExecutionManager, - mut native_call: Option, - ) -> Result, Box> - where - R: Decode + Encode + PartialEq, - NC: FnOnce() -> result::Result + UnwindSafe, - Handler: FnOnce( - CallResult, - CallResult, - ) -> CallResult + pub fn prove_execution_on_trie_backend( + trie_backend: &TrieBackend, + overlay: &mut OverlayedChanges, + exec: &Exec, + spawn_handle: Spawn, + method: &str, + call_data: &[u8], + runtime_code: &RuntimeCode, + ) -> Result<(Vec, StorageProof), Box> + where + S: trie_backend_essence::TrieBackendStorage, + H: Hasher, + H::Out: Ord + 'static + codec::Codec, + Exec: CodeExecutor + 'static + Clone, + N: crate::changes_trie::BlockNumber, + Spawn: SpawnNamed + Send + 'static, { - let changes_tries_enabled = self.changes_trie_state.is_some(); - self.overlay.set_collect_extrinsics(changes_tries_enabled); - - let result = { - match manager { - ExecutionManager::Both(on_consensus_failure) => { - self.execute_call_with_both_strategy( - native_call.take(), - on_consensus_failure, - ) - }, - ExecutionManager::NativeElseWasm => { - self.execute_call_with_native_else_wasm_strategy( - native_call.take(), - ) - }, - ExecutionManager::AlwaysWasm(trust_level) => { - let _abort_guard = match trust_level { - BackendTrustLevel::Trusted => None, - BackendTrustLevel::Untrusted => Some(sp_panic_handler::AbortGuard::never_abort()), - }; - self.execute_aux(false, native_call).0 - }, - ExecutionManager::NativeWhenPossible => { - self.execute_aux(true, native_call).0 - }, - } - }; + let mut offchain_overlay = OffchainOverlayedChanges::default(); + let proving_backend = proving_backend::ProvingBackend::new(trie_backend); + let mut sm = StateMachine::<_, H, N, Exec>::new( + &proving_backend, + None, + overlay, + &mut offchain_overlay, + exec, + method, + call_data, + Extensions::default(), + runtime_code, + spawn_handle, + ); - result.map_err(|e| Box::new(e) as _) + let result = sm.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>( + always_wasm(), + None, + )?; + let proof = sm.backend.extract_proof(); + Ok((result.into_encoded(), proof)) } -} -/// Prove execution using the given state backend, overlayed changes, and call executor. -pub fn prove_execution( - mut backend: B, - overlay: &mut OverlayedChanges, - exec: &Exec, - spawn_handle: Spawn, - method: &str, - call_data: &[u8], - runtime_code: &RuntimeCode, -) -> Result<(Vec, StorageProof), Box> -where - B: Backend, - H: Hasher, - H::Out: Ord + 'static + codec::Codec, - Exec: CodeExecutor + Clone + 'static, - N: crate::changes_trie::BlockNumber, - Spawn: SpawnNamed + Send + 'static, -{ - let trie_backend = backend.as_trie_backend() - .ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box)?; - prove_execution_on_trie_backend::<_, _, N, _, _>( - trie_backend, - overlay, - exec, - spawn_handle, - method, - call_data, - runtime_code, - ) -} - -/// Prove execution using the given trie backend, overlayed changes, and call executor. -/// Produces a state-backend-specific "transaction" which can be used to apply the changes -/// to the backing store, such as the disk. -/// Execution proof is the set of all 'touched' storage DBValues from the backend. -/// -/// On an error, no prospective changes are written to the overlay. -/// -/// Note: changes to code will be in place if this call is made again. For running partial -/// blocks (e.g. a transaction at a time), ensure a different method is used. -pub fn prove_execution_on_trie_backend( - trie_backend: &TrieBackend, - overlay: &mut OverlayedChanges, - exec: &Exec, - spawn_handle: Spawn, - method: &str, - call_data: &[u8], - runtime_code: &RuntimeCode, -) -> Result<(Vec, StorageProof), Box> -where - S: trie_backend_essence::TrieBackendStorage, - H: Hasher, - H::Out: Ord + 'static + codec::Codec, - Exec: CodeExecutor + 'static + Clone, - N: crate::changes_trie::BlockNumber, - Spawn: SpawnNamed + Send + 'static, -{ - let mut offchain_overlay = OffchainOverlayedChanges::default(); - let proving_backend = proving_backend::ProvingBackend::new(trie_backend); - let mut sm = StateMachine::<_, H, N, Exec>::new( - &proving_backend, - None, - overlay, - &mut offchain_overlay, - exec, - method, - call_data, - Extensions::default(), - runtime_code, - spawn_handle, - ); - - let result = sm.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>( - always_wasm(), - None, - )?; - let proof = sm.backend.extract_proof(); - Ok((result.into_encoded(), proof)) -} - -/// Check execution proof, generated by `prove_execution` call. -pub fn execution_proof_check( - root: H::Out, - proof: StorageProof, - overlay: &mut OverlayedChanges, - exec: &Exec, - spawn_handle: Spawn, - method: &str, - call_data: &[u8], - runtime_code: &RuntimeCode, -) -> Result, Box> -where - H: Hasher, - Exec: CodeExecutor + Clone + 'static, - H::Out: Ord + 'static + codec::Codec, - N: crate::changes_trie::BlockNumber, - Spawn: SpawnNamed + Send + 'static, -{ - let trie_backend = create_proof_check_backend::(root.into(), proof)?; - execution_proof_check_on_trie_backend::<_, N, _, _>( - &trie_backend, - overlay, - exec, - spawn_handle, - method, - call_data, - runtime_code, - ) -} + /// Check execution proof, generated by `prove_execution` call. + pub fn execution_proof_check( + root: H::Out, + proof: StorageProof, + overlay: &mut OverlayedChanges, + exec: &Exec, + spawn_handle: Spawn, + method: &str, + call_data: &[u8], + runtime_code: &RuntimeCode, + ) -> Result, Box> + where + H: Hasher, + Exec: CodeExecutor + Clone + 'static, + H::Out: Ord + 'static + codec::Codec, + N: crate::changes_trie::BlockNumber, + Spawn: SpawnNamed + Send + 'static, + { + let trie_backend = create_proof_check_backend::(root.into(), proof)?; + execution_proof_check_on_trie_backend::<_, N, _, _>( + &trie_backend, + overlay, + exec, + spawn_handle, + method, + call_data, + runtime_code, + ) + } -/// Check execution proof on proving backend, generated by `prove_execution` call. -pub fn execution_proof_check_on_trie_backend( - trie_backend: &TrieBackend, H>, - overlay: &mut OverlayedChanges, - exec: &Exec, - spawn_handle: Spawn, - method: &str, - call_data: &[u8], - runtime_code: &RuntimeCode, -) -> Result, Box> -where - H: Hasher, - H::Out: Ord + 'static + codec::Codec, - Exec: CodeExecutor + Clone + 'static, - N: crate::changes_trie::BlockNumber, - Spawn: SpawnNamed + Send + 'static, -{ - let mut offchain_overlay = OffchainOverlayedChanges::default(); - let mut sm = StateMachine::<_, H, N, Exec>::new( - trie_backend, - None, - overlay, - &mut offchain_overlay, - exec, - method, - call_data, - Extensions::default(), - runtime_code, - spawn_handle, - ); + /// Check execution proof on proving backend, generated by `prove_execution` call. + pub fn execution_proof_check_on_trie_backend( + trie_backend: &TrieBackend, H>, + overlay: &mut OverlayedChanges, + exec: &Exec, + spawn_handle: Spawn, + method: &str, + call_data: &[u8], + runtime_code: &RuntimeCode, + ) -> Result, Box> + where + H: Hasher, + H::Out: Ord + 'static + codec::Codec, + Exec: CodeExecutor + Clone + 'static, + N: crate::changes_trie::BlockNumber, + Spawn: SpawnNamed + Send + 'static, + { + let mut offchain_overlay = OffchainOverlayedChanges::default(); + let mut sm = StateMachine::<_, H, N, Exec>::new( + trie_backend, + None, + overlay, + &mut offchain_overlay, + exec, + method, + call_data, + Extensions::default(), + runtime_code, + spawn_handle, + ); - sm.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>( - always_untrusted_wasm(), - None, - ).map(NativeOrEncoded::into_encoded) -} + sm.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>( + always_untrusted_wasm(), + None, + ).map(NativeOrEncoded::into_encoded) + } -/// Generate storage read proof. -pub fn prove_read( - mut backend: B, - keys: I, -) -> Result> -where - B: Backend, - H: Hasher, - H::Out: Ord + Codec, - I: IntoIterator, - I::Item: AsRef<[u8]>, -{ - let trie_backend = backend.as_trie_backend() - .ok_or_else( - || Box::new(ExecutionError::UnableToGenerateProof) as Box - )?; - prove_read_on_trie_backend(trie_backend, keys) -} + /// Generate storage read proof. + pub fn prove_read( + mut backend: B, + keys: I, + ) -> Result> + where + B: Backend, + H: Hasher, + H::Out: Ord + Codec, + I: IntoIterator, + I::Item: AsRef<[u8]>, + { + let trie_backend = backend.as_trie_backend() + .ok_or_else( + || Box::new(ExecutionError::UnableToGenerateProof) as Box + )?; + prove_read_on_trie_backend(trie_backend, keys) + } -/// Generate child storage read proof. -pub fn prove_child_read( - mut backend: B, - child_info: &ChildInfo, - keys: I, -) -> Result> -where - B: Backend, - H: Hasher, - H::Out: Ord + Codec, - I: IntoIterator, - I::Item: AsRef<[u8]>, -{ - let trie_backend = backend.as_trie_backend() - .ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box)?; - prove_child_read_on_trie_backend(trie_backend, child_info, keys) -} + /// Generate child storage read proof. + pub fn prove_child_read( + mut backend: B, + child_info: &ChildInfo, + keys: I, + ) -> Result> + where + B: Backend, + H: Hasher, + H::Out: Ord + Codec, + I: IntoIterator, + I::Item: AsRef<[u8]>, + { + let trie_backend = backend.as_trie_backend() + .ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box)?; + prove_child_read_on_trie_backend(trie_backend, child_info, keys) + } -/// Generate storage read proof on pre-created trie backend. -pub fn prove_read_on_trie_backend( - trie_backend: &TrieBackend, - keys: I, -) -> Result> -where - S: trie_backend_essence::TrieBackendStorage, - H: Hasher, - H::Out: Ord + Codec, - I: IntoIterator, - I::Item: AsRef<[u8]>, -{ - let proving_backend = proving_backend::ProvingBackend::<_, H>::new(trie_backend); - for key in keys.into_iter() { - proving_backend - .storage(key.as_ref()) - .map_err(|e| Box::new(e) as Box)?; + /// Generate storage read proof on pre-created trie backend. + pub fn prove_read_on_trie_backend( + trie_backend: &TrieBackend, + keys: I, + ) -> Result> + where + S: trie_backend_essence::TrieBackendStorage, + H: Hasher, + H::Out: Ord + Codec, + I: IntoIterator, + I::Item: AsRef<[u8]>, + { + let proving_backend = proving_backend::ProvingBackend::<_, H>::new(trie_backend); + for key in keys.into_iter() { + proving_backend + .storage(key.as_ref()) + .map_err(|e| Box::new(e) as Box)?; + } + Ok(proving_backend.extract_proof()) } - Ok(proving_backend.extract_proof()) -} -/// Generate storage read proof on pre-created trie backend. -pub fn prove_child_read_on_trie_backend( - trie_backend: &TrieBackend, - child_info: &ChildInfo, - keys: I, -) -> Result> -where - S: trie_backend_essence::TrieBackendStorage, - H: Hasher, - H::Out: Ord + Codec, - I: IntoIterator, - I::Item: AsRef<[u8]>, -{ - let proving_backend = proving_backend::ProvingBackend::<_, H>::new(trie_backend); - for key in keys.into_iter() { - proving_backend - .child_storage(child_info, key.as_ref()) - .map_err(|e| Box::new(e) as Box)?; + /// Generate storage read proof on pre-created trie backend. + pub fn prove_child_read_on_trie_backend( + trie_backend: &TrieBackend, + child_info: &ChildInfo, + keys: I, + ) -> Result> + where + S: trie_backend_essence::TrieBackendStorage, + H: Hasher, + H::Out: Ord + Codec, + I: IntoIterator, + I::Item: AsRef<[u8]>, + { + let proving_backend = proving_backend::ProvingBackend::<_, H>::new(trie_backend); + for key in keys.into_iter() { + proving_backend + .child_storage(child_info, key.as_ref()) + .map_err(|e| Box::new(e) as Box)?; + } + Ok(proving_backend.extract_proof()) } - Ok(proving_backend.extract_proof()) -} -/// Check storage read proof, generated by `prove_read` call. -pub fn read_proof_check( - root: H::Out, - proof: StorageProof, - keys: I, -) -> Result, Option>>, Box> -where - H: Hasher, - H::Out: Ord + Codec, - I: IntoIterator, - I::Item: AsRef<[u8]>, -{ - let proving_backend = create_proof_check_backend::(root, proof)?; - let mut result = HashMap::new(); - for key in keys.into_iter() { - let value = read_proof_check_on_proving_backend(&proving_backend, key.as_ref())?; - result.insert(key.as_ref().to_vec(), value); + /// Check storage read proof, generated by `prove_read` call. + pub fn read_proof_check( + root: H::Out, + proof: StorageProof, + keys: I, + ) -> Result, Option>>, Box> + where + H: Hasher, + H::Out: Ord + Codec, + I: IntoIterator, + I::Item: AsRef<[u8]>, + { + let proving_backend = create_proof_check_backend::(root, proof)?; + let mut result = HashMap::new(); + for key in keys.into_iter() { + let value = read_proof_check_on_proving_backend(&proving_backend, key.as_ref())?; + result.insert(key.as_ref().to_vec(), value); + } + Ok(result) } - Ok(result) -} -/// Check child storage read proof, generated by `prove_child_read` call. -pub fn read_child_proof_check( - root: H::Out, - proof: StorageProof, - child_info: &ChildInfo, - keys: I, -) -> Result, Option>>, Box> -where - H: Hasher, - H::Out: Ord + Codec, - I: IntoIterator, - I::Item: AsRef<[u8]>, -{ - let proving_backend = create_proof_check_backend::(root, proof)?; - let mut result = HashMap::new(); - for key in keys.into_iter() { - let value = read_child_proof_check_on_proving_backend( - &proving_backend, - child_info, - key.as_ref(), - )?; - result.insert(key.as_ref().to_vec(), value); + /// Check child storage read proof, generated by `prove_child_read` call. + pub fn read_child_proof_check( + root: H::Out, + proof: StorageProof, + child_info: &ChildInfo, + keys: I, + ) -> Result, Option>>, Box> + where + H: Hasher, + H::Out: Ord + Codec, + I: IntoIterator, + I::Item: AsRef<[u8]>, + { + let proving_backend = create_proof_check_backend::(root, proof)?; + let mut result = HashMap::new(); + for key in keys.into_iter() { + let value = read_child_proof_check_on_proving_backend( + &proving_backend, + child_info, + key.as_ref(), + )?; + result.insert(key.as_ref().to_vec(), value); + } + Ok(result) } - Ok(result) -} -/// Check storage read proof on pre-created proving backend. -pub fn read_proof_check_on_proving_backend( - proving_backend: &TrieBackend, H>, - key: &[u8], -) -> Result>, Box> -where - H: Hasher, - H::Out: Ord + Codec, -{ - proving_backend.storage(key).map_err(|e| Box::new(e) as Box) -} + /// Check storage read proof on pre-created proving backend. + pub fn read_proof_check_on_proving_backend( + proving_backend: &TrieBackend, H>, + key: &[u8], + ) -> Result>, Box> + where + H: Hasher, + H::Out: Ord + Codec, + { + proving_backend.storage(key).map_err(|e| Box::new(e) as Box) + } -/// Check child storage read proof on pre-created proving backend. -pub fn read_child_proof_check_on_proving_backend( - proving_backend: &TrieBackend, H>, - child_info: &ChildInfo, - key: &[u8], -) -> Result>, Box> -where - H: Hasher, - H::Out: Ord + Codec, -{ - proving_backend.child_storage(child_info, key) - .map_err(|e| Box::new(e) as Box) + /// Check child storage read proof on pre-created proving backend. + pub fn read_child_proof_check_on_proving_backend( + proving_backend: &TrieBackend, H>, + child_info: &ChildInfo, + key: &[u8], + ) -> Result>, Box> + where + H: Hasher, + H::Out: Ord + Codec, + { + proving_backend.child_storage(child_info, key) + .map_err(|e| Box::new(e) as Box) + } } #[cfg(test)] @@ -772,6 +831,15 @@ mod tests { map, traits::{Externalities, RuntimeCode}, testing::TaskExecutor, }; use sp_runtime::traits::BlakeTwo256; + use std::{result, collections::HashMap}; + use codec::Decode; + use sp_core::{ + offchain::storage::OffchainOverlayedChanges, + storage::ChildInfo, NativeOrEncoded, NeverNativeValue, + traits::CodeExecutor, + }; + use crate::execution::CallResult; + #[derive(Clone)] struct DummyCodeExecutor { diff --git a/primitives/state-machine/src/overlayed_changes/changeset.rs b/primitives/state-machine/src/overlayed_changes/changeset.rs index fe43c0ea99d89..32926dc3cb3a7 100644 --- a/primitives/state-machine/src/overlayed_changes/changeset.rs +++ b/primitives/state-machine/src/overlayed_changes/changeset.rs @@ -17,19 +17,24 @@ //! Houses the code that implements the transactional overlay storage. -use super::{StorageKey, StorageValue}; +use super::{StorageKey, StorageValue, ChangeTrieOverlay}; -use itertools::Itertools; -use std::collections::{HashSet, BTreeMap, BTreeSet}; +#[cfg(feature = "std")] +use std::collections::HashSet as Set; +#[cfg(not(feature = "std"))] +use sp_std::collections::btree_set::BTreeSet as Set; + +use sp_std::collections::btree_map::BTreeMap; +use sp_std::collections::btree_set::BTreeSet; use smallvec::SmallVec; -use log::warn; +use crate::warn; const PROOF_OVERLAY_NON_EMPTY: &str = "\ An OverlayValue is always created with at least one transaction and dropped as soon as the last transaction is removed; qed"; -type DirtyKeysSets = SmallVec<[HashSet; 5]>; -type Transactions = SmallVec<[InnerValue; 5]>; +type DirtyKeysSets = SmallVec<[Set; 5]>; +type Transactions = SmallVec<[InnerValue; 5]>; /// Error returned when trying to commit or rollback while no transaction is open or /// when the runtime is trying to close a transaction started by the client. @@ -58,28 +63,28 @@ pub enum ExecutionMode { #[derive(Debug, Default, Clone)] #[cfg_attr(test, derive(PartialEq))] -struct InnerValue { +struct InnerValue { /// Current value. None if value has been deleted. value: Option, /// The set of extrinsic indices where the values has been changed. /// Is filled only if runtime has announced changes trie support. - extrinsics: BTreeSet, + extrinsics: CT, } /// An overlay that contains all versions of a value for a specific key. #[derive(Debug, Default, Clone)] #[cfg_attr(test, derive(PartialEq))] -pub struct OverlayedValue { +pub struct OverlayedValue { /// The individual versions of that value. /// One entry per transactions during that the value was actually written. - transactions: Transactions, + transactions: Transactions, } /// Holds a set of changes with the ability modify them using nested transactions. #[derive(Debug, Default, Clone)] -pub struct OverlayedChangeSet { +pub struct OverlayedChangeSet { /// Stores the changes that this overlay constitutes. - changes: BTreeMap, + changes: BTreeMap>, /// Stores which keys are dirty per transaction. Needed in order to determine which /// values to merge into the parent transaction on commit. The length of this vector /// therefore determines how many nested transactions are currently open (depth). @@ -98,15 +103,17 @@ impl Default for ExecutionMode { } } -impl OverlayedValue { +impl OverlayedValue { /// The value as seen by the current transaction. pub fn value(&self) -> Option<&StorageValue> { self.transactions.last().expect(PROOF_OVERLAY_NON_EMPTY).value.as_ref() } /// Unique list of extrinsic indices which modified the value. - pub fn extrinsics(&self) -> impl Iterator { - self.transactions.iter().flat_map(|t| t.extrinsics.iter()).unique() + pub fn extrinsics(&self) -> BTreeSet { + let mut set = BTreeSet::new(); + self.transactions.iter().for_each(|t| t.extrinsics.flush_content(&mut set)); + set } /// Mutable reference to the most recent version. @@ -115,12 +122,12 @@ impl OverlayedValue { } /// Remove the last version and return it. - fn pop_transaction(&mut self) -> InnerValue { + fn pop_transaction(&mut self) -> InnerValue { self.transactions.pop().expect(PROOF_OVERLAY_NON_EMPTY) } /// Mutable reference to the set which holds the indices for the **current transaction only**. - fn transaction_extrinsics_mut(&mut self) -> &mut BTreeSet { + fn transaction_extrinsics_mut(&mut self) -> &mut CT { &mut self.transactions.last_mut().expect(PROOF_OVERLAY_NON_EMPTY).extrinsics } @@ -157,15 +164,15 @@ fn insert_dirty(set: &mut DirtyKeysSets, key: StorageKey) -> bool { set.last_mut().map(|dk| dk.insert(key)).unwrap_or_default() } -impl OverlayedChangeSet { +impl OverlayedChangeSet { /// Create a new changeset at the same transaction state but without any contents. /// /// This changeset might be created when there are already open transactions. /// We need to catch up here so that the child is at the same transaction depth. pub fn spawn_child(&self) -> Self { - use std::iter::repeat; + use sp_std::iter::repeat; Self { - dirty_keys: repeat(HashSet::new()).take(self.transaction_depth()).collect(), + dirty_keys: repeat(Set::new()).take(self.transaction_depth()).collect(), num_client_transactions: self.num_client_transactions, execution_mode: self.execution_mode, .. Default::default() @@ -178,7 +185,7 @@ impl OverlayedChangeSet { } /// Get an optional reference to the value stored for the specified key. - pub fn get(&self, key: &[u8]) -> Option<&OverlayedValue> { + pub fn get(&self, key: &[u8]) -> Option<&OverlayedValue> { self.changes.get(key) } @@ -228,7 +235,7 @@ impl OverlayedChangeSet { /// Can be rolled back or committed when called inside a transaction. pub fn clear_where( &mut self, - predicate: impl Fn(&[u8], &OverlayedValue) -> bool, + predicate: impl Fn(&[u8], &OverlayedValue) -> bool, at_extrinsic: Option, ) { for (key, val) in self.changes.iter_mut().filter(|(k, v)| predicate(k, v)) { @@ -237,13 +244,13 @@ impl OverlayedChangeSet { } /// Get a list of all changes as seen by current transaction. - pub fn changes(&self) -> impl Iterator { + pub fn changes(&self) -> impl Iterator)> { self.changes.iter() } /// Get the change that is next to the supplied key. - pub fn next_change(&self, key: &[u8]) -> Option<(&[u8], &OverlayedValue)> { - use std::ops::Bound; + pub fn next_change(&self, key: &[u8]) -> Option<(&[u8], &OverlayedValue)> { + use sp_std::ops::Bound; let range = (Bound::Excluded(key), Bound::Unbounded); self.changes.range::<[u8], _>(range).next().map(|(k, v)| (&k[..], v)) } @@ -365,7 +372,7 @@ impl OverlayedChangeSet { if has_predecessor { let dropped_tx = overlayed.pop_transaction(); *overlayed.value_mut() = dropped_tx.value; - overlayed.transaction_extrinsics_mut().extend(dropped_tx.extrinsics); + overlayed.transaction_extrinsics_mut().merge(dropped_tx.extrinsics); } } } @@ -382,13 +389,15 @@ impl OverlayedChangeSet { mod test { use super::*; use pretty_assertions::assert_eq; + use crate::overlayed_changes::Extrinsics; + type OverlayedChangeSet = super::OverlayedChangeSet; type Changes<'a> = Vec<(&'a [u8], (Option<&'a [u8]>, Vec))>; type Drained<'a> = Vec<(&'a [u8], Option<&'a [u8]>)>; fn assert_changes(is: &OverlayedChangeSet, expected: &Changes) { let is: Changes = is.changes().map(|(k, v)| { - (k.as_ref(), (v.value().map(AsRef::as_ref), v.extrinsics().cloned().collect())) + (k.as_ref(), (v.value().map(AsRef::as_ref), v.extrinsics().into_iter().collect())) }).collect(); assert_eq!(&is, expected); } diff --git a/primitives/state-machine/src/overlayed_changes/mod.rs b/primitives/state-machine/src/overlayed_changes/mod.rs index 9a2b1c4197310..b677d0a4ef3c9 100644 --- a/primitives/state-machine/src/overlayed_changes/mod.rs +++ b/primitives/state-machine/src/overlayed_changes/mod.rs @@ -20,20 +20,31 @@ mod changeset; use crate::{ - backend::Backend, ChangesTrieTransaction, + backend::Backend, + stats::StateMachineStats, +}; +use sp_std::vec::Vec; +use self::changeset::OverlayedChangeSet; + +#[cfg(feature = "std")] +use crate::{ + ChangesTrieTransaction, changes_trie::{ NO_EXTRINSIC_INDEX, BlockNumber, build_changes_trie, State as ChangesTrieState, }, - stats::StateMachineStats, }; -use self::changeset::OverlayedChangeSet; - -use std::collections::HashMap; +#[cfg(feature = "std")] +use std::collections::HashMap as Map; +#[cfg(not(feature = "std"))] +use sp_std::collections::btree_map::BTreeMap as Map; +use sp_std::collections::btree_set::BTreeSet; use codec::{Decode, Encode}; use sp_core::storage::{well_known_keys::EXTRINSIC_INDEX, ChildInfo}; +#[cfg(feature = "std")] use sp_core::offchain::storage::OffchainOverlayedChanges; use hash_db::Hasher; +use crate::DefaultError; pub use self::changeset::{OverlayedValue, NoOpenTransaction, AlreadyInRuntime, NotInRuntime}; @@ -49,15 +60,64 @@ pub type StorageCollection = Vec<(StorageKey, Option)>; /// In memory arrays of storage values for multiple child tries. pub type ChildStorageCollection = Vec<(StorageKey, StorageCollection)>; +/// `OverlayedChanges` without the capability to record +/// change trie. +#[cfg(not(feature = "std"))] +pub(crate) type OverlayedChangesNoChangeTrie = OverlayedChanges; + +/// Technical trait that allows to disable child trie +/// footprint when not needed. +pub trait ChangeTrieOverlay: Default { + /// If define to false we will skip change trie specifics. + const CHANGE_TRIE_CAPABLE: bool = true; + /// Extracts extrinsics into a `BTreeSets`. + fn flush_content(&self, dest: &mut BTreeSet); + /// Add an extrinsics. + fn insert(&mut self, ext: u32); + /// Merge two extrinsics sets. + fn merge(&mut self, other: Self); +} + +/// Default implementation of ChangeTrieOverlay. +#[derive(Debug, Default, Eq, PartialEq)] +pub struct Extrinsics(Vec); + +#[derive(Default)] +pub(crate) struct NoExtrinsics; + +impl ChangeTrieOverlay for NoExtrinsics { + const CHANGE_TRIE_CAPABLE: bool = false; + fn flush_content(&self, _dest: &mut BTreeSet) { + } + fn insert(&mut self, _ext: u32) { + } + fn merge(&mut self, _other: Self) { + } +} + +impl ChangeTrieOverlay for Extrinsics { + fn flush_content(&self, dest: &mut BTreeSet) { + dest.extend(self.0.iter()) + } + fn insert(&mut self, ext: u32) { + if Some(&ext) != self.0.last() { + self.0.push(ext); + } + } + fn merge(&mut self, other: Self) { + self.0.extend(other.0.into_iter()); + } +} + /// The set of changes that are overlaid onto the backend. /// /// It allows changes to be modified using nestable transactions. #[derive(Debug, Default, Clone)] -pub struct OverlayedChanges { +pub struct OverlayedChanges { /// Top level storage changes. - top: OverlayedChangeSet, + top: OverlayedChangeSet, /// Child storage changes. The map key is the child storage key without the common prefix. - children: HashMap, + children: Map, ChildInfo)>, /// True if extrinsics stats must be collected. collect_extrinsics: bool, /// Collect statistic on this execution. @@ -68,6 +128,7 @@ pub struct OverlayedChanges { /// /// This contains all the changes to the storage and transactions to apply theses changes to the /// backend. +#[cfg(feature = "std")] pub struct StorageChanges { /// All changes to the main storage. /// @@ -90,6 +151,7 @@ pub struct StorageChanges { pub changes_trie_transaction: Option>, } +#[cfg(feature = "std")] impl StorageChanges { /// Deconstruct into the inner values pub fn into_inner(self) -> ( @@ -143,6 +205,7 @@ impl Default for StorageTransactionCache } } +#[cfg(feature = "std")] impl Default for StorageChanges { fn default() -> Self { Self { @@ -156,7 +219,7 @@ impl Default for StorageChanges } } -impl OverlayedChanges { +impl OverlayedChanges { /// Whether no changes are contained in the top nor in any of the child changes. pub fn is_empty(&self) -> bool { self.top.is_empty() && self.children.is_empty() @@ -379,7 +442,7 @@ impl OverlayedChanges { impl Iterator)>, impl Iterator)>, ChildInfo))>, ) { - use std::mem::take; + use sp_std::mem::take; ( take(&mut self.top).drain_commited(), take(&mut self.children).into_iter() @@ -393,22 +456,23 @@ impl OverlayedChanges { /// Get an iterator over all child changes as seen by the current transaction. pub fn children(&self) - -> impl Iterator, &ChildInfo)> { + -> impl Iterator)>, &ChildInfo)> { self.children.iter().map(|(_, v)| (v.0.changes(), &v.1)) } /// Get an iterator over all top changes as been by the current transaction. - pub fn changes(&self) -> impl Iterator { + pub fn changes(&self) -> impl Iterator)> { self.top.changes() } /// Get an optional iterator over all child changes stored under the supplied key. pub fn child_changes(&self, key: &[u8]) - -> Option<(impl Iterator, &ChildInfo)> { + -> Option<(impl Iterator)>, &ChildInfo)> { self.children.get(key).map(|(overlay, info)| (overlay.changes(), info)) } /// Convert this instance with all changes into a [`StorageChanges`] instance. + #[cfg(feature = "std")] pub fn into_storage_changes< B: Backend, H: Hasher, N: BlockNumber >( @@ -417,18 +481,19 @@ impl OverlayedChanges { changes_trie_state: Option<&ChangesTrieState>, parent_hash: H::Out, mut cache: StorageTransactionCache, - ) -> Result, String> where H::Out: Ord + Encode + 'static { + ) -> Result, DefaultError> where H::Out: Ord + Encode + 'static { self.drain_storage_changes(backend, changes_trie_state, parent_hash, &mut cache) } /// Drain all changes into a [`StorageChanges`] instance. Leave empty overlay in place. + #[cfg(feature = "std")] pub fn drain_storage_changes, H: Hasher, N: BlockNumber>( &mut self, backend: &B, changes_trie_state: Option<&ChangesTrieState>, parent_hash: H::Out, mut cache: &mut StorageTransactionCache, - ) -> Result, String> where H::Out: Ord + Encode + 'static { + ) -> Result, DefaultError> where H::Out: Ord + Encode + 'static { // If the transaction does not exist, we generate it. if cache.transaction.is_none() { self.storage_root(backend, &mut cache); @@ -528,7 +593,10 @@ impl OverlayedChanges { panic_on_storage_error: bool, cache: &mut StorageTransactionCache, ) -> Result, ()> where H::Out: Ord + Encode + 'static { - build_changes_trie::<_, H, N>( + if !CT::CHANGE_TRIE_CAPABLE { + return Ok(None); + } + build_changes_trie::<_, H, N, CT>( backend, changes_trie_state, self, @@ -544,7 +612,7 @@ impl OverlayedChanges { /// Returns the next (in lexicographic order) storage key in the overlayed alongside its value. /// If no value is next then `None` is returned. - pub fn next_storage_key_change(&self, key: &[u8]) -> Option<(&[u8], &OverlayedValue)> { + pub fn next_storage_key_change(&self, key: &[u8]) -> Option<(&[u8], &OverlayedValue)> { self.top.next_change(key) } @@ -554,7 +622,7 @@ impl OverlayedChanges { &self, storage_key: &[u8], key: &[u8] - ) -> Option<(&[u8], &OverlayedValue)> { + ) -> Option<(&[u8], &OverlayedValue)> { self.children .get(storage_key) .and_then(|(overlay, _)| @@ -572,13 +640,15 @@ mod tests { use super::*; use std::collections::BTreeMap; + type OverlayedChanges = super::OverlayedChanges; + fn assert_extrinsics( - overlay: &OverlayedChangeSet, + overlay: &OverlayedChangeSet, key: impl AsRef<[u8]>, expected: Vec, ) { assert_eq!( - overlay.get(key.as_ref()).unwrap().extrinsics().cloned().collect::>(), + overlay.get(key.as_ref()).unwrap().extrinsics().into_iter().collect::>(), expected ) } diff --git a/primitives/state-machine/src/stats.rs b/primitives/state-machine/src/stats.rs index a8ca5a3b416f4..3ed0b96d16d5e 100644 --- a/primitives/state-machine/src/stats.rs +++ b/primitives/state-machine/src/stats.rs @@ -17,8 +17,22 @@ //! Usage statistics for state db +#[cfg(feature = "std")] use std::time::{Instant, Duration}; -use std::cell::RefCell; +#[cfg(not(feature = "std"))] +type Instant = (); +#[cfg(not(feature = "std"))] +use core::time::Duration; +use sp_std::cell::RefCell; + +#[cfg(feature = "std")] +fn now() -> Instant { + Instant::now() +} +#[cfg(not(feature = "std"))] +fn now() -> Instant { + () +} /// Measured count of operations and total bytes. #[derive(Clone, Debug, Default)] @@ -89,6 +103,7 @@ impl UsageInfo { /// Empty statistics. /// /// Means no data was collected. + #[cfg(feature = "std")] pub fn empty() -> Self { Self { reads: UsageUnit::default(), @@ -99,7 +114,7 @@ impl UsageInfo { cache_reads: UsageUnit::default(), modified_reads: UsageUnit::default(), memory: 0, - started: Instant::now(), + started: now(), span: Default::default(), } } diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index e0a86bbd193a1..39d5f611e0e4a 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -17,7 +17,7 @@ //! Trie-based state machine backend. -use log::{warn, debug}; +use crate::{warn, debug}; use hash_db::Hasher; use sp_trie::{Trie, delta_trie_root, empty_child_trie_root, child_delta_trie_root}; use sp_trie::trie_types::{TrieDB, TrieError, Layout}; @@ -27,6 +27,8 @@ use crate::{ StorageKey, StorageValue, Backend, trie_backend_essence::{TrieBackendEssence, TrieBackendStorage, Ephemeral}, }; +use sp_std::boxed::Box; +use sp_std::vec::Vec; /// Patricia trie-based backend. Transaction type is an overlay of changes to commit. pub struct TrieBackend, H: Hasher> { @@ -67,8 +69,8 @@ impl, H: Hasher> TrieBackend where H::Out: Codec } } -impl, H: Hasher> std::fmt::Debug for TrieBackend { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl, H: Hasher> sp_std::fmt::Debug for TrieBackend { + fn fmt(&self, f: &mut sp_std::fmt::Formatter<'_>) -> sp_std::fmt::Result { write!(f, "TrieBackend") } } @@ -76,7 +78,7 @@ impl, H: Hasher> std::fmt::Debug for TrieBackend impl, H: Hasher> Backend for TrieBackend where H::Out: Ord + Codec, { - type Error = String; + type Error = crate::DefaultError; type Transaction = S::Overlay; type TrieBackendStorage = S; diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index 72864e312b6ab..e43b266e1057b 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -18,9 +18,10 @@ //! Trie-based state machine backend essence used to read values //! from storage. -use std::ops::Deref; +#[cfg(feature = "std")] use std::sync::Arc; -use log::{debug, warn}; +use sp_std::ops::Deref; +use crate::{warn, debug}; use hash_db::{self, Hasher, Prefix}; use sp_trie::{Trie, MemoryDB, PrefixedMemoryDB, DBValue, empty_child_trie_root, read_trie_value, read_child_trie_value, @@ -29,11 +30,24 @@ use sp_trie::trie_types::{TrieDB, TrieError, Layout}; use crate::{backend::Consolidate, StorageKey, StorageValue}; use sp_core::storage::ChildInfo; use codec::Encode; +use sp_std::boxed::Box; +use sp_std::vec::Vec; + +#[cfg(feature = "std")] +use std::fmt::format; +#[cfg(not(feature = "std"))] +macro_rules! format { + ($($arg:tt)+) => ( + () + ); +} + +type Result = sp_std::result::Result; /// Patricia trie-based storage trait. pub trait Storage: Send + Sync { /// Get a trie node. - fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String>; + fn get(&self, key: &H::Out, prefix: Prefix) -> Result>; } /// Patricia trie-based pairs storage essence. @@ -80,12 +94,12 @@ impl, H: Hasher> TrieBackendEssence where H::Out: /// Return the next key in the trie i.e. the minimum key that is strictly superior to `key` in /// lexicographic order. - pub fn next_storage_key(&self, key: &[u8]) -> Result, String> { + pub fn next_storage_key(&self, key: &[u8]) -> Result> { self.next_storage_key_from_root(&self.root, None, key) } /// Access the root of the child storage in its parent trie - fn child_root(&self, child_info: &ChildInfo) -> Result, String> { + fn child_root(&self, child_info: &ChildInfo) -> Result> { self.storage(child_info.prefixed_storage_key().as_slice()) } @@ -95,7 +109,7 @@ impl, H: Hasher> TrieBackendEssence where H::Out: &self, child_info: &ChildInfo, key: &[u8], - ) -> Result, String> { + ) -> Result> { let child_root = match self.child_root(child_info)? { Some(child_root) => child_root, None => return Ok(None), @@ -118,7 +132,7 @@ impl, H: Hasher> TrieBackendEssence where H::Out: root: &H::Out, child_info: Option<&ChildInfo>, key: &[u8], - ) -> Result, String> { + ) -> Result> { let dyn_eph: &dyn hash_db::HashDBRef<_, _>; let keyspace_eph; if let Some(child_info) = child_info.as_ref() { @@ -158,7 +172,7 @@ impl, H: Hasher> TrieBackendEssence where H::Out: } /// Get the value of storage at given key. - pub fn storage(&self, key: &[u8]) -> Result, String> { + pub fn storage(&self, key: &[u8]) -> Result> { let map_e = |e| format!("Trie lookup error: {}", e); read_trie_value::, _>(self, &self.root, key).map_err(map_e) @@ -169,7 +183,7 @@ impl, H: Hasher> TrieBackendEssence where H::Out: &self, child_info: &ChildInfo, key: &[u8], - ) -> Result, String> { + ) -> Result> { let root = self.child_root(child_info)? .unwrap_or_else(|| empty_child_trie_root::>().encode()); @@ -234,7 +248,7 @@ impl, H: Hasher> TrieBackendEssence where H::Out: mut f: F, child_info: Option<&ChildInfo>, ) { - let mut iter = move |db| -> Result<(), Box>> { + let mut iter = move |db| -> sp_std::result::Result<(), Box>> { let trie = TrieDB::::new(db, root)?; for x in TrieDBIterator::new_prefixed(&trie, prefix)? { @@ -337,14 +351,15 @@ pub trait TrieBackendStorage: Send + Sync { /// Type of in-memory overlay. type Overlay: hash_db::HashDB + Default + Consolidate; /// Get the value stored at key. - fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String>; + fn get(&self, key: &H::Out, prefix: Prefix) -> Result>; } // This implementation is used by normal storage trie clients. +#[cfg(feature = "std")] impl TrieBackendStorage for Arc> { type Overlay = PrefixedMemoryDB; - fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String> { + fn get(&self, key: &H::Out, prefix: Prefix) -> Result> { Storage::::get(self.deref(), key, prefix) } } @@ -353,7 +368,7 @@ impl TrieBackendStorage for Arc> { impl TrieBackendStorage for PrefixedMemoryDB { type Overlay = PrefixedMemoryDB; - fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String> { + fn get(&self, key: &H::Out, prefix: Prefix) -> Result> { Ok(hash_db::HashDB::get(self, key, prefix)) } } @@ -361,7 +376,7 @@ impl TrieBackendStorage for PrefixedMemoryDB { impl TrieBackendStorage for MemoryDB { type Overlay = MemoryDB; - fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String> { + fn get(&self, key: &H::Out, prefix: Prefix) -> Result> { Ok(hash_db::HashDB::get(self, key, prefix)) } } diff --git a/primitives/state-machine/src/witness_backend.rs b/primitives/state-machine/src/witness_backend.rs new file mode 100644 index 0000000000000..ddb1aac585645 --- /dev/null +++ b/primitives/state-machine/src/witness_backend.rs @@ -0,0 +1,404 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Substrate backend runing on a trie proof, no_std compatible. + +use hash_db::Hasher; +use sp_externalities::{Externalities, ExtensionStore, Error, Extension}; +use sp_core::storage::{ChildInfo, TrackedStorageKey}; +use sp_std::{any::{TypeId, Any}}; +use sp_std::boxed::Box; +use sp_std::vec::Vec; +use sp_trie::MemoryDB; +use crate::trie_backend::TrieBackend; + + +// TODO replace with use crate::StorageValue; +pub type StorageValue = Vec; +// TODO replace with use crate::StorageKey; +pub type StorageKey = Vec; +/// The backend runnig on a trie proof. +pub struct WitnessBackend { +// overlay: OverlayedChangesNoChangeTrie, + trie: TrieBackend, H>, +} + +impl WitnessBackend + where + H: Hasher, + H::Out: Ord + 'static + codec::Codec, +{ + /// Create a new backend. + pub fn new(db: MemoryDB, root: H::Out) -> Self { + WitnessBackend { + trie: TrieBackend::new(db, root), + } + } +} + +impl Externalities for WitnessBackend + where + H: Hasher, + H::Out: Ord + 'static + codec::Codec, +{ + fn set_offchain_storage(&mut self, _key: &[u8], _value: Option<&[u8]>) { + // no ops + } + + fn storage(&self, key: &[u8]) -> Option { + unimplemented!() +/* let result = self.overlay.storage(key).map(|x| x.map(|x| x.to_vec())).unwrap_or_else(|| + self.backend.storage(key).expect(EXT_NOT_ALLOWED_TO_FAIL)); + trace!(target: "state", "{:04x}: Get {}={:?}", + self.id, + HexDisplay::from(&key), + result.as_ref().map(HexDisplay::from) + ); + result*/ + } + + fn storage_hash(&self, key: &[u8]) -> Option> { + unimplemented!() +/* let result = self.overlay + .storage(key) + .map(|x| x.map(|x| H::hash(x))) + .unwrap_or_else(|| self.backend.storage_hash(key).expect(EXT_NOT_ALLOWED_TO_FAIL)); + + result.map(|r| r.encode())*/ + } + + fn child_storage( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Option { + unimplemented!() +/* let result = self.overlay + .child_storage(child_info, key) + .map(|x| x.map(|x| x.to_vec())) + .unwrap_or_else(|| + self.backend.child_storage(child_info, key) + .expect(EXT_NOT_ALLOWED_TO_FAIL) + ); + + result +*/ + } + + fn child_storage_hash( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Option> { + unimplemented!() +/* let result = self.overlay + .child_storage(child_info, key) + .map(|x| x.map(|x| H::hash(x))) + .unwrap_or_else(|| + self.backend.child_storage_hash(child_info, key) + .expect(EXT_NOT_ALLOWED_TO_FAIL) + ); + result.map(|r| r.encode()) +*/ + } + + fn exists_storage(&self, key: &[u8]) -> bool { + unimplemented!() +/*let result = match self.overlay.storage(key) { + Some(x) => x.is_some(), + _ => self.backend.exists_storage(key).expect(EXT_NOT_ALLOWED_TO_FAIL), + }; + + result*/ + } + + fn exists_child_storage( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> bool { + unimplemented!() +/* + let result = match self.overlay.child_storage(child_info, key) { + Some(x) => x.is_some(), + _ => self.backend + .exists_child_storage(child_info, key) + .expect(EXT_NOT_ALLOWED_TO_FAIL), + }; + result*/ + } + + fn next_storage_key(&self, key: &[u8]) -> Option { + unimplemented!() + /* + let next_backend_key = self.backend.next_storage_key(key).expect(EXT_NOT_ALLOWED_TO_FAIL); + let next_overlay_key_change = self.overlay.next_storage_key_change(key); + + match (next_backend_key, next_overlay_key_change) { + (Some(backend_key), Some(overlay_key)) if &backend_key[..] < overlay_key.0 => Some(backend_key), + (backend_key, None) => backend_key, + (_, Some(overlay_key)) => if overlay_key.1.value().is_some() { + Some(overlay_key.0.to_vec()) + } else { + self.next_storage_key(&overlay_key.0[..]) + }, + } + */ + } + + fn next_child_storage_key( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Option { + unimplemented!() +/* let next_backend_key = self.backend + .next_child_storage_key(child_info, key) + .expect(EXT_NOT_ALLOWED_TO_FAIL); + let next_overlay_key_change = self.overlay.next_child_storage_key_change( + child_info.storage_key(), + key + ); + + match (next_backend_key, next_overlay_key_change) { + (Some(backend_key), Some(overlay_key)) if &backend_key[..] < overlay_key.0 => Some(backend_key), + (backend_key, None) => backend_key, + (_, Some(overlay_key)) => if overlay_key.1.value().is_some() { + Some(overlay_key.0.to_vec()) + } else { + self.next_child_storage_key( + child_info, + &overlay_key.0[..], + ) + }, + }*/ + } + + fn place_storage(&mut self, key: StorageKey, value: Option) { + unimplemented!() +/* if is_child_storage_key(&key) { + return; + } + + self.mark_dirty(); + self.overlay.set_storage(key, value);*/ + } + + fn place_child_storage( + &mut self, + child_info: &ChildInfo, + key: StorageKey, + value: Option, + ) { + unimplemented!() +/* + self.mark_dirty(); + self.overlay.set_child_storage(child_info, key, value); +*/ + } + + fn kill_child_storage( + &mut self, + child_info: &ChildInfo, + ) { + unimplemented!() + /* + self.mark_dirty(); + self.overlay.clear_child_storage(child_info); + self.backend.for_keys_in_child_storage(child_info, |key| { + self.overlay.set_child_storage(child_info, key.to_vec(), None); + }); + */ + } + + fn clear_prefix(&mut self, prefix: &[u8]) { + unimplemented!() +/* + if is_child_storage_key(prefix) { + return; + } + + self.mark_dirty(); + self.overlay.clear_prefix(prefix); + self.backend.for_keys_with_prefix(prefix, |key| { + self.overlay.set_storage(key.to_vec(), None); + }); +*/ + } + + fn clear_child_prefix( + &mut self, + child_info: &ChildInfo, + prefix: &[u8], + ) { + unimplemented!() +/* + self.mark_dirty(); + self.overlay.clear_child_prefix(child_info, prefix); + self.backend.for_child_keys_with_prefix(child_info, prefix, |key| { + self.overlay.set_child_storage(child_info, key.to_vec(), None); + }); +*/ + } + + fn storage_append( + &mut self, + key: Vec, + value: Vec, + ) { + unimplemented!() +/* + self.mark_dirty(); + + let backend = &mut self.backend; + let current_value = self.overlay.value_mut_or_insert_with( + &key, + || backend.storage(&key).expect(EXT_NOT_ALLOWED_TO_FAIL).unwrap_or_default() + ); + StorageAppend::new(current_value).append(value); +*/ + } + + fn chain_id(&self) -> u64 { + 42 + } + + fn storage_root(&mut self) -> Vec { + unimplemented!() + /*if let Some(ref root) = self.storage_transaction_cache.transaction_storage_root { + return root.encode(); + } + + let root = self.overlay.storage_root(self.backend, self.storage_transaction_cache); + root.encode() + */ + } + + fn child_storage_root( + &mut self, + child_info: &ChildInfo, + ) -> Vec { + unimplemented!() + /* + let storage_key = child_info.storage_key(); + let prefixed_storage_key = child_info.prefixed_storage_key(); + if self.storage_transaction_cache.transaction_storage_root.is_some() { + let root = self + .storage(prefixed_storage_key.as_slice()) + .and_then(|k| Decode::decode(&mut &k[..]).ok()) + .unwrap_or_else( + || empty_child_trie_root::>() + ); + root.encode() + } else { + let root = if let Some((changes, info)) = self.overlay.child_changes(storage_key) { + let delta = changes.map(|(k, v)| (k.as_ref(), v.value().map(AsRef::as_ref))); + Some(self.backend.child_storage_root(info, delta)) + } else { + None + }; + + if let Some((root, is_empty, _)) = root { + let root = root.encode(); + if is_empty { + self.overlay.set_storage(prefixed_storage_key.into_inner(), None); + } else { + self.overlay.set_storage(prefixed_storage_key.into_inner(), Some(root.clone())); + } + root + } else { + let root = self + .storage(prefixed_storage_key.as_slice()) + .and_then(|k| Decode::decode(&mut &k[..]).ok()) + .unwrap_or_else( + || empty_child_trie_root::>() + ); + root.encode() + } + } + */ + } + + fn storage_changes_root(&mut self, parent_hash: &[u8]) -> Result>, ()> { + unimplemented!() + } + + fn storage_start_transaction(&mut self) { + unimplemented!() + //self.overlay.start_transaction() + } + + fn storage_rollback_transaction(&mut self) -> Result<(), ()> { + unimplemented!() + //self.mark_dirty(); + //self.overlay.rollback_transaction().map_err(|_| ()) + } + + fn storage_commit_transaction(&mut self) -> Result<(), ()> { + unimplemented!() + //self.overlay.commit_transaction().map_err(|_| ()) + } + + fn wipe(&mut self) { + unimplemented!() + } + + fn commit(&mut self) { + unimplemented!() + } + + fn read_write_count(&self) -> (u32, u32, u32, u32) { + unimplemented!() + } + + fn reset_read_write_count(&mut self) { + unimplemented!() + } + + fn get_whitelist(&self) -> Vec { + unimplemented!() + } + + fn set_whitelist(&mut self, new: Vec) { + unimplemented!() + } +} + +impl ExtensionStore for WitnessBackend +where + H: Hasher, + H::Out: Ord + 'static + codec::Codec, +{ + fn extension_by_type_id(&mut self, _type_id: TypeId) -> Option<&mut dyn Any> { + None + } + + fn register_extension_with_type_id( + &mut self, + _type_id: TypeId, + _extension: Box, + ) -> Result<(), Error> { + Err(Error::ExtensionsAreNotSupported) + } + + fn deregister_extension_by_type_id( + &mut self, + _type_id: TypeId, + ) -> Result<(), Error> { + Err(Error::ExtensionsAreNotSupported) + } +} diff --git a/test-utils/runtime/Cargo.toml b/test-utils/runtime/Cargo.toml index 08e1b955ab43c..dbbac8652ad84 100644 --- a/test-utils/runtime/Cargo.toml +++ b/test-utils/runtime/Cargo.toml @@ -42,6 +42,7 @@ sp-transaction-pool = { version = "2.0.0-rc5", default-features = false, path = trie-db = { version = "0.22.0", default-features = false } parity-util-mem = { version = "0.7.0", default-features = false, features = ["primitive-types"] } sc-service = { version = "0.8.0-rc5", default-features = false, optional = true, features = ["test-helpers"], path = "../../client/service" } +sp-state-machine = { version = "0.8.0-rc5", default-features = false, path = "../../primitives/state-machine" } # 3rd party cfg-if = "0.1.10" @@ -52,7 +53,6 @@ serde = { version = "1.0.101", optional = true, features = ["derive"] } sc-block-builder = { version = "0.8.0-rc5", path = "../../client/block-builder" } sc-executor = { version = "0.8.0-rc5", path = "../../client/executor" } substrate-test-runtime-client = { version = "2.0.0-rc5", path = "./client" } -sp-state-machine = { version = "0.8.0-rc5", path = "../../primitives/state-machine" } [build-dependencies] wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-runner", path = "../../utils/wasm-builder-runner" } @@ -84,6 +84,7 @@ std = [ "sp-session/std", "sp-api/std", "sp-runtime/std", + "sp-state-machine/std", "pallet-babe/std", "frame-system-rpc-runtime-api/std", "frame-system/std", diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index fedbff5a109ba..ef7f71147428f 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -145,6 +145,7 @@ pub enum Extrinsic { IncludeData(Vec), StorageChange(Vec, Option>), ChangesTrieConfigUpdate(Option), + WitnessBackend, } parity_util_mem::malloc_size_of_is_0!(Extrinsic); // non-opaque extrinsic does not need this @@ -173,6 +174,8 @@ impl BlindCheckable for Extrinsic { Extrinsic::StorageChange(key, value) => Ok(Extrinsic::StorageChange(key, value)), Extrinsic::ChangesTrieConfigUpdate(new_config) => Ok(Extrinsic::ChangesTrieConfigUpdate(new_config)), + Extrinsic::WitnessBackend => + Ok(Extrinsic::WitnessBackend), } } } diff --git a/test-utils/runtime/src/system.rs b/test-utils/runtime/src/system.rs index 818487a89e518..713e1df423545 100644 --- a/test-utils/runtime/src/system.rs +++ b/test-utils/runtime/src/system.rs @@ -261,6 +261,8 @@ fn execute_transaction_backend(utx: &Extrinsic, extrinsic_index: u32) -> ApplyEx execute_storage_change(key, value.as_ref().map(|v| &**v)), Extrinsic::ChangesTrieConfigUpdate(ref new_config) => execute_changes_trie_config_update(new_config.clone()), + Extrinsic::WitnessBackend => + execute_witness_backend(), } } @@ -303,6 +305,14 @@ fn execute_storage_change(key: &[u8], value: Option<&[u8]>) -> ApplyExtrinsicRes Ok(Ok(())) } +fn execute_witness_backend() -> ApplyExtrinsicResult { + use sp_state_machine::witness_backend::WitnessBackend; + let mut backend = WitnessBackend::::new(); + unimplemented!("make use of witness backend for no std"); + Ok(Ok(())) +} + + fn execute_changes_trie_config_update(new_config: Option) -> ApplyExtrinsicResult { match new_config.clone() { Some(new_config) => storage::unhashed::put_raw( From c0e25aebc9e8bf62c6e8d298153ec3e26666fad9 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 21 Aug 2020 15:51:36 +0200 Subject: [PATCH 02/21] before trie backend without tx --- Cargo.lock | 1 + .../state-machine/src/changes_trie/build.rs | 6 +- .../state-machine/src/changes_trie/mod.rs | 3 - primitives/state-machine/src/ext.rs | 490 +++++++++++++----- primitives/state-machine/src/lib.rs | 53 +- .../src/overlayed_changes/changeset.rs | 12 +- .../src/overlayed_changes/mod.rs | 76 ++- primitives/state-machine/src/stats.rs | 1 - primitives/state-machine/src/trie_backend.rs | 6 + .../state-machine/src/trie_backend_essence.rs | 4 +- .../state-machine/src/witness_backend.rs | 404 --------------- primitives/state-machine/src/witness_ext.rs | 252 +++++++++ test-utils/runtime/Cargo.toml | 2 + test-utils/runtime/src/lib.rs | 6 +- test-utils/runtime/src/system.rs | 14 +- 15 files changed, 746 insertions(+), 584 deletions(-) delete mode 100644 primitives/state-machine/src/witness_backend.rs create mode 100644 primitives/state-machine/src/witness_ext.rs diff --git a/Cargo.lock b/Cargo.lock index 1796a4ebcbf22..4c98590696ad7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8558,6 +8558,7 @@ dependencies = [ "sp-consensus-aura", "sp-consensus-babe", "sp-core", + "sp-externalities", "sp-finality-grandpa", "sp-inherents", "sp-io", diff --git a/primitives/state-machine/src/changes_trie/build.rs b/primitives/state-machine/src/changes_trie/build.rs index 80c3048053ecd..ff9a03090e117 100644 --- a/primitives/state-machine/src/changes_trie/build.rs +++ b/primitives/state-machine/src/changes_trie/build.rs @@ -25,7 +25,7 @@ use num_traits::One; use crate::{ StorageKey, backend::Backend, - overlayed_changes::{OverlayedChanges, OverlayedValue as OverlayedValueInner, ChangeTrieOverlay}, + overlayed_changes::{OverlayedChanges, OverlayedValue, ChangeTrieOverlay}, trie_backend_essence::TrieBackendEssence, changes_trie::{ AnchorBlockId, ConfigurationRange, Storage, BlockNumber, @@ -35,8 +35,6 @@ use crate::{ }; use sp_core::storage::{ChildInfo, PrefixedStorageKey}; -type OverlayedValue = OverlayedValueInner; - /// Prepare input pairs for building a changes trie of given block. /// /// Returns Err if storage error has occurred OR if storage haven't returned @@ -136,7 +134,7 @@ fn prepare_extrinsics_input_inner<'a, B, H, Number, CT>( block: &Number, overlay: &'a OverlayedChanges, child_info: Option, - mut changes: impl Iterator)> + mut changes: impl Iterator)> ) -> Result> + 'a, String> where B: Backend, diff --git a/primitives/state-machine/src/changes_trie/mod.rs b/primitives/state-machine/src/changes_trie/mod.rs index 4452cbbb727b0..a5e4321f5de9c 100644 --- a/primitives/state-machine/src/changes_trie/mod.rs +++ b/primitives/state-machine/src/changes_trie/mod.rs @@ -85,9 +85,6 @@ use crate::{ }, }; -/// Changes that are made outside of extrinsics are marked with this index; -pub const NO_EXTRINSIC_INDEX: u32 = 0xffffffff; - /// Requirements for block number that can be used with changes tries. pub trait BlockNumber: Send + Sync + 'static + diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index e57636b300a5f..ffe4b528015b0 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -18,23 +18,29 @@ //! Concrete externalities implementation. use crate::{ - StorageKey, StorageValue, OverlayedChanges, StorageTransactionCache, + StorageKey, StorageValue, OverlayedChanges, + overlayed_changes::{ChangeTrieOverlay, Extrinsics}, backend::Backend, - changes_trie::State as ChangesTrieState, }; - use hash_db::Hasher; use sp_core::{ - offchain::storage::OffchainOverlayedChanges, storage::{well_known_keys::is_child_storage_key, ChildInfo, TrackedStorageKey}, - traits::Externalities, hexdisplay::HexDisplay, + hexdisplay::HexDisplay, }; use sp_trie::{trie_types::Layout, empty_child_trie_root}; -use sp_externalities::{Extensions, Extension}; +use sp_externalities::{Externalities, Extensions, Extension}; use codec::{Decode, Encode, EncodeAppend}; -use std::{error, fmt, any::{Any, TypeId}}; -use log::{warn, trace}; +use sp_std::{fmt, any::{Any, TypeId}, vec::Vec, vec, boxed::Box}; +use crate::{warn, trace, log_error}; +#[cfg(feature = "std")] +use sp_core::offchain::storage::OffchainOverlayedChanges; +#[cfg(feature = "std")] +use crate::changes_trie::State as ChangesTrieState; +#[cfg(feature = "std")] +use crate:: StorageTransactionCache; +#[cfg(feature = "std")] +use std::error; const EXT_NOT_ALLOWED_TO_FAIL: &str = "Externalities not allowed to fail within runtime"; const BENCHMARKING_FN: &str = "\ @@ -42,7 +48,19 @@ const BENCHMARKING_FN: &str = "\ For that reason client started transactions before calling into runtime are not allowed. Without client transactions the loop condition garantuees the success of the tx close."; + +#[cfg(feature = "std")] +fn guard() -> sp_panic_handler::AbortGuard { + sp_panic_handler::AbortGuard::force_abort() +} + +#[cfg(not(feature = "std"))] +fn guard() -> () { + () +} + /// Errors that can occur when interacting with the externalities. +#[cfg(feature = "std")] #[derive(Debug, Copy, Clone)] pub enum Error { /// Failure to load state data from the backend. @@ -53,6 +71,7 @@ pub enum Error { Executor(E), } +#[cfg(feature = "std")] impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match *self { @@ -62,6 +81,7 @@ impl fmt::Display for Error { } } +#[cfg(feature = "std")] impl error::Error for Error { fn description(&self) -> &str { match *self { @@ -72,30 +92,61 @@ impl error::Error for Error { } /// Wraps a read-only backend, call executor, and current overlayed changes. +#[cfg(feature = "std")] pub struct Ext<'a, H, N, B> where H: Hasher, B: 'a + Backend, N: crate::changes_trie::BlockNumber, { - /// The overlayed changes to write to. - overlay: &'a mut OverlayedChanges, + /// Inner Ext (change overlay and backend). + inner: ExtInnerMut<'a, H, B>, /// The overlayed changes destined for the Offchain DB. offchain_overlay: &'a mut OffchainOverlayedChanges, - /// The storage backend to read from. - backend: &'a B, /// The cache for the storage transactions. storage_transaction_cache: &'a mut StorageTransactionCache, /// Changes trie state to read from. changes_trie_state: Option>, - /// Pseudo-unique id used for tracing. - pub id: u16, /// Dummy usage of N arg. _phantom: std::marker::PhantomData, /// Extensions registered with this instance. extensions: Option<&'a mut Extensions>, } +/// Basis implementation for `Externalities` trait. +pub struct ExtInnerMut<'a, H, B, CT: ChangeTrieOverlay = Extrinsics> + where + H: Hasher, + B: Backend, +{ + /// The overlayed changes to write to. + pub overlay: &'a mut OverlayedChanges, + /// The storage backend to read from. + pub backend: &'a B, + /// Pseudo-unique id used for tracing. + pub id: u16, + /// Dummy usage of H arg. + pub _phantom: sp_std::marker::PhantomData, +} + +/// Basis implementation for `Externalities` trait. +pub struct ExtInner<'a, H, B, CT: ChangeTrieOverlay = Extrinsics> + where + H: Hasher, + B: Backend, +{ + /// The overlayed changes to write to. + pub overlay: &'a OverlayedChanges, + /// The storage backend to read from. + pub backend: &'a B, + /// Pseudo-unique id used for tracing. + pub id: u16, + /// Dummy usage of H arg. + pub _phantom: sp_std::marker::PhantomData, +} + + +#[cfg(feature = "std")] impl<'a, H, N, B> Ext<'a, H, N, B> where H: Hasher, @@ -103,7 +154,6 @@ where B: 'a + Backend, N: crate::changes_trie::BlockNumber, { - /// Create a new `Ext` from overlayed changes and read-only backend pub fn new( overlay: &'a mut OverlayedChanges, @@ -114,12 +164,15 @@ where extensions: Option<&'a mut Extensions>, ) -> Self { Self { - overlay, + inner: ExtInnerMut { + overlay, + backend, + id: rand::random(), + _phantom: Default::default(), + }, offchain_overlay, - backend, changes_trie_state, storage_transaction_cache, - id: rand::random(), _phantom: Default::default(), extensions, } @@ -136,6 +189,11 @@ where pub fn get_offchain_storage_changes(&self) -> &OffchainOverlayedChanges { &*self.offchain_overlay } + + /// Access internal random id. + pub fn id(&self) -> u16 { + self.inner.id + } } #[cfg(test)] @@ -149,9 +207,9 @@ where pub fn storage_pairs(&self) -> Vec<(StorageKey, StorageValue)> { use std::collections::HashMap; - self.backend.pairs().iter() + self.inner.backend.pairs().iter() .map(|&(ref k, ref v)| (k.to_vec(), Some(v.to_vec()))) - .chain(self.overlay.changes().map(|(k, v)| (k.clone(), v.value().cloned()))) + .chain(self.inner.overlay.changes().map(|(k, v)| (k.clone(), v.value().cloned()))) .collect::>() .into_iter() .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) @@ -159,24 +217,15 @@ where } } -impl<'a, H, B, N> Externalities for Ext<'a, H, N, B> +impl<'a, H, B, CT> ExtInner<'a, H, B, CT> where H: Hasher, H::Out: Ord + 'static + codec::Codec, - B: 'a + Backend, - N: crate::changes_trie::BlockNumber, + B: Backend, + CT: ChangeTrieOverlay, { - - fn set_offchain_storage(&mut self, key: &[u8], value: Option<&[u8]>) { - use ::sp_core::offchain::STORAGE_PREFIX; - match value { - Some(value) => self.offchain_overlay.set(STORAGE_PREFIX, key, value), - None => self.offchain_overlay.remove(STORAGE_PREFIX, key), - } - } - - fn storage(&self, key: &[u8]) -> Option { - let _guard = sp_panic_handler::AbortGuard::force_abort(); + pub(crate) fn storage(&self, key: &[u8]) -> Option { + let _guard = guard(); let result = self.overlay.storage(key).map(|x| x.map(|x| x.to_vec())).unwrap_or_else(|| self.backend.storage(key).expect(EXT_NOT_ALLOWED_TO_FAIL)); trace!(target: "state", "{:04x}: Get {}={:?}", @@ -187,8 +236,8 @@ where result } - fn storage_hash(&self, key: &[u8]) -> Option> { - let _guard = sp_panic_handler::AbortGuard::force_abort(); + pub(crate) fn storage_hash(&self, key: &[u8]) -> Option> { + let _guard = guard(); let result = self.overlay .storage(key) .map(|x| x.map(|x| H::hash(x))) @@ -202,12 +251,12 @@ where result.map(|r| r.encode()) } - fn child_storage( + pub(crate) fn child_storage( &self, child_info: &ChildInfo, key: &[u8], ) -> Option { - let _guard = sp_panic_handler::AbortGuard::force_abort(); + let _guard = guard(); let result = self.overlay .child_storage(child_info, key) .map(|x| x.map(|x| x.to_vec())) @@ -226,12 +275,12 @@ where result } - fn child_storage_hash( + pub(crate) fn child_storage_hash( &self, child_info: &ChildInfo, key: &[u8], ) -> Option> { - let _guard = sp_panic_handler::AbortGuard::force_abort(); + let _guard = guard(); let result = self.overlay .child_storage(child_info, key) .map(|x| x.map(|x| H::hash(x))) @@ -250,8 +299,8 @@ where result.map(|r| r.encode()) } - fn exists_storage(&self, key: &[u8]) -> bool { - let _guard = sp_panic_handler::AbortGuard::force_abort(); + pub(crate) fn exists_storage(&self, key: &[u8]) -> bool { + let _guard = guard(); let result = match self.overlay.storage(key) { Some(x) => x.is_some(), _ => self.backend.exists_storage(key).expect(EXT_NOT_ALLOWED_TO_FAIL), @@ -266,12 +315,12 @@ where result } - fn exists_child_storage( + pub(crate) fn exists_child_storage( &self, child_info: &ChildInfo, key: &[u8], ) -> bool { - let _guard = sp_panic_handler::AbortGuard::force_abort(); + let _guard = guard(); let result = match self.overlay.child_storage(child_info, key) { Some(x) => x.is_some(), @@ -289,7 +338,7 @@ where result } - fn next_storage_key(&self, key: &[u8]) -> Option { + pub(crate) fn next_storage_key(&self, key: &[u8]) -> Option { let next_backend_key = self.backend.next_storage_key(key).expect(EXT_NOT_ALLOWED_TO_FAIL); let next_overlay_key_change = self.overlay.next_storage_key_change(key); @@ -304,7 +353,7 @@ where } } - fn next_child_storage_key( + pub(crate) fn next_child_storage_key( &self, child_info: &ChildInfo, key: &[u8], @@ -331,23 +380,43 @@ where } } - fn place_storage(&mut self, key: StorageKey, value: Option) { + pub(crate) fn read_write_count(&self) -> (u32, u32, u32, u32) { + self.backend.read_write_count() + } +} + +impl<'a, H, B, CT> ExtInnerMut<'a, H, B, CT> +where + H: Hasher, + H::Out: Ord + 'static + codec::Codec, + B: Backend, + CT: ChangeTrieOverlay, +{ + pub(crate) fn inner(&'a self) -> ExtInner<'a, H, B, CT> { + ExtInner { + overlay: self.overlay, + backend: self.backend, + id: self.id, + _phantom: Default::default(), + } + } + + pub(crate) fn place_storage(&mut self, key: StorageKey, value: Option) { trace!(target: "state", "{:04x}: Put {}={:?}", self.id, HexDisplay::from(&key), value.as_ref().map(HexDisplay::from) ); - let _guard = sp_panic_handler::AbortGuard::force_abort(); + let _guard = guard(); if is_child_storage_key(&key) { warn!(target: "trie", "Refuse to directly set child storage key"); return; } - self.mark_dirty(); self.overlay.set_storage(key, value); } - fn place_child_storage( + pub(crate) fn place_child_storage( &mut self, child_info: &ChildInfo, key: StorageKey, @@ -359,13 +428,12 @@ where HexDisplay::from(&key), value.as_ref().map(HexDisplay::from) ); - let _guard = sp_panic_handler::AbortGuard::force_abort(); + let _guard = guard(); - self.mark_dirty(); self.overlay.set_child_storage(child_info, key, value); } - fn kill_child_storage( + pub(crate) fn kill_child_storage( &mut self, child_info: &ChildInfo, ) { @@ -373,34 +441,32 @@ where self.id, HexDisplay::from(&child_info.storage_key()), ); - let _guard = sp_panic_handler::AbortGuard::force_abort(); + let _guard = guard(); - self.mark_dirty(); self.overlay.clear_child_storage(child_info); self.backend.for_keys_in_child_storage(child_info, |key| { self.overlay.set_child_storage(child_info, key.to_vec(), None); }); } - fn clear_prefix(&mut self, prefix: &[u8]) { + pub(crate) fn clear_prefix(&mut self, prefix: &[u8]) { trace!(target: "state", "{:04x}: ClearPrefix {}", self.id, HexDisplay::from(&prefix), ); - let _guard = sp_panic_handler::AbortGuard::force_abort(); + let _guard = guard(); if is_child_storage_key(prefix) { warn!(target: "trie", "Refuse to directly clear prefix that is part of child storage key"); return; } - self.mark_dirty(); self.overlay.clear_prefix(prefix); self.backend.for_keys_with_prefix(prefix, |key| { self.overlay.set_storage(key.to_vec(), None); }); } - fn clear_child_prefix( + pub(crate) fn clear_child_prefix( &mut self, child_info: &ChildInfo, prefix: &[u8], @@ -410,16 +476,15 @@ where HexDisplay::from(&child_info.storage_key()), HexDisplay::from(&prefix), ); - let _guard = sp_panic_handler::AbortGuard::force_abort(); + let _guard = guard(); - self.mark_dirty(); self.overlay.clear_child_prefix(child_info, prefix); self.backend.for_child_keys_with_prefix(child_info, prefix, |key| { self.overlay.set_child_storage(child_info, key.to_vec(), None); }); } - fn storage_append( + pub(crate) fn storage_append( &mut self, key: Vec, value: Vec, @@ -430,8 +495,7 @@ where HexDisplay::from(&value), ); - let _guard = sp_panic_handler::AbortGuard::force_abort(); - self.mark_dirty(); + let _guard = guard(); let backend = &mut self.backend; let current_value = self.overlay.value_mut_or_insert_with( @@ -441,22 +505,214 @@ where StorageAppend::new(current_value).append(value); } + pub(crate) fn storage_root(&mut self) -> Vec { + let _guard = guard(); + let root = self.overlay.storage_root_no_cache(self.backend); + trace!(target: "state", "{:04x}: Root {}", self.id, HexDisplay::from(&root.as_ref())); + root.encode() + } + + pub(crate) fn child_storage_root( + &mut self, + child_info: &ChildInfo, + ) -> Vec { + let _guard = guard(); + let storage_key = child_info.storage_key(); + let prefixed_storage_key = child_info.prefixed_storage_key(); + + let root = if let Some((changes, info)) = self.overlay.child_changes(storage_key) { + let delta = changes.map(|(k, v)| (k.as_ref(), v.value().map(AsRef::as_ref))); + Some(self.backend.child_storage_root(info, delta)) + } else { + None + }; + + if let Some((root, is_empty, _)) = root { + let root = root.encode(); + // We store update in the overlay in order to be able to use 'self.storage_transaction' + // cache. This is brittle as it rely on Ext only querying the trie backend for + // storage root. + // A better design would be to manage 'child_storage_transaction' in a + // similar way as 'storage_transaction' but for each child trie. + if is_empty { + self.overlay.set_storage(prefixed_storage_key.into_inner(), None); + } else { + self.overlay.set_storage(prefixed_storage_key.into_inner(), Some(root.clone())); + } + + trace!(target: "state", "{:04x}: ChildRoot({}) {}", + self.id, + HexDisplay::from(&storage_key.as_ref()), + HexDisplay::from(&root.as_ref()), + ); + root + } else { + // empty overlay + let root = self.inner() + .storage(prefixed_storage_key.as_slice()) + .and_then(|k| Decode::decode(&mut &k[..]).ok()) + .unwrap_or_else( + || empty_child_trie_root::>() + ); + trace!(target: "state", "{:04x}: ChildRoot({})(no_change) {}", + self.id, + HexDisplay::from(&storage_key.as_ref()), + HexDisplay::from(&root.as_ref()), + ); + root.encode() + } + } + + pub(crate) fn storage_start_transaction(&mut self) { + self.overlay.start_transaction() + } + + pub(crate) fn storage_rollback_transaction(&mut self) -> Result<(), ()> { + self.overlay.rollback_transaction().map_err(|_| ()) + } + + pub(crate) fn storage_commit_transaction(&mut self) -> Result<(), ()> { + self.overlay.commit_transaction().map_err(|_| ()) + } + + pub(crate) fn reset_read_write_count(&mut self) { + self.backend.reset_read_write_count() + } +} + +#[cfg(feature = "std")] +impl<'a, H, B, N> Externalities for Ext<'a, H, N, B> +where + H: Hasher, + H::Out: Ord + 'static + codec::Codec, + B: 'a + Backend, + N: crate::changes_trie::BlockNumber, +{ + + fn set_offchain_storage(&mut self, key: &[u8], value: Option<&[u8]>) { + use ::sp_core::offchain::STORAGE_PREFIX; + match value { + Some(value) => self.offchain_overlay.set(STORAGE_PREFIX, key, value), + None => self.offchain_overlay.remove(STORAGE_PREFIX, key), + } + } + + fn storage(&self, key: &[u8]) -> Option { + self.inner.inner().storage(key) + } + + fn storage_hash(&self, key: &[u8]) -> Option> { + self.inner.inner().storage_hash(key) + } + + fn child_storage( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Option { + self.inner.inner().child_storage(child_info, key) + } + + fn child_storage_hash( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Option> { + self.inner.inner().child_storage_hash(child_info, key) + } + + fn exists_storage(&self, key: &[u8]) -> bool { + self.inner.inner().exists_storage(key) + } + + fn exists_child_storage( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> bool { + self.inner.inner().exists_child_storage(child_info, key) + } + + fn next_storage_key(&self, key: &[u8]) -> Option { + self.inner.inner().next_storage_key(key) + } + + fn next_child_storage_key( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Option { + self.inner.inner().next_child_storage_key(child_info, key) + } + + fn place_storage(&mut self, key: StorageKey, value: Option) { + let result = self.inner.place_storage(key, value); + self.mark_dirty(); + result + } + + fn place_child_storage( + &mut self, + child_info: &ChildInfo, + key: StorageKey, + value: Option, + ) { + let result = self.inner.place_child_storage(child_info, key, value); + self.mark_dirty(); + result + } + + fn kill_child_storage( + &mut self, + child_info: &ChildInfo, + ) { + let result = self.inner.kill_child_storage(child_info); + self.mark_dirty(); + result + } + + fn clear_prefix(&mut self, prefix: &[u8]) { + let result = self.inner.clear_prefix(prefix); + self.mark_dirty(); + result + } + + fn clear_child_prefix( + &mut self, + child_info: &ChildInfo, + prefix: &[u8], + ) { + let result = self.inner.clear_child_prefix(child_info, prefix); + self.mark_dirty(); + result + } + + fn storage_append( + &mut self, + key: Vec, + value: Vec, + ) { + let result = self.inner.storage_append(key, value); + self.mark_dirty(); + result + } + fn chain_id(&self) -> u64 { 42 } fn storage_root(&mut self) -> Vec { - let _guard = sp_panic_handler::AbortGuard::force_abort(); + let _guard = guard(); if let Some(ref root) = self.storage_transaction_cache.transaction_storage_root { trace!(target: "state", "{:04x}: Root(cached) {}", - self.id, + self.inner.id, HexDisplay::from(&root.as_ref()), ); return root.encode(); } - let root = self.overlay.storage_root(self.backend, self.storage_transaction_cache); - trace!(target: "state", "{:04x}: Root {}", self.id, HexDisplay::from(&root.as_ref())); + let root = self.inner.overlay.storage_root(self.inner.backend, self.storage_transaction_cache); + trace!(target: "state", "{:04x}: Root {}", self.inner.id, HexDisplay::from(&root.as_ref())); root.encode() } @@ -464,7 +720,7 @@ where &mut self, child_info: &ChildInfo, ) -> Vec { - let _guard = sp_panic_handler::AbortGuard::force_abort(); + let _guard = guard(); let storage_key = child_info.storage_key(); let prefixed_storage_key = child_info.prefixed_storage_key(); if self.storage_transaction_cache.transaction_storage_root.is_some() { @@ -475,60 +731,20 @@ where || empty_child_trie_root::>() ); trace!(target: "state", "{:04x}: ChildRoot({})(cached) {}", - self.id, + self.inner.id, HexDisplay::from(&storage_key), HexDisplay::from(&root.as_ref()), ); root.encode() } else { - let root = if let Some((changes, info)) = self.overlay.child_changes(storage_key) { - let delta = changes.map(|(k, v)| (k.as_ref(), v.value().map(AsRef::as_ref))); - Some(self.backend.child_storage_root(info, delta)) - } else { - None - }; - - if let Some((root, is_empty, _)) = root { - let root = root.encode(); - // We store update in the overlay in order to be able to use 'self.storage_transaction' - // cache. This is brittle as it rely on Ext only querying the trie backend for - // storage root. - // A better design would be to manage 'child_storage_transaction' in a - // similar way as 'storage_transaction' but for each child trie. - if is_empty { - self.overlay.set_storage(prefixed_storage_key.into_inner(), None); - } else { - self.overlay.set_storage(prefixed_storage_key.into_inner(), Some(root.clone())); - } - - trace!(target: "state", "{:04x}: ChildRoot({}) {}", - self.id, - HexDisplay::from(&storage_key.as_ref()), - HexDisplay::from(&root.as_ref()), - ); - root - } else { - // empty overlay - let root = self - .storage(prefixed_storage_key.as_slice()) - .and_then(|k| Decode::decode(&mut &k[..]).ok()) - .unwrap_or_else( - || empty_child_trie_root::>() - ); - trace!(target: "state", "{:04x}: ChildRoot({})(no_change) {}", - self.id, - HexDisplay::from(&storage_key.as_ref()), - HexDisplay::from(&root.as_ref()), - ); - root.encode() - } + self.inner.child_storage_root(child_info) } } fn storage_changes_root(&mut self, parent_hash: &[u8]) -> Result>, ()> { - let _guard = sp_panic_handler::AbortGuard::force_abort(); - let root = self.overlay.changes_trie_root( - self.backend, + let _guard = guard(); + let root = self.inner.overlay.changes_trie_root( + self.inner.backend, self.changes_trie_state.as_ref(), Decode::decode(&mut &parent_hash[..]).map_err(|e| trace!( @@ -542,7 +758,7 @@ where ); trace!(target: "state", "{:04x}: ChangesRoot({}) {:?}", - self.id, + self.inner.id, HexDisplay::from(&parent_hash), root, ); @@ -551,74 +767,73 @@ where } fn storage_start_transaction(&mut self) { - self.overlay.start_transaction() + self.inner.storage_start_transaction() } fn storage_rollback_transaction(&mut self) -> Result<(), ()> { self.mark_dirty(); - self.overlay.rollback_transaction().map_err(|_| ()) + self.inner.storage_rollback_transaction() } fn storage_commit_transaction(&mut self) -> Result<(), ()> { - self.overlay.commit_transaction().map_err(|_| ()) + self.inner.storage_commit_transaction() } fn wipe(&mut self) { - for _ in 0..self.overlay.transaction_depth() { - self.overlay.rollback_transaction().expect(BENCHMARKING_FN); + for _ in 0..self.inner.overlay.transaction_depth() { + self.inner.overlay.rollback_transaction().expect(BENCHMARKING_FN); } - self.overlay.drain_storage_changes( - &self.backend, + self.inner.overlay.drain_storage_changes( + &self.inner.backend, None, Default::default(), self.storage_transaction_cache, ).expect(EXT_NOT_ALLOWED_TO_FAIL); - self.backend.wipe().expect(EXT_NOT_ALLOWED_TO_FAIL); + self.inner.backend.wipe().expect(EXT_NOT_ALLOWED_TO_FAIL); self.mark_dirty(); - self.overlay + self.inner.overlay .enter_runtime() .expect("We have reset the overlay above, so we can not be in the runtime; qed"); } fn commit(&mut self) { - for _ in 0..self.overlay.transaction_depth() { - self.overlay.commit_transaction().expect(BENCHMARKING_FN); + for _ in 0..self.inner.overlay.transaction_depth() { + self.inner.overlay.commit_transaction().expect(BENCHMARKING_FN); } - let changes = self.overlay.drain_storage_changes( - &self.backend, + let changes = self.inner.overlay.drain_storage_changes( + &self.inner.backend, None, Default::default(), self.storage_transaction_cache, ).expect(EXT_NOT_ALLOWED_TO_FAIL); - self.backend.commit( + self.inner.backend.commit( changes.transaction_storage_root, changes.transaction, changes.main_storage_changes, ).expect(EXT_NOT_ALLOWED_TO_FAIL); self.mark_dirty(); - self.overlay + self.inner.overlay .enter_runtime() .expect("We have reset the overlay above, so we can not be in the runtime; qed"); } fn read_write_count(&self) -> (u32, u32, u32, u32) { - self.backend.read_write_count() + self.inner.backend.read_write_count() } fn reset_read_write_count(&mut self) { - self.backend.reset_read_write_count() + self.inner.backend.reset_read_write_count() } fn get_whitelist(&self) -> Vec { - self.backend.get_whitelist() + self.inner.backend.get_whitelist() } fn set_whitelist(&mut self, new: Vec) { - self.backend.set_whitelist(new) + self.inner.backend.set_whitelist(new) } } - /// Implement `Encode` by forwarding the stored raw vec. struct EncodeOpaqueValue(Vec); @@ -643,12 +858,12 @@ impl<'a> StorageAppend<'a> { pub fn append(&mut self, value: Vec) { let value = vec![EncodeOpaqueValue(value)]; - let item = std::mem::take(self.0); + let item = sp_std::mem::take(self.0); *self.0 = match Vec::::append_or_new(item, &value) { Ok(item) => item, Err(_) => { - log::error!( + log_error!( target: "runtime", "Failed to append value, resetting storage item to `[value]`.", ); @@ -658,6 +873,7 @@ impl<'a> StorageAppend<'a> { } } +#[cfg(feature = "std")] impl<'a, H, B, N> sp_externalities::ExtensionStore for Ext<'a, H, N, B> where H: Hasher, diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index c66aa3d7fd948..15027bde11489 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -21,13 +21,12 @@ #![cfg_attr(not(feature = "std"), no_std)] pub mod backend; -pub mod witness_backend; +pub mod witness_ext; #[cfg(feature = "std")] mod in_memory_backend; #[cfg(feature = "std")] mod changes_trie; mod error; -#[cfg(feature = "std")] mod ext; #[cfg(feature = "std")] mod testing; @@ -50,10 +49,12 @@ pub use execution::*; #[cfg(feature = "std")] -pub use log::{debug, warn}; +pub use log::{debug, warn, trace, error as log_error}; +/// In no_std we skip logs for state_machine, this macro +/// is a noops. #[cfg(not(feature = "std"))] -#[macro_export] // TODO try remove export +#[macro_export] macro_rules! warn { (target: $target:expr, $($arg:tt)+) => ( () @@ -62,6 +63,9 @@ macro_rules! warn { () ); } + +/// In no_std we skip logs for state_machine, this macro +/// is a noops. #[cfg(not(feature = "std"))] #[macro_export] macro_rules! debug { @@ -73,16 +77,49 @@ macro_rules! debug { ); } +/// In no_std we skip logs for state_machine, this macro +/// is a noops. +#[cfg(not(feature = "std"))] +#[macro_export] +macro_rules! trace { + (target: $target:expr, $($arg:tt)+) => ( + () + ); + ($($arg:tt)+) => ( + () + ); +} + +/// In no_std we skip logs for state_machine, this macro +/// is a noops. +#[cfg(not(feature = "std"))] +#[macro_export] +macro_rules! log_error { + (target: $target:expr, $($arg:tt)+) => ( + () + ); + ($($arg:tt)+) => ( + () + ); +} + /// Default rror type to use with state machine trie backend. #[cfg(feature = "std")] pub type DefaultError = String; /// Error type to use with state machine trie backend. #[cfg(not(feature = "std"))] -pub type DefaultError = (); +#[derive(Debug, Default, Clone, Copy, Eq, PartialEq)] +pub struct DefaultError; +#[cfg(not(feature = "std"))] +impl sp_std::fmt::Display for DefaultError { + fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { + write!(f, "Default Error") + } +} pub use crate::overlayed_changes::{ - OverlayedChanges, StorageTransactionCache, StorageKey, StorageValue, + OverlayedChanges, StorageKey, StorageValue, StorageCollection, ChildStorageCollection, }; pub use crate::backend::Backend; @@ -93,7 +130,7 @@ pub use error::{Error, ExecutionError}; #[cfg(feature = "std")] mod std_reexport { - pub use crate::overlayed_changes::StorageChanges; + pub use crate::overlayed_changes::{StorageChanges, StorageTransactionCache}; pub use sp_trie::{trie_types::{Layout, TrieDBMut}, StorageProof, TrieMut, DBValue, MemoryDB}; pub use crate::testing::TestExternalities; pub use crate::basic::BasicExternalities; @@ -369,7 +406,7 @@ mod execution { Some(&mut self.extensions), ); - let id = ext.id; + let id = ext.id(); trace!( target: "state", "{:04x}: Call {} at {:?}. Input={:?}", id, diff --git a/primitives/state-machine/src/overlayed_changes/changeset.rs b/primitives/state-machine/src/overlayed_changes/changeset.rs index 32926dc3cb3a7..771bda79e6640 100644 --- a/primitives/state-machine/src/overlayed_changes/changeset.rs +++ b/primitives/state-machine/src/overlayed_changes/changeset.rs @@ -150,8 +150,10 @@ impl OverlayedValue { *self.value_mut() = value; } - if let Some(extrinsic) = at_extrinsic { - self.transaction_extrinsics_mut().insert(extrinsic); + if CT::CHANGE_TRIE_CAPABLE { + if let Some(extrinsic) = at_extrinsic { + self.transaction_extrinsics_mut().insert(extrinsic); + } } } } @@ -239,7 +241,7 @@ impl OverlayedChangeSet { at_extrinsic: Option, ) { for (key, val) in self.changes.iter_mut().filter(|(k, v)| predicate(k, v)) { - val.set(None, insert_dirty(&mut self.dirty_keys, key.to_owned()), at_extrinsic); + val.set(None, insert_dirty(&mut self.dirty_keys, key.clone()), at_extrinsic); } } @@ -372,7 +374,9 @@ impl OverlayedChangeSet { if has_predecessor { let dropped_tx = overlayed.pop_transaction(); *overlayed.value_mut() = dropped_tx.value; - overlayed.transaction_extrinsics_mut().merge(dropped_tx.extrinsics); + if CT::CHANGE_TRIE_CAPABLE { + overlayed.transaction_extrinsics_mut().merge(dropped_tx.extrinsics); + } } } } diff --git a/primitives/state-machine/src/overlayed_changes/mod.rs b/primitives/state-machine/src/overlayed_changes/mod.rs index b677d0a4ef3c9..7ba86159a356e 100644 --- a/primitives/state-machine/src/overlayed_changes/mod.rs +++ b/primitives/state-machine/src/overlayed_changes/mod.rs @@ -30,7 +30,7 @@ use self::changeset::OverlayedChangeSet; use crate::{ ChangesTrieTransaction, changes_trie::{ - NO_EXTRINSIC_INDEX, BlockNumber, build_changes_trie, + BlockNumber, build_changes_trie, State as ChangesTrieState, }, }; @@ -48,6 +48,10 @@ use crate::DefaultError; pub use self::changeset::{OverlayedValue, NoOpenTransaction, AlreadyInRuntime, NotInRuntime}; + +/// Changes that are made outside of extrinsics are marked with this index; +pub const NO_EXTRINSIC_INDEX: u32 = 0xffffffff; + /// Storage key. pub type StorageKey = Vec; @@ -60,11 +64,6 @@ pub type StorageCollection = Vec<(StorageKey, Option)>; /// In memory arrays of storage values for multiple child tries. pub type ChildStorageCollection = Vec<(StorageKey, StorageCollection)>; -/// `OverlayedChanges` without the capability to record -/// change trie. -#[cfg(not(feature = "std"))] -pub(crate) type OverlayedChangesNoChangeTrie = OverlayedChanges; - /// Technical trait that allows to disable child trie /// footprint when not needed. pub trait ChangeTrieOverlay: Default { @@ -82,8 +81,9 @@ pub trait ChangeTrieOverlay: Default { #[derive(Debug, Default, Eq, PartialEq)] pub struct Extrinsics(Vec); +/// Skip registering extrinsics when using state machine. #[derive(Default)] -pub(crate) struct NoExtrinsics; +pub struct NoExtrinsics; impl ChangeTrieOverlay for NoExtrinsics { const CHANGE_TRIE_CAPABLE: bool = false; @@ -176,6 +176,7 @@ impl StorageChanges { /// The storage transaction are calculated as part of the `storage_root` and /// `changes_trie_storage_root`. These transactions can be reused for importing the block into the /// storage. So, we cache them to not require a recomputation of those transactions. +#[cfg(feature = "std")] pub struct StorageTransactionCache { /// Contains the changes for the main and the child storages as one transaction. pub(crate) transaction: Option, @@ -187,6 +188,7 @@ pub struct StorageTransactionCache { pub(crate) changes_trie_transaction_storage_root: Option>, } +#[cfg(feature = "std")] impl StorageTransactionCache { /// Reset the cached transactions. pub fn reset(&mut self) { @@ -194,6 +196,7 @@ impl StorageTransactionCache Default for StorageTransactionCache { fn default() -> Self { Self { @@ -253,7 +256,7 @@ impl OverlayedChanges { key: &[u8], init: impl Fn() -> StorageValue, ) -> &mut StorageValue { - let value = self.top.modify(key.to_owned(), init, self.extrinsic_index()); + let value = self.top.modify(key.to_vec(), init, self.extrinsic_index()); // if the value was deleted initialise it back with an empty vec value.get_or_insert_with(StorageValue::default) @@ -298,7 +301,7 @@ impl OverlayedChanges { let (changeset, info) = self.children.entry(storage_key).or_insert_with(|| ( top.spawn_child(), - child_info.to_owned() + child_info.clone() ) ); let updatable = info.try_update(child_info); @@ -319,7 +322,7 @@ impl OverlayedChanges { let (changeset, info) = self.children.entry(storage_key).or_insert_with(|| ( top.spawn_child(), - child_info.to_owned() + child_info.clone() ) ); let updatable = info.try_update(child_info); @@ -348,7 +351,7 @@ impl OverlayedChanges { let (changeset, info) = self.children.entry(storage_key).or_insert_with(|| ( top.spawn_child(), - child_info.to_owned() + child_info.clone() ) ); let updatable = info.try_update(child_info); @@ -385,7 +388,7 @@ impl OverlayedChanges { /// there is no open transaction that can be rolled back. pub fn rollback_transaction(&mut self) -> Result<(), NoOpenTransaction> { self.top.rollback_transaction()?; - self.children.retain(|_, (changeset, _)| { + retain_map(&mut self.children, |_, (changeset, _)| { changeset.rollback_transaction() .expect("Top and children changesets are started in lockstep; qed"); !changeset.is_empty() @@ -544,6 +547,9 @@ impl OverlayedChanges { /// Changes that are made outside of extrinsics, are marked with /// `NO_EXTRINSIC_INDEX` index. fn extrinsic_index(&self) -> Option { + if !CT::CHANGE_TRIE_CAPABLE { + return None; + } match self.collect_extrinsics { true => Some( self.storage(EXTRINSIC_INDEX) @@ -557,6 +563,7 @@ impl OverlayedChanges { /// as seen by the current transaction. /// /// Returns the storage root and caches storage transaction in the given `cache`. + #[cfg(feature = "std")] pub fn storage_root>( &self, backend: &B, @@ -578,6 +585,27 @@ impl OverlayedChanges { root } + /// Generate the storage root using `backend` and all changes + /// as seen by the current transaction. + /// + /// Returns the storage root and do not cache. + pub fn storage_root_no_cache>( + &self, + backend: &B, + ) -> H::Out + where H::Out: Ord + Encode, + { + let delta = self.changes().map(|(k, v)| (&k[..], v.value().map(|v| &v[..]))); + let child_delta = self.children() + .map(|(changes, info)| (info, changes.map( + |(k, v)| (&k[..], v.value().map(|v| &v[..])) + ))); + + let (root, _transaction) = backend.full_storage_root(delta, child_delta); + + root + } + /// Generate the changes trie root. /// /// Returns the changes trie root and caches the storage transaction into the given `cache`. @@ -585,6 +613,7 @@ impl OverlayedChanges { /// # Panics /// /// Panics on storage error, when `panic_on_storage_error` is set. + #[cfg(feature = "std")] pub fn changes_trie_root<'a, H: Hasher, N: BlockNumber, B: Backend>( &self, backend: &B, @@ -631,6 +660,29 @@ impl OverlayedChanges { } } +#[cfg(feature = "std")] +fn retain_map(map: &mut Map, f: F) + where + K: std::cmp::Eq + std::hash::Hash, + F: FnMut(&K, &mut V) -> bool, +{ + map.retain(f); +} + +#[cfg(not(feature = "std"))] +fn retain_map(map: &mut Map, mut f: F) + where + K: Ord, + F: FnMut(&K, &mut V) -> bool, +{ + let old = sp_std::mem::replace(map, Map::default()); + for (k, mut v) in old.into_iter() { + if f(&k, &mut v) { + map.insert(k, v); + } + } +} + #[cfg(test)] mod tests { use hex_literal::hex; diff --git a/primitives/state-machine/src/stats.rs b/primitives/state-machine/src/stats.rs index 3ed0b96d16d5e..024411c00fe69 100644 --- a/primitives/state-machine/src/stats.rs +++ b/primitives/state-machine/src/stats.rs @@ -103,7 +103,6 @@ impl UsageInfo { /// Empty statistics. /// /// Means no data was collected. - #[cfg(feature = "std")] pub fn empty() -> Self { Self { reads: UsageUnit::default(), diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 39d5f611e0e4a..e25fb77807c5b 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -35,6 +35,12 @@ pub struct TrieBackend, H: Hasher> { pub (crate) essence: TrieBackendEssence, } +/// Patricia trie-based backend. +/// This is a variant of `TrieBackend` that produce no transactions content. +pub struct TrieBackendNoTransaction, H: Hasher> ( + pub TrieBackend, +) + impl, H: Hasher> TrieBackend where H::Out: Codec { /// Create new trie-based backend. pub fn new(storage: S, root: H::Out) -> Self { diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index e43b266e1057b..a270ac857677c 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -33,12 +33,10 @@ use codec::Encode; use sp_std::boxed::Box; use sp_std::vec::Vec; -#[cfg(feature = "std")] -use std::fmt::format; #[cfg(not(feature = "std"))] macro_rules! format { ($($arg:tt)+) => ( - () + crate::DefaultError ); } diff --git a/primitives/state-machine/src/witness_backend.rs b/primitives/state-machine/src/witness_backend.rs deleted file mode 100644 index ddb1aac585645..0000000000000 --- a/primitives/state-machine/src/witness_backend.rs +++ /dev/null @@ -1,404 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Substrate backend runing on a trie proof, no_std compatible. - -use hash_db::Hasher; -use sp_externalities::{Externalities, ExtensionStore, Error, Extension}; -use sp_core::storage::{ChildInfo, TrackedStorageKey}; -use sp_std::{any::{TypeId, Any}}; -use sp_std::boxed::Box; -use sp_std::vec::Vec; -use sp_trie::MemoryDB; -use crate::trie_backend::TrieBackend; - - -// TODO replace with use crate::StorageValue; -pub type StorageValue = Vec; -// TODO replace with use crate::StorageKey; -pub type StorageKey = Vec; -/// The backend runnig on a trie proof. -pub struct WitnessBackend { -// overlay: OverlayedChangesNoChangeTrie, - trie: TrieBackend, H>, -} - -impl WitnessBackend - where - H: Hasher, - H::Out: Ord + 'static + codec::Codec, -{ - /// Create a new backend. - pub fn new(db: MemoryDB, root: H::Out) -> Self { - WitnessBackend { - trie: TrieBackend::new(db, root), - } - } -} - -impl Externalities for WitnessBackend - where - H: Hasher, - H::Out: Ord + 'static + codec::Codec, -{ - fn set_offchain_storage(&mut self, _key: &[u8], _value: Option<&[u8]>) { - // no ops - } - - fn storage(&self, key: &[u8]) -> Option { - unimplemented!() -/* let result = self.overlay.storage(key).map(|x| x.map(|x| x.to_vec())).unwrap_or_else(|| - self.backend.storage(key).expect(EXT_NOT_ALLOWED_TO_FAIL)); - trace!(target: "state", "{:04x}: Get {}={:?}", - self.id, - HexDisplay::from(&key), - result.as_ref().map(HexDisplay::from) - ); - result*/ - } - - fn storage_hash(&self, key: &[u8]) -> Option> { - unimplemented!() -/* let result = self.overlay - .storage(key) - .map(|x| x.map(|x| H::hash(x))) - .unwrap_or_else(|| self.backend.storage_hash(key).expect(EXT_NOT_ALLOWED_TO_FAIL)); - - result.map(|r| r.encode())*/ - } - - fn child_storage( - &self, - child_info: &ChildInfo, - key: &[u8], - ) -> Option { - unimplemented!() -/* let result = self.overlay - .child_storage(child_info, key) - .map(|x| x.map(|x| x.to_vec())) - .unwrap_or_else(|| - self.backend.child_storage(child_info, key) - .expect(EXT_NOT_ALLOWED_TO_FAIL) - ); - - result -*/ - } - - fn child_storage_hash( - &self, - child_info: &ChildInfo, - key: &[u8], - ) -> Option> { - unimplemented!() -/* let result = self.overlay - .child_storage(child_info, key) - .map(|x| x.map(|x| H::hash(x))) - .unwrap_or_else(|| - self.backend.child_storage_hash(child_info, key) - .expect(EXT_NOT_ALLOWED_TO_FAIL) - ); - result.map(|r| r.encode()) -*/ - } - - fn exists_storage(&self, key: &[u8]) -> bool { - unimplemented!() -/*let result = match self.overlay.storage(key) { - Some(x) => x.is_some(), - _ => self.backend.exists_storage(key).expect(EXT_NOT_ALLOWED_TO_FAIL), - }; - - result*/ - } - - fn exists_child_storage( - &self, - child_info: &ChildInfo, - key: &[u8], - ) -> bool { - unimplemented!() -/* - let result = match self.overlay.child_storage(child_info, key) { - Some(x) => x.is_some(), - _ => self.backend - .exists_child_storage(child_info, key) - .expect(EXT_NOT_ALLOWED_TO_FAIL), - }; - result*/ - } - - fn next_storage_key(&self, key: &[u8]) -> Option { - unimplemented!() - /* - let next_backend_key = self.backend.next_storage_key(key).expect(EXT_NOT_ALLOWED_TO_FAIL); - let next_overlay_key_change = self.overlay.next_storage_key_change(key); - - match (next_backend_key, next_overlay_key_change) { - (Some(backend_key), Some(overlay_key)) if &backend_key[..] < overlay_key.0 => Some(backend_key), - (backend_key, None) => backend_key, - (_, Some(overlay_key)) => if overlay_key.1.value().is_some() { - Some(overlay_key.0.to_vec()) - } else { - self.next_storage_key(&overlay_key.0[..]) - }, - } - */ - } - - fn next_child_storage_key( - &self, - child_info: &ChildInfo, - key: &[u8], - ) -> Option { - unimplemented!() -/* let next_backend_key = self.backend - .next_child_storage_key(child_info, key) - .expect(EXT_NOT_ALLOWED_TO_FAIL); - let next_overlay_key_change = self.overlay.next_child_storage_key_change( - child_info.storage_key(), - key - ); - - match (next_backend_key, next_overlay_key_change) { - (Some(backend_key), Some(overlay_key)) if &backend_key[..] < overlay_key.0 => Some(backend_key), - (backend_key, None) => backend_key, - (_, Some(overlay_key)) => if overlay_key.1.value().is_some() { - Some(overlay_key.0.to_vec()) - } else { - self.next_child_storage_key( - child_info, - &overlay_key.0[..], - ) - }, - }*/ - } - - fn place_storage(&mut self, key: StorageKey, value: Option) { - unimplemented!() -/* if is_child_storage_key(&key) { - return; - } - - self.mark_dirty(); - self.overlay.set_storage(key, value);*/ - } - - fn place_child_storage( - &mut self, - child_info: &ChildInfo, - key: StorageKey, - value: Option, - ) { - unimplemented!() -/* - self.mark_dirty(); - self.overlay.set_child_storage(child_info, key, value); -*/ - } - - fn kill_child_storage( - &mut self, - child_info: &ChildInfo, - ) { - unimplemented!() - /* - self.mark_dirty(); - self.overlay.clear_child_storage(child_info); - self.backend.for_keys_in_child_storage(child_info, |key| { - self.overlay.set_child_storage(child_info, key.to_vec(), None); - }); - */ - } - - fn clear_prefix(&mut self, prefix: &[u8]) { - unimplemented!() -/* - if is_child_storage_key(prefix) { - return; - } - - self.mark_dirty(); - self.overlay.clear_prefix(prefix); - self.backend.for_keys_with_prefix(prefix, |key| { - self.overlay.set_storage(key.to_vec(), None); - }); -*/ - } - - fn clear_child_prefix( - &mut self, - child_info: &ChildInfo, - prefix: &[u8], - ) { - unimplemented!() -/* - self.mark_dirty(); - self.overlay.clear_child_prefix(child_info, prefix); - self.backend.for_child_keys_with_prefix(child_info, prefix, |key| { - self.overlay.set_child_storage(child_info, key.to_vec(), None); - }); -*/ - } - - fn storage_append( - &mut self, - key: Vec, - value: Vec, - ) { - unimplemented!() -/* - self.mark_dirty(); - - let backend = &mut self.backend; - let current_value = self.overlay.value_mut_or_insert_with( - &key, - || backend.storage(&key).expect(EXT_NOT_ALLOWED_TO_FAIL).unwrap_or_default() - ); - StorageAppend::new(current_value).append(value); -*/ - } - - fn chain_id(&self) -> u64 { - 42 - } - - fn storage_root(&mut self) -> Vec { - unimplemented!() - /*if let Some(ref root) = self.storage_transaction_cache.transaction_storage_root { - return root.encode(); - } - - let root = self.overlay.storage_root(self.backend, self.storage_transaction_cache); - root.encode() - */ - } - - fn child_storage_root( - &mut self, - child_info: &ChildInfo, - ) -> Vec { - unimplemented!() - /* - let storage_key = child_info.storage_key(); - let prefixed_storage_key = child_info.prefixed_storage_key(); - if self.storage_transaction_cache.transaction_storage_root.is_some() { - let root = self - .storage(prefixed_storage_key.as_slice()) - .and_then(|k| Decode::decode(&mut &k[..]).ok()) - .unwrap_or_else( - || empty_child_trie_root::>() - ); - root.encode() - } else { - let root = if let Some((changes, info)) = self.overlay.child_changes(storage_key) { - let delta = changes.map(|(k, v)| (k.as_ref(), v.value().map(AsRef::as_ref))); - Some(self.backend.child_storage_root(info, delta)) - } else { - None - }; - - if let Some((root, is_empty, _)) = root { - let root = root.encode(); - if is_empty { - self.overlay.set_storage(prefixed_storage_key.into_inner(), None); - } else { - self.overlay.set_storage(prefixed_storage_key.into_inner(), Some(root.clone())); - } - root - } else { - let root = self - .storage(prefixed_storage_key.as_slice()) - .and_then(|k| Decode::decode(&mut &k[..]).ok()) - .unwrap_or_else( - || empty_child_trie_root::>() - ); - root.encode() - } - } - */ - } - - fn storage_changes_root(&mut self, parent_hash: &[u8]) -> Result>, ()> { - unimplemented!() - } - - fn storage_start_transaction(&mut self) { - unimplemented!() - //self.overlay.start_transaction() - } - - fn storage_rollback_transaction(&mut self) -> Result<(), ()> { - unimplemented!() - //self.mark_dirty(); - //self.overlay.rollback_transaction().map_err(|_| ()) - } - - fn storage_commit_transaction(&mut self) -> Result<(), ()> { - unimplemented!() - //self.overlay.commit_transaction().map_err(|_| ()) - } - - fn wipe(&mut self) { - unimplemented!() - } - - fn commit(&mut self) { - unimplemented!() - } - - fn read_write_count(&self) -> (u32, u32, u32, u32) { - unimplemented!() - } - - fn reset_read_write_count(&mut self) { - unimplemented!() - } - - fn get_whitelist(&self) -> Vec { - unimplemented!() - } - - fn set_whitelist(&mut self, new: Vec) { - unimplemented!() - } -} - -impl ExtensionStore for WitnessBackend -where - H: Hasher, - H::Out: Ord + 'static + codec::Codec, -{ - fn extension_by_type_id(&mut self, _type_id: TypeId) -> Option<&mut dyn Any> { - None - } - - fn register_extension_with_type_id( - &mut self, - _type_id: TypeId, - _extension: Box, - ) -> Result<(), Error> { - Err(Error::ExtensionsAreNotSupported) - } - - fn deregister_extension_by_type_id( - &mut self, - _type_id: TypeId, - ) -> Result<(), Error> { - Err(Error::ExtensionsAreNotSupported) - } -} diff --git a/primitives/state-machine/src/witness_ext.rs b/primitives/state-machine/src/witness_ext.rs new file mode 100644 index 0000000000000..50d56ec28a15b --- /dev/null +++ b/primitives/state-machine/src/witness_ext.rs @@ -0,0 +1,252 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Substrate backend runing on a trie proof, no_std compatible. + +use hash_db::Hasher; +use sp_externalities::{Externalities, ExtensionStore, Error, Extension}; +use sp_core::storage::{ChildInfo, TrackedStorageKey}; +use sp_std::{any::{TypeId, Any}}; +use sp_std::boxed::Box; +use sp_std::vec::Vec; +use sp_trie::MemoryDB; +use crate::trie_backend::TrieBackend; +use crate::ext::{ExtInner, ExtInnerMut}; +use crate::overlayed_changes::{OverlayedChanges, NoExtrinsics}; +use crate::{StorageValue, StorageKey}; + +/// The backend runnig on a trie proof. +pub struct WitnessExt { + /// The overlayed changes to write to. + pub overlay: OverlayedChanges, + /// The storage backend to read from. + pub backend: TrieBackend, H>, // TODO switch to a trie backend that do not produce tx !!! +} + +impl WitnessExt + where + H: Hasher, + H::Out: Ord + 'static + codec::Codec, +{ + /// Create a new backend. + pub fn new(db: MemoryDB, root: H::Out) -> Self { + WitnessExt { + backend: TrieBackend::new(db, root), + overlay: OverlayedChanges::::default(), + } + } + + /// Access methods for `ExtInnerMut`. + fn ext_inner_mut(&mut self) -> ExtInnerMut, H>, NoExtrinsics> { + ExtInnerMut { + overlay: &mut self.overlay, + backend: &self.backend, + id: 0, + _phantom: Default::default(), + } + } + + /// Access methods for `ExtInner`. + fn ext_inner(&self) -> ExtInner, H>, NoExtrinsics> { + ExtInner { + overlay: &self.overlay, + backend: &self.backend, + id: 0, + _phantom: Default::default(), + } + } +} + + + +impl Externalities for WitnessExt + where + H: Hasher, + H::Out: Ord + 'static + codec::Codec, +{ + fn set_offchain_storage(&mut self, _key: &[u8], _value: Option<&[u8]>) { + unimplemented!("Unsupported"); + } + + fn storage(&self, key: &[u8]) -> Option { + self.ext_inner().storage(key) + } + + fn storage_hash(&self, key: &[u8]) -> Option> { + self.ext_inner().storage_hash(key) + } + + fn child_storage( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Option { + self.ext_inner().child_storage(child_info, key) + } + + fn child_storage_hash( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Option> { + self.ext_inner().child_storage_hash(child_info, key) + } + + fn exists_storage(&self, key: &[u8]) -> bool { + self.ext_inner().exists_storage(key) + } + + fn exists_child_storage( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> bool { + self.ext_inner().exists_child_storage(child_info, key) + } + + fn next_storage_key(&self, key: &[u8]) -> Option { + self.ext_inner().next_storage_key(key) + } + + fn next_child_storage_key( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Option { + self.ext_inner().next_child_storage_key(child_info, key) + } + + fn place_storage(&mut self, key: StorageKey, value: Option) { + self.ext_inner_mut().place_storage(key, value) + } + + fn place_child_storage( + &mut self, + child_info: &ChildInfo, + key: StorageKey, + value: Option, + ) { + self.ext_inner_mut().place_child_storage(child_info, key, value) + } + + fn kill_child_storage( + &mut self, + child_info: &ChildInfo, + ) { + self.ext_inner_mut().kill_child_storage(child_info) + } + + fn clear_prefix(&mut self, prefix: &[u8]) { + self.ext_inner_mut().clear_prefix(prefix) + } + + fn clear_child_prefix( + &mut self, + child_info: &ChildInfo, + prefix: &[u8], + ) { + self.ext_inner_mut().clear_child_prefix(child_info, prefix) + } + + fn storage_append( + &mut self, + key: Vec, + value: Vec, + ) { + self.ext_inner_mut().storage_append(key, value) + } + + fn chain_id(&self) -> u64 { + 42 + } + + fn storage_root(&mut self) -> Vec { + self.ext_inner_mut().storage_root() + } + + fn child_storage_root( + &mut self, + child_info: &ChildInfo, + ) -> Vec { + self.ext_inner_mut().child_storage_root(child_info) + } + + fn storage_changes_root(&mut self, _parent_hash: &[u8]) -> Result>, ()> { + unimplemented!("Unsupported"); + } + + fn storage_start_transaction(&mut self) { + self.ext_inner_mut().storage_start_transaction() + } + + fn storage_rollback_transaction(&mut self) -> Result<(), ()> { + self.ext_inner_mut().storage_rollback_transaction() + } + + fn storage_commit_transaction(&mut self) -> Result<(), ()> { + self.ext_inner_mut().storage_commit_transaction() + } + + fn wipe(&mut self) { + unimplemented!("Unsupported"); + } + + fn commit(&mut self) { + unimplemented!("Unsupported"); + } + + fn read_write_count(&self) -> (u32, u32, u32, u32) { + self.ext_inner().read_write_count() + } + + fn reset_read_write_count(&mut self) { + self.ext_inner_mut().reset_read_write_count() + } + + fn get_whitelist(&self) -> Vec { + unimplemented!("Unsupported"); + } + + fn set_whitelist(&mut self, _new: Vec) { + unimplemented!("Unsupported"); + } +} + +impl ExtensionStore for WitnessExt +where + H: Hasher, + H::Out: Ord + 'static + codec::Codec, +{ + fn extension_by_type_id(&mut self, _type_id: TypeId) -> Option<&mut dyn Any> { + None + } + + fn register_extension_with_type_id( + &mut self, + _type_id: TypeId, + _extension: Box, + ) -> Result<(), Error> { + Err(Error::ExtensionsAreNotSupported) + } + + fn deregister_extension_by_type_id( + &mut self, + _type_id: TypeId, + ) -> Result<(), Error> { + Err(Error::ExtensionsAreNotSupported) + } +} diff --git a/test-utils/runtime/Cargo.toml b/test-utils/runtime/Cargo.toml index dbbac8652ad84..142361cf07367 100644 --- a/test-utils/runtime/Cargo.toml +++ b/test-utils/runtime/Cargo.toml @@ -43,6 +43,7 @@ trie-db = { version = "0.22.0", default-features = false } parity-util-mem = { version = "0.7.0", default-features = false, features = ["primitive-types"] } sc-service = { version = "0.8.0-rc5", default-features = false, optional = true, features = ["test-helpers"], path = "../../client/service" } sp-state-machine = { version = "0.8.0-rc5", default-features = false, path = "../../primitives/state-machine" } +sp-externalities = { version = "0.8.0-rc5", default-features = false, path = "../../primitives/externalities" } # 3rd party cfg-if = "0.1.10" @@ -84,6 +85,7 @@ std = [ "sp-session/std", "sp-api/std", "sp-runtime/std", + "sp-externalities/std", "sp-state-machine/std", "pallet-babe/std", "frame-system-rpc-runtime-api/std", diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index ef7f71147428f..da22ad351db02 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -145,7 +145,7 @@ pub enum Extrinsic { IncludeData(Vec), StorageChange(Vec, Option>), ChangesTrieConfigUpdate(Option), - WitnessBackend, + WitnessExt, } parity_util_mem::malloc_size_of_is_0!(Extrinsic); // non-opaque extrinsic does not need this @@ -174,8 +174,8 @@ impl BlindCheckable for Extrinsic { Extrinsic::StorageChange(key, value) => Ok(Extrinsic::StorageChange(key, value)), Extrinsic::ChangesTrieConfigUpdate(new_config) => Ok(Extrinsic::ChangesTrieConfigUpdate(new_config)), - Extrinsic::WitnessBackend => - Ok(Extrinsic::WitnessBackend), + Extrinsic::WitnessExt => + Ok(Extrinsic::WitnessExt), } } } diff --git a/test-utils/runtime/src/system.rs b/test-utils/runtime/src/system.rs index 713e1df423545..bb924e5fef6c5 100644 --- a/test-utils/runtime/src/system.rs +++ b/test-utils/runtime/src/system.rs @@ -261,7 +261,7 @@ fn execute_transaction_backend(utx: &Extrinsic, extrinsic_index: u32) -> ApplyEx execute_storage_change(key, value.as_ref().map(|v| &**v)), Extrinsic::ChangesTrieConfigUpdate(ref new_config) => execute_changes_trie_config_update(new_config.clone()), - Extrinsic::WitnessBackend => + Extrinsic::WitnessExt => execute_witness_backend(), } } @@ -306,13 +306,17 @@ fn execute_storage_change(key: &[u8], value: Option<&[u8]>) -> ApplyExtrinsicRes } fn execute_witness_backend() -> ApplyExtrinsicResult { - use sp_state_machine::witness_backend::WitnessBackend; - let mut backend = WitnessBackend::::new(); - unimplemented!("make use of witness backend for no std"); + use sp_state_machine::witness_ext::WitnessExt; + use sp_externalities::Externalities; + let mut backend = WitnessExt::::new( + Default::default(), + Default::default(), + ); + backend.place_storage(vec![0], Some(vec![1])); + assert!(backend.storage(&[0]).is_some()); Ok(Ok(())) } - fn execute_changes_trie_config_update(new_config: Option) -> ApplyExtrinsicResult { match new_config.clone() { Some(new_config) => storage::unhashed::put_raw( From 2cb6378c7a85c15b7764b139fd1b26fd0b95eaae Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 21 Aug 2020 16:06:34 +0200 Subject: [PATCH 03/21] undo --- primitives/state-machine/src/witness_ext.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/state-machine/src/witness_ext.rs b/primitives/state-machine/src/witness_ext.rs index 50d56ec28a15b..3387f5d59533f 100644 --- a/primitives/state-machine/src/witness_ext.rs +++ b/primitives/state-machine/src/witness_ext.rs @@ -34,7 +34,7 @@ pub struct WitnessExt { /// The overlayed changes to write to. pub overlay: OverlayedChanges, /// The storage backend to read from. - pub backend: TrieBackend, H>, // TODO switch to a trie backend that do not produce tx !!! + pub backend: TrieBackend, H>, } impl WitnessExt From b0e04a450f30715755e3ac7691a7d44dac925bfe Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 21 Aug 2020 16:18:53 +0200 Subject: [PATCH 04/21] Started no transaction, but would need using a different root calculation method, out of the scope of this pr, will roll back. --- primitives/state-machine/src/trie_backend.rs | 104 ++++++++++++++++++- primitives/state-machine/src/witness_ext.rs | 10 +- 2 files changed, 108 insertions(+), 6 deletions(-) diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index e25fb77807c5b..b1582f6134eba 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -39,7 +39,7 @@ pub struct TrieBackend, H: Hasher> { /// This is a variant of `TrieBackend` that produce no transactions content. pub struct TrieBackendNoTransaction, H: Hasher> ( pub TrieBackend, -) +); impl, H: Hasher> TrieBackend where H::Out: Codec { /// Create new trie-based backend. @@ -81,6 +81,12 @@ impl, H: Hasher> sp_std::fmt::Debug for TrieBackend, H: Hasher> sp_std::fmt::Debug for TrieBackendNoTransaction { + fn fmt(&self, f: &mut sp_std::fmt::Formatter<'_>) -> sp_std::fmt::Result { + write!(f, "TrieBackendNoTransaction") + } +} + impl, H: Hasher> Backend for TrieBackend where H::Out: Ord + Codec, { @@ -254,6 +260,102 @@ impl, H: Hasher> Backend for TrieBackend where } } +impl, H: Hasher> Backend for TrieBackendNoTransaction where + H::Out: Ord + Codec, +{ + type Error = crate::DefaultError; + type Transaction = S::Overlay; + type TrieBackendStorage = S; + + fn storage(&self, key: &[u8]) -> Result, Self::Error> { + self.0.storage(key) + } + + fn child_storage( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Result, Self::Error> { + self.0.child_storage(child_info, key) + } + + fn next_storage_key(&self, key: &[u8]) -> Result, Self::Error> { + self.0.next_storage_key(key) + } + + fn next_child_storage_key( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Result, Self::Error> { + self.0.next_child_storage_key(child_info, key) + } + + fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { + self.0.for_keys_with_prefix(prefix, f) + } + + fn for_key_values_with_prefix(&self, prefix: &[u8], f: F) { + self.0.for_key_values_with_prefix(prefix, f) + } + + fn for_keys_in_child_storage( + &self, + child_info: &ChildInfo, + f: F, + ) { + self.0.for_keys_in_child_storage(child_info, f) + } + + fn for_child_keys_with_prefix( + &self, + child_info: &ChildInfo, + prefix: &[u8], + f: F, + ) { + self.0.for_child_keys_with_prefix(child_info, prefix, f) + } + + fn pairs(&self) -> Vec<(StorageKey, StorageValue)> { + self.0.pairs() + } + + fn keys(&self, prefix: &[u8]) -> Vec { + self.0.keys(prefix) + } + + fn storage_root<'a>( + &self, + delta: impl Iterator)>, + ) -> (H::Out, Self::Transaction) where H::Out: Ord { + self.0.storage_root(delta) + } + + fn child_storage_root<'a>( + &self, + child_info: &ChildInfo, + delta: impl Iterator)>, + ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord { + self.0.child_storage_root(child_info, delta) + } + + fn as_trie_backend(&mut self) -> Option<&TrieBackend> { + self.0.as_trie_backend() + } + + fn register_overlay_stats(&mut self, stats: &crate::stats::StateMachineStats) { + self.0.register_overlay_stats(stats) + } + + fn usage_info(&self) -> crate::UsageInfo { + self.0.usage_info() + } + + fn wipe(&self) -> Result<(), Self::Error> { + self.0.wipe() + } +} + #[cfg(test)] pub mod tests { use std::{collections::HashSet, iter}; diff --git a/primitives/state-machine/src/witness_ext.rs b/primitives/state-machine/src/witness_ext.rs index 3387f5d59533f..5d8f122b537e2 100644 --- a/primitives/state-machine/src/witness_ext.rs +++ b/primitives/state-machine/src/witness_ext.rs @@ -24,7 +24,7 @@ use sp_std::{any::{TypeId, Any}}; use sp_std::boxed::Box; use sp_std::vec::Vec; use sp_trie::MemoryDB; -use crate::trie_backend::TrieBackend; +use crate::trie_backend::{TrieBackend, TrieBackendNoTransaction}; use crate::ext::{ExtInner, ExtInnerMut}; use crate::overlayed_changes::{OverlayedChanges, NoExtrinsics}; use crate::{StorageValue, StorageKey}; @@ -34,7 +34,7 @@ pub struct WitnessExt { /// The overlayed changes to write to. pub overlay: OverlayedChanges, /// The storage backend to read from. - pub backend: TrieBackend, H>, + pub backend: TrieBackendNoTransaction, H>, } impl WitnessExt @@ -45,13 +45,13 @@ impl WitnessExt /// Create a new backend. pub fn new(db: MemoryDB, root: H::Out) -> Self { WitnessExt { - backend: TrieBackend::new(db, root), + backend: TrieBackendNoTransaction(TrieBackend::new(db, root)), overlay: OverlayedChanges::::default(), } } /// Access methods for `ExtInnerMut`. - fn ext_inner_mut(&mut self) -> ExtInnerMut, H>, NoExtrinsics> { + fn ext_inner_mut(&mut self) -> ExtInnerMut, H>, NoExtrinsics> { ExtInnerMut { overlay: &mut self.overlay, backend: &self.backend, @@ -61,7 +61,7 @@ impl WitnessExt } /// Access methods for `ExtInner`. - fn ext_inner(&self) -> ExtInner, H>, NoExtrinsics> { + fn ext_inner(&self) -> ExtInner, H>, NoExtrinsics> { ExtInner { overlay: &self.overlay, backend: &self.backend, From a9aa27cf2c76aedd696f5053a2b60d9e459b9c95 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 21 Aug 2020 16:21:57 +0200 Subject: [PATCH 05/21] Remove NoTransaction. --- primitives/state-machine/src/trie_backend.rs | 108 ------------------- primitives/state-machine/src/witness_ext.rs | 12 +-- 2 files changed, 5 insertions(+), 115 deletions(-) diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index b1582f6134eba..39d5f611e0e4a 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -35,12 +35,6 @@ pub struct TrieBackend, H: Hasher> { pub (crate) essence: TrieBackendEssence, } -/// Patricia trie-based backend. -/// This is a variant of `TrieBackend` that produce no transactions content. -pub struct TrieBackendNoTransaction, H: Hasher> ( - pub TrieBackend, -); - impl, H: Hasher> TrieBackend where H::Out: Codec { /// Create new trie-based backend. pub fn new(storage: S, root: H::Out) -> Self { @@ -81,12 +75,6 @@ impl, H: Hasher> sp_std::fmt::Debug for TrieBackend, H: Hasher> sp_std::fmt::Debug for TrieBackendNoTransaction { - fn fmt(&self, f: &mut sp_std::fmt::Formatter<'_>) -> sp_std::fmt::Result { - write!(f, "TrieBackendNoTransaction") - } -} - impl, H: Hasher> Backend for TrieBackend where H::Out: Ord + Codec, { @@ -260,102 +248,6 @@ impl, H: Hasher> Backend for TrieBackend where } } -impl, H: Hasher> Backend for TrieBackendNoTransaction where - H::Out: Ord + Codec, -{ - type Error = crate::DefaultError; - type Transaction = S::Overlay; - type TrieBackendStorage = S; - - fn storage(&self, key: &[u8]) -> Result, Self::Error> { - self.0.storage(key) - } - - fn child_storage( - &self, - child_info: &ChildInfo, - key: &[u8], - ) -> Result, Self::Error> { - self.0.child_storage(child_info, key) - } - - fn next_storage_key(&self, key: &[u8]) -> Result, Self::Error> { - self.0.next_storage_key(key) - } - - fn next_child_storage_key( - &self, - child_info: &ChildInfo, - key: &[u8], - ) -> Result, Self::Error> { - self.0.next_child_storage_key(child_info, key) - } - - fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { - self.0.for_keys_with_prefix(prefix, f) - } - - fn for_key_values_with_prefix(&self, prefix: &[u8], f: F) { - self.0.for_key_values_with_prefix(prefix, f) - } - - fn for_keys_in_child_storage( - &self, - child_info: &ChildInfo, - f: F, - ) { - self.0.for_keys_in_child_storage(child_info, f) - } - - fn for_child_keys_with_prefix( - &self, - child_info: &ChildInfo, - prefix: &[u8], - f: F, - ) { - self.0.for_child_keys_with_prefix(child_info, prefix, f) - } - - fn pairs(&self) -> Vec<(StorageKey, StorageValue)> { - self.0.pairs() - } - - fn keys(&self, prefix: &[u8]) -> Vec { - self.0.keys(prefix) - } - - fn storage_root<'a>( - &self, - delta: impl Iterator)>, - ) -> (H::Out, Self::Transaction) where H::Out: Ord { - self.0.storage_root(delta) - } - - fn child_storage_root<'a>( - &self, - child_info: &ChildInfo, - delta: impl Iterator)>, - ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord { - self.0.child_storage_root(child_info, delta) - } - - fn as_trie_backend(&mut self) -> Option<&TrieBackend> { - self.0.as_trie_backend() - } - - fn register_overlay_stats(&mut self, stats: &crate::stats::StateMachineStats) { - self.0.register_overlay_stats(stats) - } - - fn usage_info(&self) -> crate::UsageInfo { - self.0.usage_info() - } - - fn wipe(&self) -> Result<(), Self::Error> { - self.0.wipe() - } -} - #[cfg(test)] pub mod tests { use std::{collections::HashSet, iter}; diff --git a/primitives/state-machine/src/witness_ext.rs b/primitives/state-machine/src/witness_ext.rs index 5d8f122b537e2..70ca5a0751123 100644 --- a/primitives/state-machine/src/witness_ext.rs +++ b/primitives/state-machine/src/witness_ext.rs @@ -24,7 +24,7 @@ use sp_std::{any::{TypeId, Any}}; use sp_std::boxed::Box; use sp_std::vec::Vec; use sp_trie::MemoryDB; -use crate::trie_backend::{TrieBackend, TrieBackendNoTransaction}; +use crate::trie_backend::TrieBackend; use crate::ext::{ExtInner, ExtInnerMut}; use crate::overlayed_changes::{OverlayedChanges, NoExtrinsics}; use crate::{StorageValue, StorageKey}; @@ -34,7 +34,7 @@ pub struct WitnessExt { /// The overlayed changes to write to. pub overlay: OverlayedChanges, /// The storage backend to read from. - pub backend: TrieBackendNoTransaction, H>, + pub backend: TrieBackend, H>, } impl WitnessExt @@ -45,13 +45,13 @@ impl WitnessExt /// Create a new backend. pub fn new(db: MemoryDB, root: H::Out) -> Self { WitnessExt { - backend: TrieBackendNoTransaction(TrieBackend::new(db, root)), + backend: TrieBackend::new(db, root), overlay: OverlayedChanges::::default(), } } /// Access methods for `ExtInnerMut`. - fn ext_inner_mut(&mut self) -> ExtInnerMut, H>, NoExtrinsics> { + fn ext_inner_mut(&mut self) -> ExtInnerMut, H>, NoExtrinsics> { ExtInnerMut { overlay: &mut self.overlay, backend: &self.backend, @@ -61,7 +61,7 @@ impl WitnessExt } /// Access methods for `ExtInner`. - fn ext_inner(&self) -> ExtInner, H>, NoExtrinsics> { + fn ext_inner(&self) -> ExtInner, H>, NoExtrinsics> { ExtInner { overlay: &self.overlay, backend: &self.backend, @@ -71,8 +71,6 @@ impl WitnessExt } } - - impl Externalities for WitnessExt where H: Hasher, From cac22803b196c8262815ad19736376909c2778f5 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Fri, 21 Aug 2020 19:10:09 +0200 Subject: [PATCH 06/21] partially address review. dummy stats implementation for no_std. --- primitives/externalities/src/extensions.rs | 7 +-- primitives/state-machine/src/stats.rs | 57 ++++++++++++++++----- primitives/state-machine/src/witness_ext.rs | 2 +- 3 files changed, 46 insertions(+), 20 deletions(-) diff --git a/primitives/externalities/src/extensions.rs b/primitives/externalities/src/extensions.rs index 8eb3c9a568324..bb611952521ba 100644 --- a/primitives/externalities/src/extensions.rs +++ b/primitives/externalities/src/extensions.rs @@ -22,10 +22,7 @@ //! //! It is required that each extension implements the [`Extension`] trait. -#[cfg(feature = "std")] -use std::{collections::HashMap as Map, collections::hash_map::Entry}; -#[cfg(not(feature = "std"))] -use sp_std::collections::btree_map::{BTreeMap as Map, Entry}; +use sp_std::collections::btree_map::{BTreeMap, Entry}; use sp_std::{any::{Any, TypeId}, ops::DerefMut, boxed::Box}; use crate::Error; @@ -108,7 +105,7 @@ pub trait ExtensionStore { /// Stores extensions that should be made available through the externalities. #[derive(Default)] pub struct Extensions { - extensions: Map>, + extensions: BTreeMap>, } #[cfg(feature = "std")] diff --git a/primitives/state-machine/src/stats.rs b/primitives/state-machine/src/stats.rs index 024411c00fe69..2b5557620b8c8 100644 --- a/primitives/state-machine/src/stats.rs +++ b/primitives/state-machine/src/stats.rs @@ -19,22 +19,10 @@ #[cfg(feature = "std")] use std::time::{Instant, Duration}; -#[cfg(not(feature = "std"))] -type Instant = (); -#[cfg(not(feature = "std"))] -use core::time::Duration; use sp_std::cell::RefCell; -#[cfg(feature = "std")] -fn now() -> Instant { - Instant::now() -} -#[cfg(not(feature = "std"))] -fn now() -> Instant { - () -} - /// Measured count of operations and total bytes. +#[cfg(feature = "std")] #[derive(Clone, Debug, Default)] pub struct UsageUnit { /// Number of operations. @@ -43,7 +31,13 @@ pub struct UsageUnit { pub bytes: u64, } +/// Ignore stat for no_std. +#[cfg(not(feature = "std"))] +#[derive(Clone, Debug, Default)] +pub struct UsageUnit; + /// Usage statistics for state backend. +#[cfg(feature = "std")] #[derive(Clone, Debug)] pub struct UsageInfo { /// Read statistics (total). @@ -70,8 +64,13 @@ pub struct UsageInfo { pub span: Duration, } +#[cfg(not(feature = "std"))] +#[derive(Clone, Debug, Default)] +pub struct UsageInfo; + /// Accumulated usage statistics specific to state machine /// crate. +#[cfg(feature = "std")] #[derive(Debug, Default, Clone)] pub struct StateMachineStats { /// Number of read query from runtime @@ -89,6 +88,11 @@ pub struct StateMachineStats { pub bytes_writes_overlay: RefCell, } +#[cfg(not(feature = "std"))] +#[derive(Clone, Debug, Default)] +pub struct StateMachineStats; + +#[cfg(feature = "std")] impl StateMachineStats { /// Accumulates some registered stats. pub fn add(&self, other: &StateMachineStats) { @@ -99,6 +103,7 @@ impl StateMachineStats { } } +#[cfg(feature = "std")] impl UsageInfo { /// Empty statistics. /// @@ -113,7 +118,8 @@ impl UsageInfo { cache_reads: UsageUnit::default(), modified_reads: UsageUnit::default(), memory: 0, - started: now(), + started: Instant::now(), + #[cfg(feature = "std")] span: Default::default(), } } @@ -126,6 +132,19 @@ impl UsageInfo { } } +/// Ignore stat for no_std. +#[cfg(not(feature = "std"))] +impl UsageInfo { + /// Ignore stat for no_std. + pub fn empty() -> Self { + UsageInfo + } + /// Ignore stat for no_std. + pub fn include_state_machine_states(&mut self, _count: &StateMachineStats) { + } +} + +#[cfg(feature = "std")] impl StateMachineStats { /// Tally one read modified operation, of some length. pub fn tally_read_modified(&self, data_bytes: u64) { @@ -138,3 +157,13 @@ impl StateMachineStats { *self.bytes_writes_overlay.borrow_mut() += data_bytes; } } + +#[cfg(not(feature = "std"))] +impl StateMachineStats { + /// Ignore stat for no_std. + pub fn tally_read_modified(&self, _data_bytes: u64) { + } + /// Ignore stat for no_std. + pub fn tally_write_overlay(&self, _data_bytes: u64) { + } +} diff --git a/primitives/state-machine/src/witness_ext.rs b/primitives/state-machine/src/witness_ext.rs index 70ca5a0751123..71cf563ac2233 100644 --- a/primitives/state-machine/src/witness_ext.rs +++ b/primitives/state-machine/src/witness_ext.rs @@ -184,7 +184,7 @@ impl Externalities for WitnessExt } fn storage_changes_root(&mut self, _parent_hash: &[u8]) -> Result>, ()> { - unimplemented!("Unsupported"); + Ok(None) } fn storage_start_transaction(&mut self) { From 11305d1f7d8da6fd765bf335045c6942c4a24232 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 24 Aug 2020 15:37:45 +0200 Subject: [PATCH 07/21] Remove ChangeTrieOverlay. --- .../state-machine/src/changes_trie/build.rs | 19 +++-- .../state-machine/src/changes_trie/mod.rs | 11 ++- primitives/state-machine/src/ext.rs | 17 ++--- .../src/overlayed_changes/changeset.rs | 44 +++++------- .../src/overlayed_changes/mod.rs | 69 +++++-------------- primitives/state-machine/src/witness_ext.rs | 10 +-- 6 files changed, 62 insertions(+), 108 deletions(-) diff --git a/primitives/state-machine/src/changes_trie/build.rs b/primitives/state-machine/src/changes_trie/build.rs index ff9a03090e117..2628d8d15358c 100644 --- a/primitives/state-machine/src/changes_trie/build.rs +++ b/primitives/state-machine/src/changes_trie/build.rs @@ -25,7 +25,7 @@ use num_traits::One; use crate::{ StorageKey, backend::Backend, - overlayed_changes::{OverlayedChanges, OverlayedValue, ChangeTrieOverlay}, + overlayed_changes::{OverlayedChanges, OverlayedValue}, trie_backend_essence::TrieBackendEssence, changes_trie::{ AnchorBlockId, ConfigurationRange, Storage, BlockNumber, @@ -39,11 +39,11 @@ use sp_core::storage::{ChildInfo, PrefixedStorageKey}; /// /// Returns Err if storage error has occurred OR if storage haven't returned /// required data. -pub(crate) fn prepare_input<'a, B, H, Number, CT>( +pub(crate) fn prepare_input<'a, B, H, Number>( backend: &'a B, storage: &'a dyn Storage, config: ConfigurationRange<'a, Number>, - overlay: &'a OverlayedChanges, + overlay: &'a OverlayedChanges, parent: &'a AnchorBlockId, ) -> Result<( impl Iterator> + 'a, @@ -55,7 +55,6 @@ pub(crate) fn prepare_input<'a, B, H, Number, CT>( H: Hasher + 'a, H::Out: Encode, Number: BlockNumber, - CT: ChangeTrieOverlay, { let number = parent.number.clone() + One::one(); let (extrinsics_input, children_extrinsics_input) = prepare_extrinsics_input( @@ -94,10 +93,10 @@ pub(crate) fn prepare_input<'a, B, H, Number, CT>( )) } /// Prepare ExtrinsicIndex input pairs. -fn prepare_extrinsics_input<'a, B, H, Number, CT>( +fn prepare_extrinsics_input<'a, B, H, Number>( backend: &'a B, block: &Number, - overlay: &'a OverlayedChanges, + overlay: &'a OverlayedChanges, ) -> Result<( impl Iterator> + 'a, BTreeMap, impl Iterator> + 'a>, @@ -106,7 +105,6 @@ fn prepare_extrinsics_input<'a, B, H, Number, CT>( B: Backend, H: Hasher + 'a, Number: BlockNumber, - CT: ChangeTrieOverlay, { let mut children_result = BTreeMap::new(); @@ -129,18 +127,17 @@ fn prepare_extrinsics_input<'a, B, H, Number, CT>( Ok((top, children_result)) } -fn prepare_extrinsics_input_inner<'a, B, H, Number, CT>( +fn prepare_extrinsics_input_inner<'a, B, H, Number>( backend: &'a B, block: &Number, - overlay: &'a OverlayedChanges, + overlay: &'a OverlayedChanges, child_info: Option, - mut changes: impl Iterator)> + mut changes: impl Iterator ) -> Result> + 'a, String> where B: Backend, H: Hasher, Number: BlockNumber, - CT: ChangeTrieOverlay, { changes .try_fold(BTreeMap::new(), |mut map: BTreeMap<&[u8], (ExtrinsicIndex, Vec)>, (k, v)| { diff --git a/primitives/state-machine/src/changes_trie/mod.rs b/primitives/state-machine/src/changes_trie/mod.rs index a5e4321f5de9c..fd7b38c052f9e 100644 --- a/primitives/state-machine/src/changes_trie/mod.rs +++ b/primitives/state-machine/src/changes_trie/mod.rs @@ -78,7 +78,7 @@ use sp_trie::trie_types::TrieDBMut; use crate::{ StorageKey, backend::Backend, - overlayed_changes::{OverlayedChanges, ChangeTrieOverlay}, + overlayed_changes::OverlayedChanges, changes_trie::{ build::prepare_input, build_cache::{IncompleteCachedBuildData, IncompleteCacheAction}, @@ -220,19 +220,16 @@ pub fn disabled_state<'a, H, Number>() -> Option> { /// Returns Err(()) if unknown `parent_hash` has been passed. /// Returns Ok(None) if there's no data to perform computation. /// Panics if background storage returns an error OR if insert to MemoryDB fails. -pub fn build_changes_trie<'a, B: Backend, H: Hasher, Number: BlockNumber, CT: ChangeTrieOverlay>( +pub fn build_changes_trie<'a, B: Backend, H: Hasher, Number: BlockNumber>( backend: &B, state: Option<&'a State<'a, H, Number>>, - changes: &OverlayedChanges, + changes: &OverlayedChanges, parent_hash: H::Out, panic_on_storage_error: bool, ) -> Result, H::Out, CacheAction)>, ()> where H::Out: Ord + 'static + Encode, { - if !CT::CHANGE_TRIE_CAPABLE { - return Ok(None); - } /// Panics when `res.is_err() && panic`, otherwise it returns `Err(())` on an error. fn maybe_panic( res: std::result::Result, @@ -271,7 +268,7 @@ pub fn build_changes_trie<'a, B: Backend, H: Hasher, Number: BlockNumber, CT: // storage errors are considered fatal (similar to situations when runtime fetches values from storage) let (input_pairs, child_input_pairs, digest_input_blocks) = maybe_panic( - prepare_input::( + prepare_input::( backend, state.storage, config_range.clone(), diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index ffe4b528015b0..cdf92a5c8c95d 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -19,7 +19,6 @@ use crate::{ StorageKey, StorageValue, OverlayedChanges, - overlayed_changes::{ChangeTrieOverlay, Extrinsics}, backend::Backend, }; use hash_db::Hasher; @@ -114,13 +113,13 @@ pub struct Ext<'a, H, N, B> } /// Basis implementation for `Externalities` trait. -pub struct ExtInnerMut<'a, H, B, CT: ChangeTrieOverlay = Extrinsics> +pub struct ExtInnerMut<'a, H, B> where H: Hasher, B: Backend, { /// The overlayed changes to write to. - pub overlay: &'a mut OverlayedChanges, + pub overlay: &'a mut OverlayedChanges, /// The storage backend to read from. pub backend: &'a B, /// Pseudo-unique id used for tracing. @@ -130,13 +129,13 @@ pub struct ExtInnerMut<'a, H, B, CT: ChangeTrieOverlay = Extrinsics> } /// Basis implementation for `Externalities` trait. -pub struct ExtInner<'a, H, B, CT: ChangeTrieOverlay = Extrinsics> +pub struct ExtInner<'a, H, B> where H: Hasher, B: Backend, { /// The overlayed changes to write to. - pub overlay: &'a OverlayedChanges, + pub overlay: &'a OverlayedChanges, /// The storage backend to read from. pub backend: &'a B, /// Pseudo-unique id used for tracing. @@ -217,12 +216,11 @@ where } } -impl<'a, H, B, CT> ExtInner<'a, H, B, CT> +impl<'a, H, B> ExtInner<'a, H, B> where H: Hasher, H::Out: Ord + 'static + codec::Codec, B: Backend, - CT: ChangeTrieOverlay, { pub(crate) fn storage(&self, key: &[u8]) -> Option { let _guard = guard(); @@ -385,14 +383,13 @@ where } } -impl<'a, H, B, CT> ExtInnerMut<'a, H, B, CT> +impl<'a, H, B> ExtInnerMut<'a, H, B> where H: Hasher, H::Out: Ord + 'static + codec::Codec, B: Backend, - CT: ChangeTrieOverlay, { - pub(crate) fn inner(&'a self) -> ExtInner<'a, H, B, CT> { + pub(crate) fn inner(&'a self) -> ExtInner<'a, H, B> { ExtInner { overlay: self.overlay, backend: self.backend, diff --git a/primitives/state-machine/src/overlayed_changes/changeset.rs b/primitives/state-machine/src/overlayed_changes/changeset.rs index 771bda79e6640..50cfb4efe6e92 100644 --- a/primitives/state-machine/src/overlayed_changes/changeset.rs +++ b/primitives/state-machine/src/overlayed_changes/changeset.rs @@ -17,7 +17,7 @@ //! Houses the code that implements the transactional overlay storage. -use super::{StorageKey, StorageValue, ChangeTrieOverlay}; +use super::{StorageKey, StorageValue, Extrinsics}; #[cfg(feature = "std")] use std::collections::HashSet as Set; @@ -34,7 +34,7 @@ const PROOF_OVERLAY_NON_EMPTY: &str = "\ as the last transaction is removed; qed"; type DirtyKeysSets = SmallVec<[Set; 5]>; -type Transactions = SmallVec<[InnerValue; 5]>; +type Transactions = SmallVec<[InnerValue; 5]>; /// Error returned when trying to commit or rollback while no transaction is open or /// when the runtime is trying to close a transaction started by the client. @@ -63,28 +63,28 @@ pub enum ExecutionMode { #[derive(Debug, Default, Clone)] #[cfg_attr(test, derive(PartialEq))] -struct InnerValue { +struct InnerValue { /// Current value. None if value has been deleted. value: Option, /// The set of extrinsic indices where the values has been changed. /// Is filled only if runtime has announced changes trie support. - extrinsics: CT, + extrinsics: Extrinsics, } /// An overlay that contains all versions of a value for a specific key. #[derive(Debug, Default, Clone)] #[cfg_attr(test, derive(PartialEq))] -pub struct OverlayedValue { +pub struct OverlayedValue { /// The individual versions of that value. /// One entry per transactions during that the value was actually written. - transactions: Transactions, + transactions: Transactions, } /// Holds a set of changes with the ability modify them using nested transactions. #[derive(Debug, Default, Clone)] -pub struct OverlayedChangeSet { +pub struct OverlayedChangeSet { /// Stores the changes that this overlay constitutes. - changes: BTreeMap>, + changes: BTreeMap, /// Stores which keys are dirty per transaction. Needed in order to determine which /// values to merge into the parent transaction on commit. The length of this vector /// therefore determines how many nested transactions are currently open (depth). @@ -103,7 +103,7 @@ impl Default for ExecutionMode { } } -impl OverlayedValue { +impl OverlayedValue { /// The value as seen by the current transaction. pub fn value(&self) -> Option<&StorageValue> { self.transactions.last().expect(PROOF_OVERLAY_NON_EMPTY).value.as_ref() @@ -122,12 +122,12 @@ impl OverlayedValue { } /// Remove the last version and return it. - fn pop_transaction(&mut self) -> InnerValue { + fn pop_transaction(&mut self) -> InnerValue { self.transactions.pop().expect(PROOF_OVERLAY_NON_EMPTY) } /// Mutable reference to the set which holds the indices for the **current transaction only**. - fn transaction_extrinsics_mut(&mut self) -> &mut CT { + fn transaction_extrinsics_mut(&mut self) -> &mut Extrinsics { &mut self.transactions.last_mut().expect(PROOF_OVERLAY_NON_EMPTY).extrinsics } @@ -150,10 +150,8 @@ impl OverlayedValue { *self.value_mut() = value; } - if CT::CHANGE_TRIE_CAPABLE { - if let Some(extrinsic) = at_extrinsic { - self.transaction_extrinsics_mut().insert(extrinsic); - } + if let Some(extrinsic) = at_extrinsic { + self.transaction_extrinsics_mut().insert(extrinsic); } } } @@ -166,7 +164,7 @@ fn insert_dirty(set: &mut DirtyKeysSets, key: StorageKey) -> bool { set.last_mut().map(|dk| dk.insert(key)).unwrap_or_default() } -impl OverlayedChangeSet { +impl OverlayedChangeSet { /// Create a new changeset at the same transaction state but without any contents. /// /// This changeset might be created when there are already open transactions. @@ -187,7 +185,7 @@ impl OverlayedChangeSet { } /// Get an optional reference to the value stored for the specified key. - pub fn get(&self, key: &[u8]) -> Option<&OverlayedValue> { + pub fn get(&self, key: &[u8]) -> Option<&OverlayedValue> { self.changes.get(key) } @@ -237,7 +235,7 @@ impl OverlayedChangeSet { /// Can be rolled back or committed when called inside a transaction. pub fn clear_where( &mut self, - predicate: impl Fn(&[u8], &OverlayedValue) -> bool, + predicate: impl Fn(&[u8], &OverlayedValue) -> bool, at_extrinsic: Option, ) { for (key, val) in self.changes.iter_mut().filter(|(k, v)| predicate(k, v)) { @@ -246,12 +244,12 @@ impl OverlayedChangeSet { } /// Get a list of all changes as seen by current transaction. - pub fn changes(&self) -> impl Iterator)> { + pub fn changes(&self) -> impl Iterator { self.changes.iter() } /// Get the change that is next to the supplied key. - pub fn next_change(&self, key: &[u8]) -> Option<(&[u8], &OverlayedValue)> { + pub fn next_change(&self, key: &[u8]) -> Option<(&[u8], &OverlayedValue)> { use sp_std::ops::Bound; let range = (Bound::Excluded(key), Bound::Unbounded); self.changes.range::<[u8], _>(range).next().map(|(k, v)| (&k[..], v)) @@ -374,9 +372,7 @@ impl OverlayedChangeSet { if has_predecessor { let dropped_tx = overlayed.pop_transaction(); *overlayed.value_mut() = dropped_tx.value; - if CT::CHANGE_TRIE_CAPABLE { - overlayed.transaction_extrinsics_mut().merge(dropped_tx.extrinsics); - } + overlayed.transaction_extrinsics_mut().merge(dropped_tx.extrinsics); } } } @@ -393,9 +389,7 @@ impl OverlayedChangeSet { mod test { use super::*; use pretty_assertions::assert_eq; - use crate::overlayed_changes::Extrinsics; - type OverlayedChangeSet = super::OverlayedChangeSet; type Changes<'a> = Vec<(&'a [u8], (Option<&'a [u8]>, Vec))>; type Drained<'a> = Vec<(&'a [u8], Option<&'a [u8]>)>; diff --git a/primitives/state-machine/src/overlayed_changes/mod.rs b/primitives/state-machine/src/overlayed_changes/mod.rs index 7ba86159a356e..19d8730b4bf21 100644 --- a/primitives/state-machine/src/overlayed_changes/mod.rs +++ b/primitives/state-machine/src/overlayed_changes/mod.rs @@ -48,7 +48,6 @@ use crate::DefaultError; pub use self::changeset::{OverlayedValue, NoOpenTransaction, AlreadyInRuntime, NotInRuntime}; - /// Changes that are made outside of extrinsics are marked with this index; pub const NO_EXTRINSIC_INDEX: u32 = 0xffffffff; @@ -64,46 +63,24 @@ pub type StorageCollection = Vec<(StorageKey, Option)>; /// In memory arrays of storage values for multiple child tries. pub type ChildStorageCollection = Vec<(StorageKey, StorageCollection)>; -/// Technical trait that allows to disable child trie -/// footprint when not needed. -pub trait ChangeTrieOverlay: Default { - /// If define to false we will skip change trie specifics. - const CHANGE_TRIE_CAPABLE: bool = true; - /// Extracts extrinsics into a `BTreeSets`. - fn flush_content(&self, dest: &mut BTreeSet); - /// Add an extrinsics. - fn insert(&mut self, ext: u32); - /// Merge two extrinsics sets. - fn merge(&mut self, other: Self); -} - -/// Default implementation of ChangeTrieOverlay. -#[derive(Debug, Default, Eq, PartialEq)] +/// Keep trace of extrinsics index for a modified value. +#[derive(Debug, Default, Eq, PartialEq, Clone)] pub struct Extrinsics(Vec); -/// Skip registering extrinsics when using state machine. -#[derive(Default)] -pub struct NoExtrinsics; - -impl ChangeTrieOverlay for NoExtrinsics { - const CHANGE_TRIE_CAPABLE: bool = false; - fn flush_content(&self, _dest: &mut BTreeSet) { - } - fn insert(&mut self, _ext: u32) { - } - fn merge(&mut self, _other: Self) { - } -} - -impl ChangeTrieOverlay for Extrinsics { +impl Extrinsics { + /// Extracts extrinsics into a `BTreeSets`. fn flush_content(&self, dest: &mut BTreeSet) { dest.extend(self.0.iter()) } + + /// Add an extrinsics. fn insert(&mut self, ext: u32) { if Some(&ext) != self.0.last() { self.0.push(ext); } } + + ///Merge two extrinsics sets. fn merge(&mut self, other: Self) { self.0.extend(other.0.into_iter()); } @@ -113,11 +90,11 @@ impl ChangeTrieOverlay for Extrinsics { /// /// It allows changes to be modified using nestable transactions. #[derive(Debug, Default, Clone)] -pub struct OverlayedChanges { +pub struct OverlayedChanges { /// Top level storage changes. - top: OverlayedChangeSet, + top: OverlayedChangeSet, /// Child storage changes. The map key is the child storage key without the common prefix. - children: Map, ChildInfo)>, + children: Map, /// True if extrinsics stats must be collected. collect_extrinsics: bool, /// Collect statistic on this execution. @@ -222,7 +199,7 @@ impl Default for StorageChanges } } -impl OverlayedChanges { +impl OverlayedChanges { /// Whether no changes are contained in the top nor in any of the child changes. pub fn is_empty(&self) -> bool { self.top.is_empty() && self.children.is_empty() @@ -459,18 +436,18 @@ impl OverlayedChanges { /// Get an iterator over all child changes as seen by the current transaction. pub fn children(&self) - -> impl Iterator)>, &ChildInfo)> { + -> impl Iterator, &ChildInfo)> { self.children.iter().map(|(_, v)| (v.0.changes(), &v.1)) } /// Get an iterator over all top changes as been by the current transaction. - pub fn changes(&self) -> impl Iterator)> { + pub fn changes(&self) -> impl Iterator { self.top.changes() } /// Get an optional iterator over all child changes stored under the supplied key. pub fn child_changes(&self, key: &[u8]) - -> Option<(impl Iterator)>, &ChildInfo)> { + -> Option<(impl Iterator, &ChildInfo)> { self.children.get(key).map(|(overlay, info)| (overlay.changes(), info)) } @@ -547,9 +524,6 @@ impl OverlayedChanges { /// Changes that are made outside of extrinsics, are marked with /// `NO_EXTRINSIC_INDEX` index. fn extrinsic_index(&self) -> Option { - if !CT::CHANGE_TRIE_CAPABLE { - return None; - } match self.collect_extrinsics { true => Some( self.storage(EXTRINSIC_INDEX) @@ -622,10 +596,7 @@ impl OverlayedChanges { panic_on_storage_error: bool, cache: &mut StorageTransactionCache, ) -> Result, ()> where H::Out: Ord + Encode + 'static { - if !CT::CHANGE_TRIE_CAPABLE { - return Ok(None); - } - build_changes_trie::<_, H, N, CT>( + build_changes_trie::<_, H, N>( backend, changes_trie_state, self, @@ -641,7 +612,7 @@ impl OverlayedChanges { /// Returns the next (in lexicographic order) storage key in the overlayed alongside its value. /// If no value is next then `None` is returned. - pub fn next_storage_key_change(&self, key: &[u8]) -> Option<(&[u8], &OverlayedValue)> { + pub fn next_storage_key_change(&self, key: &[u8]) -> Option<(&[u8], &OverlayedValue)> { self.top.next_change(key) } @@ -651,7 +622,7 @@ impl OverlayedChanges { &self, storage_key: &[u8], key: &[u8] - ) -> Option<(&[u8], &OverlayedValue)> { + ) -> Option<(&[u8], &OverlayedValue)> { self.children .get(storage_key) .and_then(|(overlay, _)| @@ -692,10 +663,8 @@ mod tests { use super::*; use std::collections::BTreeMap; - type OverlayedChanges = super::OverlayedChanges; - fn assert_extrinsics( - overlay: &OverlayedChangeSet, + overlay: &OverlayedChangeSet, key: impl AsRef<[u8]>, expected: Vec, ) { diff --git a/primitives/state-machine/src/witness_ext.rs b/primitives/state-machine/src/witness_ext.rs index 71cf563ac2233..0c5f8ee14c678 100644 --- a/primitives/state-machine/src/witness_ext.rs +++ b/primitives/state-machine/src/witness_ext.rs @@ -26,13 +26,13 @@ use sp_std::vec::Vec; use sp_trie::MemoryDB; use crate::trie_backend::TrieBackend; use crate::ext::{ExtInner, ExtInnerMut}; -use crate::overlayed_changes::{OverlayedChanges, NoExtrinsics}; +use crate::overlayed_changes::OverlayedChanges; use crate::{StorageValue, StorageKey}; /// The backend runnig on a trie proof. pub struct WitnessExt { /// The overlayed changes to write to. - pub overlay: OverlayedChanges, + pub overlay: OverlayedChanges, /// The storage backend to read from. pub backend: TrieBackend, H>, } @@ -46,12 +46,12 @@ impl WitnessExt pub fn new(db: MemoryDB, root: H::Out) -> Self { WitnessExt { backend: TrieBackend::new(db, root), - overlay: OverlayedChanges::::default(), + overlay: OverlayedChanges::default(), } } /// Access methods for `ExtInnerMut`. - fn ext_inner_mut(&mut self) -> ExtInnerMut, H>, NoExtrinsics> { + fn ext_inner_mut(&mut self) -> ExtInnerMut, H>> { ExtInnerMut { overlay: &mut self.overlay, backend: &self.backend, @@ -61,7 +61,7 @@ impl WitnessExt } /// Access methods for `ExtInner`. - fn ext_inner(&self) -> ExtInner, H>, NoExtrinsics> { + fn ext_inner(&self) -> ExtInner, H>> { ExtInner { overlay: &self.overlay, backend: &self.backend, From 4f05206c2ec78b224a87ea30cebb08aca9dfe0d7 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 25 Aug 2020 09:25:51 +0200 Subject: [PATCH 08/21] modified function --- primitives/state-machine/src/witness_ext.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/primitives/state-machine/src/witness_ext.rs b/primitives/state-machine/src/witness_ext.rs index 0c5f8ee14c678..152825e98e4fa 100644 --- a/primitives/state-machine/src/witness_ext.rs +++ b/primitives/state-machine/src/witness_ext.rs @@ -69,6 +69,12 @@ impl WitnessExt _phantom: Default::default(), } } + + /// Retrieve the value for the given key only if modified. + pub fn modified(&self, key: &[u8]) -> Option>> { + self.overlay.storage(key) + .map(|x| x.map(|x| x.to_vec())) + } } impl Externalities for WitnessExt From 147ef68913b22d42047aa182b86b5cfd82c4d298 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 25 Aug 2020 17:57:10 +0200 Subject: [PATCH 09/21] Remove witness_ext --- primitives/externalities/Cargo.toml | 7 +- primitives/externalities/src/lib.rs | 2 - primitives/state-machine/src/ext.rs | 172 ++++++++----- primitives/state-machine/src/lib.rs | 2 +- primitives/state-machine/src/witness_ext.rs | 256 -------------------- test-utils/runtime/src/system.rs | 12 +- 6 files changed, 116 insertions(+), 335 deletions(-) delete mode 100644 primitives/state-machine/src/witness_ext.rs diff --git a/primitives/externalities/Cargo.toml b/primitives/externalities/Cargo.toml index 4e6db3fc2bfb2..10f060a059594 100644 --- a/primitives/externalities/Cargo.toml +++ b/primitives/externalities/Cargo.toml @@ -14,16 +14,15 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] sp-storage = { version = "2.0.0-rc6", path = "../storage", default-features = false } -sp-std = { version = "2.0.0-rc6", path = "../std", default-features = false } -environmental = { version = "1.1.1", optional = true } +sp-std = { version = "2.0.0-rc5", path = "../std", default-features = false } +environmental = { version = "1.1.1", default-features = false } codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } [features] default = ["std"] std = [ "codec/std", + "environmental/std", "sp-std/std", "sp-storage/std", - - "environmental", ] diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index e0df7ea8cbb67..7bd1a00c9bc4c 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -31,12 +31,10 @@ use sp_std::boxed::Box; use sp_storage::{ChildInfo, TrackedStorageKey}; -#[cfg(feature = "std")] pub use scope_limited::{set_and_run_with_externalities, with_externalities}; pub use extensions::{Extension, Extensions, ExtensionStore}; mod extensions; -#[cfg(feature = "std")] mod scope_limited; /// Externalities error. diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 390e2dc9078ed..c1d7bfdd9cf40 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -27,7 +27,8 @@ use sp_core::{ hexdisplay::HexDisplay, }; use sp_trie::{trie_types::Layout, empty_child_trie_root}; -use sp_externalities::{Externalities, Extensions, Extension}; +use sp_externalities::{Externalities, Extensions, Extension, + ExtensionStore, Error as ExtensionError}; use codec::{Decode, Encode, EncodeAppend}; use sp_std::{fmt, any::{Any, TypeId}, vec::Vec, vec, boxed::Box}; @@ -99,7 +100,7 @@ pub struct Ext<'a, H, N, B> N: crate::changes_trie::BlockNumber, { /// Inner Ext (change overlay and backend). - inner: ExtInnerMut<'a, H, B>, + inner: ExtInner<'a, H, B>, /// The overlayed changes destined for the Offchain DB. offchain_overlay: &'a mut OffchainOverlayedChanges, /// The cache for the storage transactions. @@ -113,38 +114,42 @@ pub struct Ext<'a, H, N, B> } /// Basis implementation for `Externalities` trait. -pub struct ExtInnerMut<'a, H, B> +pub struct ExtInner<'a, H, B> where H: Hasher, B: Backend, { /// The overlayed changes to write to. - pub overlay: &'a mut OverlayedChanges, + overlay: &'a mut OverlayedChanges, /// The storage backend to read from. - pub backend: &'a B, + backend: &'a B, /// Pseudo-unique id used for tracing. - pub id: u16, + id: u16, /// Dummy usage of H arg. - pub _phantom: sp_std::marker::PhantomData, + _phantom: sp_std::marker::PhantomData, } -/// Basis implementation for `Externalities` trait. -pub struct ExtInner<'a, H, B> +impl<'a, H, B> ExtInner<'a, H, B> where H: Hasher, B: Backend, { - /// The overlayed changes to write to. - pub overlay: &'a OverlayedChanges, - /// The storage backend to read from. - pub backend: &'a B, - /// Pseudo-unique id used for tracing. - pub id: u16, - /// Dummy usage of H arg. - pub _phantom: sp_std::marker::PhantomData, + /// Create a new `ExtInner`. Warning, this + /// do not init its inner id with a random + /// value. + pub fn new( + overlay: &'a mut OverlayedChanges, + backend: &'a B, + ) -> Self { + ExtInner { + overlay, + backend, + id: 0, + _phantom: Default::default(), + } + } } - - + #[cfg(feature = "std")] impl<'a, H, N, B> Ext<'a, H, N, B> where @@ -163,7 +168,7 @@ where extensions: Option<&'a mut Extensions>, ) -> Self { Self { - inner: ExtInnerMut { + inner: ExtInner { overlay, backend, id: rand::random(), @@ -216,13 +221,13 @@ where } } -impl<'a, H, B> ExtInner<'a, H, B> +impl<'a, H, B> Externalities for ExtInner<'a, H, B> where H: Hasher, H::Out: Ord + 'static + codec::Codec, B: Backend, { - pub(crate) fn storage(&self, key: &[u8]) -> Option { + fn storage(&self, key: &[u8]) -> Option { let _guard = guard(); let result = self.overlay.storage(key).map(|x| x.map(|x| x.to_vec())).unwrap_or_else(|| self.backend.storage(key).expect(EXT_NOT_ALLOWED_TO_FAIL)); @@ -234,7 +239,7 @@ where result } - pub(crate) fn storage_hash(&self, key: &[u8]) -> Option> { + fn storage_hash(&self, key: &[u8]) -> Option> { let _guard = guard(); let result = self.overlay .storage(key) @@ -249,7 +254,7 @@ where result.map(|r| r.encode()) } - pub(crate) fn child_storage( + fn child_storage( &self, child_info: &ChildInfo, key: &[u8], @@ -273,7 +278,7 @@ where result } - pub(crate) fn child_storage_hash( + fn child_storage_hash( &self, child_info: &ChildInfo, key: &[u8], @@ -297,7 +302,7 @@ where result.map(|r| r.encode()) } - pub(crate) fn exists_storage(&self, key: &[u8]) -> bool { + fn exists_storage(&self, key: &[u8]) -> bool { let _guard = guard(); let result = match self.overlay.storage(key) { Some(x) => x.is_some(), @@ -313,7 +318,7 @@ where result } - pub(crate) fn exists_child_storage( + fn exists_child_storage( &self, child_info: &ChildInfo, key: &[u8], @@ -336,7 +341,7 @@ where result } - pub(crate) fn next_storage_key(&self, key: &[u8]) -> Option { + fn next_storage_key(&self, key: &[u8]) -> Option { let next_backend_key = self.backend.next_storage_key(key).expect(EXT_NOT_ALLOWED_TO_FAIL); let next_overlay_key_change = self.overlay.next_storage_key_change(key); @@ -351,7 +356,7 @@ where } } - pub(crate) fn next_child_storage_key( + fn next_child_storage_key( &self, child_info: &ChildInfo, key: &[u8], @@ -378,27 +383,11 @@ where } } - pub(crate) fn read_write_count(&self) -> (u32, u32, u32, u32) { + fn read_write_count(&self) -> (u32, u32, u32, u32) { self.backend.read_write_count() } -} -impl<'a, H, B> ExtInnerMut<'a, H, B> -where - H: Hasher, - H::Out: Ord + 'static + codec::Codec, - B: Backend, -{ - pub(crate) fn inner(&'a self) -> ExtInner<'a, H, B> { - ExtInner { - overlay: self.overlay, - backend: self.backend, - id: self.id, - _phantom: Default::default(), - } - } - - pub(crate) fn place_storage(&mut self, key: StorageKey, value: Option) { + fn place_storage(&mut self, key: StorageKey, value: Option) { trace!(target: "state", "{:04x}: Put {}={:?}", self.id, HexDisplay::from(&key), @@ -413,7 +402,7 @@ where self.overlay.set_storage(key, value); } - pub(crate) fn place_child_storage( + fn place_child_storage( &mut self, child_info: &ChildInfo, key: StorageKey, @@ -430,7 +419,7 @@ where self.overlay.set_child_storage(child_info, key, value); } - pub(crate) fn kill_child_storage( + fn kill_child_storage( &mut self, child_info: &ChildInfo, ) { @@ -446,7 +435,7 @@ where }); } - pub(crate) fn clear_prefix(&mut self, prefix: &[u8]) { + fn clear_prefix(&mut self, prefix: &[u8]) { trace!(target: "state", "{:04x}: ClearPrefix {}", self.id, HexDisplay::from(&prefix), @@ -463,7 +452,7 @@ where }); } - pub(crate) fn clear_child_prefix( + fn clear_child_prefix( &mut self, child_info: &ChildInfo, prefix: &[u8], @@ -481,7 +470,7 @@ where }); } - pub(crate) fn storage_append( + fn storage_append( &mut self, key: Vec, value: Vec, @@ -502,14 +491,14 @@ where StorageAppend::new(current_value).append(value); } - pub(crate) fn storage_root(&mut self) -> Vec { + fn storage_root(&mut self) -> Vec { let _guard = guard(); let root = self.overlay.storage_root_no_cache(self.backend); trace!(target: "state", "{:04x}: Root {}", self.id, HexDisplay::from(&root.as_ref())); root.encode() } - pub(crate) fn child_storage_root( + fn child_storage_root( &mut self, child_info: &ChildInfo, ) -> Vec { @@ -545,7 +534,7 @@ where root } else { // empty overlay - let root = self.inner() + let root = self .storage(prefixed_storage_key.as_slice()) .and_then(|k| Decode::decode(&mut &k[..]).ok()) .unwrap_or_else( @@ -560,21 +549,70 @@ where } } - pub(crate) fn storage_start_transaction(&mut self) { + fn storage_start_transaction(&mut self) { self.overlay.start_transaction() } - pub(crate) fn storage_rollback_transaction(&mut self) -> Result<(), ()> { + fn storage_rollback_transaction(&mut self) -> Result<(), ()> { self.overlay.rollback_transaction().map_err(|_| ()) } - pub(crate) fn storage_commit_transaction(&mut self) -> Result<(), ()> { + fn storage_commit_transaction(&mut self) -> Result<(), ()> { self.overlay.commit_transaction().map_err(|_| ()) } - pub(crate) fn reset_read_write_count(&mut self) { + fn reset_read_write_count(&mut self) { self.backend.reset_read_write_count() } + + fn chain_id(&self) -> u64 { + 42 + } + + fn set_offchain_storage(&mut self, _key: &[u8], _value: Option<&[u8]>) { + } + + fn storage_changes_root(&mut self, _parent_hash: &[u8]) -> Result>, ()> { + unimplemented!("Unsupported") + } + + fn wipe(&mut self) { + unimplemented!("Unsupported") + } + + fn commit(&mut self) { + unimplemented!("Unsupported") + } + + fn set_whitelist(&mut self, _new: Vec>) { + unimplemented!("Unsupported") + } +} + +impl<'a, H, B> ExtensionStore for ExtInner<'a, H, B> +where + H: Hasher, + H::Out: Ord + 'static + codec::Codec, + B: Backend, +{ + fn extension_by_type_id(&mut self, _type_id: TypeId) -> Option<&mut dyn Any> { + None + } + + fn register_extension_with_type_id( + &mut self, + _type_id: TypeId, + _extension: Box, + ) -> Result<(), ExtensionError> { + Err(ExtensionError::ExtensionsAreNotSupported) + } + + fn deregister_extension_by_type_id( + &mut self, + _type_id: TypeId, + ) -> Result<(), ExtensionError> { + Err(ExtensionError::ExtensionsAreNotSupported) + } } #[cfg(feature = "std")] @@ -595,11 +633,11 @@ where } fn storage(&self, key: &[u8]) -> Option { - self.inner.inner().storage(key) + self.inner.storage(key) } fn storage_hash(&self, key: &[u8]) -> Option> { - self.inner.inner().storage_hash(key) + self.inner.storage_hash(key) } fn child_storage( @@ -607,7 +645,7 @@ where child_info: &ChildInfo, key: &[u8], ) -> Option { - self.inner.inner().child_storage(child_info, key) + self.inner.child_storage(child_info, key) } fn child_storage_hash( @@ -615,11 +653,11 @@ where child_info: &ChildInfo, key: &[u8], ) -> Option> { - self.inner.inner().child_storage_hash(child_info, key) + self.inner.child_storage_hash(child_info, key) } fn exists_storage(&self, key: &[u8]) -> bool { - self.inner.inner().exists_storage(key) + self.inner.exists_storage(key) } fn exists_child_storage( @@ -627,11 +665,11 @@ where child_info: &ChildInfo, key: &[u8], ) -> bool { - self.inner.inner().exists_child_storage(child_info, key) + self.inner.exists_child_storage(child_info, key) } fn next_storage_key(&self, key: &[u8]) -> Option { - self.inner.inner().next_storage_key(key) + self.inner.next_storage_key(key) } fn next_child_storage_key( @@ -639,7 +677,7 @@ where child_info: &ChildInfo, key: &[u8], ) -> Option { - self.inner.inner().next_child_storage_key(child_info, key) + self.inner.next_child_storage_key(child_info, key) } fn place_storage(&mut self, key: StorageKey, value: Option) { diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 15027bde11489..146fc085c2088 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -21,7 +21,6 @@ #![cfg_attr(not(feature = "std"), no_std)] pub mod backend; -pub mod witness_ext; #[cfg(feature = "std")] mod in_memory_backend; #[cfg(feature = "std")] @@ -127,6 +126,7 @@ pub use crate::trie_backend_essence::{TrieBackendStorage, Storage}; pub use crate::trie_backend::TrieBackend; pub use crate::stats::{UsageInfo, UsageUnit, StateMachineStats}; pub use error::{Error, ExecutionError}; +pub use crate::ext::ExtInner; #[cfg(feature = "std")] mod std_reexport { diff --git a/primitives/state-machine/src/witness_ext.rs b/primitives/state-machine/src/witness_ext.rs deleted file mode 100644 index 152825e98e4fa..0000000000000 --- a/primitives/state-machine/src/witness_ext.rs +++ /dev/null @@ -1,256 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Substrate backend runing on a trie proof, no_std compatible. - -use hash_db::Hasher; -use sp_externalities::{Externalities, ExtensionStore, Error, Extension}; -use sp_core::storage::{ChildInfo, TrackedStorageKey}; -use sp_std::{any::{TypeId, Any}}; -use sp_std::boxed::Box; -use sp_std::vec::Vec; -use sp_trie::MemoryDB; -use crate::trie_backend::TrieBackend; -use crate::ext::{ExtInner, ExtInnerMut}; -use crate::overlayed_changes::OverlayedChanges; -use crate::{StorageValue, StorageKey}; - -/// The backend runnig on a trie proof. -pub struct WitnessExt { - /// The overlayed changes to write to. - pub overlay: OverlayedChanges, - /// The storage backend to read from. - pub backend: TrieBackend, H>, -} - -impl WitnessExt - where - H: Hasher, - H::Out: Ord + 'static + codec::Codec, -{ - /// Create a new backend. - pub fn new(db: MemoryDB, root: H::Out) -> Self { - WitnessExt { - backend: TrieBackend::new(db, root), - overlay: OverlayedChanges::default(), - } - } - - /// Access methods for `ExtInnerMut`. - fn ext_inner_mut(&mut self) -> ExtInnerMut, H>> { - ExtInnerMut { - overlay: &mut self.overlay, - backend: &self.backend, - id: 0, - _phantom: Default::default(), - } - } - - /// Access methods for `ExtInner`. - fn ext_inner(&self) -> ExtInner, H>> { - ExtInner { - overlay: &self.overlay, - backend: &self.backend, - id: 0, - _phantom: Default::default(), - } - } - - /// Retrieve the value for the given key only if modified. - pub fn modified(&self, key: &[u8]) -> Option>> { - self.overlay.storage(key) - .map(|x| x.map(|x| x.to_vec())) - } -} - -impl Externalities for WitnessExt - where - H: Hasher, - H::Out: Ord + 'static + codec::Codec, -{ - fn set_offchain_storage(&mut self, _key: &[u8], _value: Option<&[u8]>) { - unimplemented!("Unsupported"); - } - - fn storage(&self, key: &[u8]) -> Option { - self.ext_inner().storage(key) - } - - fn storage_hash(&self, key: &[u8]) -> Option> { - self.ext_inner().storage_hash(key) - } - - fn child_storage( - &self, - child_info: &ChildInfo, - key: &[u8], - ) -> Option { - self.ext_inner().child_storage(child_info, key) - } - - fn child_storage_hash( - &self, - child_info: &ChildInfo, - key: &[u8], - ) -> Option> { - self.ext_inner().child_storage_hash(child_info, key) - } - - fn exists_storage(&self, key: &[u8]) -> bool { - self.ext_inner().exists_storage(key) - } - - fn exists_child_storage( - &self, - child_info: &ChildInfo, - key: &[u8], - ) -> bool { - self.ext_inner().exists_child_storage(child_info, key) - } - - fn next_storage_key(&self, key: &[u8]) -> Option { - self.ext_inner().next_storage_key(key) - } - - fn next_child_storage_key( - &self, - child_info: &ChildInfo, - key: &[u8], - ) -> Option { - self.ext_inner().next_child_storage_key(child_info, key) - } - - fn place_storage(&mut self, key: StorageKey, value: Option) { - self.ext_inner_mut().place_storage(key, value) - } - - fn place_child_storage( - &mut self, - child_info: &ChildInfo, - key: StorageKey, - value: Option, - ) { - self.ext_inner_mut().place_child_storage(child_info, key, value) - } - - fn kill_child_storage( - &mut self, - child_info: &ChildInfo, - ) { - self.ext_inner_mut().kill_child_storage(child_info) - } - - fn clear_prefix(&mut self, prefix: &[u8]) { - self.ext_inner_mut().clear_prefix(prefix) - } - - fn clear_child_prefix( - &mut self, - child_info: &ChildInfo, - prefix: &[u8], - ) { - self.ext_inner_mut().clear_child_prefix(child_info, prefix) - } - - fn storage_append( - &mut self, - key: Vec, - value: Vec, - ) { - self.ext_inner_mut().storage_append(key, value) - } - - fn chain_id(&self) -> u64 { - 42 - } - - fn storage_root(&mut self) -> Vec { - self.ext_inner_mut().storage_root() - } - - fn child_storage_root( - &mut self, - child_info: &ChildInfo, - ) -> Vec { - self.ext_inner_mut().child_storage_root(child_info) - } - - fn storage_changes_root(&mut self, _parent_hash: &[u8]) -> Result>, ()> { - Ok(None) - } - - fn storage_start_transaction(&mut self) { - self.ext_inner_mut().storage_start_transaction() - } - - fn storage_rollback_transaction(&mut self) -> Result<(), ()> { - self.ext_inner_mut().storage_rollback_transaction() - } - - fn storage_commit_transaction(&mut self) -> Result<(), ()> { - self.ext_inner_mut().storage_commit_transaction() - } - - fn wipe(&mut self) { - unimplemented!("Unsupported"); - } - - fn commit(&mut self) { - unimplemented!("Unsupported"); - } - - fn read_write_count(&self) -> (u32, u32, u32, u32) { - self.ext_inner().read_write_count() - } - - fn reset_read_write_count(&mut self) { - self.ext_inner_mut().reset_read_write_count() - } - - fn get_whitelist(&self) -> Vec { - unimplemented!("Unsupported"); - } - - fn set_whitelist(&mut self, _new: Vec) { - unimplemented!("Unsupported"); - } -} - -impl ExtensionStore for WitnessExt -where - H: Hasher, - H::Out: Ord + 'static + codec::Codec, -{ - fn extension_by_type_id(&mut self, _type_id: TypeId) -> Option<&mut dyn Any> { - None - } - - fn register_extension_with_type_id( - &mut self, - _type_id: TypeId, - _extension: Box, - ) -> Result<(), Error> { - Err(Error::ExtensionsAreNotSupported) - } - - fn deregister_extension_by_type_id( - &mut self, - _type_id: TypeId, - ) -> Result<(), Error> { - Err(Error::ExtensionsAreNotSupported) - } -} diff --git a/test-utils/runtime/src/system.rs b/test-utils/runtime/src/system.rs index bb924e5fef6c5..1cf4017f2a92c 100644 --- a/test-utils/runtime/src/system.rs +++ b/test-utils/runtime/src/system.rs @@ -306,14 +306,16 @@ fn execute_storage_change(key: &[u8], value: Option<&[u8]>) -> ApplyExtrinsicRes } fn execute_witness_backend() -> ApplyExtrinsicResult { - use sp_state_machine::witness_ext::WitnessExt; use sp_externalities::Externalities; - let mut backend = WitnessExt::::new( - Default::default(), + let db = sp_trie::MemoryDB::::default(); + let backend = sp_state_machine::TrieBackend::<_, crate::Hashing>::new( + db, Default::default(), ); - backend.place_storage(vec![0], Some(vec![1])); - assert!(backend.storage(&[0]).is_some()); + let mut overlay = sp_state_machine::OverlayedChanges::default(); + let mut ext = sp_state_machine::ExtInner::new(&mut overlay, &backend); + ext.place_storage(vec![0], Some(vec![1])); + assert!(ext.storage(&[0]).is_some()); Ok(Ok(())) } From d067b0c234733322b39308dd06f2c7993aec786f Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 25 Aug 2020 19:10:25 +0200 Subject: [PATCH 10/21] need noops changes root --- primitives/state-machine/src/ext.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index c1d7bfdd9cf40..881802634cfa7 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -573,7 +573,7 @@ where } fn storage_changes_root(&mut self, _parent_hash: &[u8]) -> Result>, ()> { - unimplemented!("Unsupported") + Ok(None) } fn wipe(&mut self) { From 0e6b642776eee4a1dd6fa33d3d42afda4250e15e Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 25 Aug 2020 19:21:25 +0200 Subject: [PATCH 11/21] update from cumulus branch --- primitives/state-machine/src/ext.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 881802634cfa7..b0f8e63fd48ba 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -584,7 +584,11 @@ where unimplemented!("Unsupported") } - fn set_whitelist(&mut self, _new: Vec>) { + fn get_whitelist(&self) -> Vec { + unimplemented!("Unsupported") + } + + fn set_whitelist(&mut self, new: Vec) { unimplemented!("Unsupported") } } From 4c454cc64dd318cbb00e8640059b76658c8d215a Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 25 Aug 2020 19:30:34 +0200 Subject: [PATCH 12/21] line break --- primitives/state-machine/src/lib.rs | 3 ++- primitives/state-machine/src/overlayed_changes/mod.rs | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 146fc085c2088..28742c8c87f04 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -194,7 +194,8 @@ mod execution { /// Strategy for executing a call into the runtime. #[derive(Copy, Clone, Eq, PartialEq, Debug)] pub enum ExecutionStrategy { - /// Execute with the native equivalent if it is compatible with the given wasm module; otherwise fall back to the wasm. + /// Execute with the native equivalent if it is compatible with the given wasm module; + /// otherwise fall back to the wasm. NativeWhenPossible, /// Use the given wasm module. AlwaysWasm, diff --git a/primitives/state-machine/src/overlayed_changes/mod.rs b/primitives/state-machine/src/overlayed_changes/mod.rs index 19d8730b4bf21..e81a2c1be11aa 100644 --- a/primitives/state-machine/src/overlayed_changes/mod.rs +++ b/primitives/state-machine/src/overlayed_changes/mod.rs @@ -461,7 +461,8 @@ impl OverlayedChanges { changes_trie_state: Option<&ChangesTrieState>, parent_hash: H::Out, mut cache: StorageTransactionCache, - ) -> Result, DefaultError> where H::Out: Ord + Encode + 'static { + ) -> Result, DefaultError> + where H::Out: Ord + Encode + 'static { self.drain_storage_changes(backend, changes_trie_state, parent_hash, &mut cache) } @@ -473,7 +474,8 @@ impl OverlayedChanges { changes_trie_state: Option<&ChangesTrieState>, parent_hash: H::Out, mut cache: &mut StorageTransactionCache, - ) -> Result, DefaultError> where H::Out: Ord + Encode + 'static { + ) -> Result, DefaultError> + where H::Out: Ord + Encode + 'static { // If the transaction does not exist, we generate it. if cache.transaction.is_none() { self.storage_root(backend, &mut cache); From d32e3fdd4c8e1cc3f5d7b8bfb8aace9a5a68b952 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 26 Aug 2020 08:55:35 +0200 Subject: [PATCH 13/21] remove warning --- primitives/state-machine/src/ext.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index b0f8e63fd48ba..bac464f7aee74 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -588,7 +588,7 @@ where unimplemented!("Unsupported") } - fn set_whitelist(&mut self, new: Vec) { + fn set_whitelist(&mut self, _new: Vec) { unimplemented!("Unsupported") } } From 753a8d7af17d1f02e052f911eda546803ebdb6ba Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 26 Aug 2020 08:59:59 +0200 Subject: [PATCH 14/21] line break --- primitives/state-machine/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 28742c8c87f04..ac0c35f71f81f 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -219,7 +219,8 @@ mod execution { /// Like `ExecutionStrategy` only it also stores a handler in case of consensus failure. #[derive(Clone)] pub enum ExecutionManager { - /// Execute with the native equivalent if it is compatible with the given wasm module; otherwise fall back to the wasm. + /// Execute with the native equivalent if it is compatible with the given wasm module; + /// otherwise fall back to the wasm. NativeWhenPossible, /// Use the given wasm module. The backend on which code is executed code could be /// trusted to provide all storage or not (i.e. the light client cannot be trusted to provide From 2d63cf7e58a1a5e677e053945687e65afa84fece Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 8 Sep 2020 09:48:55 +0200 Subject: [PATCH 15/21] From review: renamings and stats active in no std (except time). --- .../src/overlayed_changes/changeset.rs | 7 ++- .../src/overlayed_changes/mod.rs | 8 ++-- primitives/state-machine/src/stats.rs | 44 ++----------------- primitives/state-machine/src/trie_backend.rs | 3 +- 4 files changed, 11 insertions(+), 51 deletions(-) diff --git a/primitives/state-machine/src/overlayed_changes/changeset.rs b/primitives/state-machine/src/overlayed_changes/changeset.rs index 50cfb4efe6e92..5e4fd77c68563 100644 --- a/primitives/state-machine/src/overlayed_changes/changeset.rs +++ b/primitives/state-machine/src/overlayed_changes/changeset.rs @@ -24,8 +24,7 @@ use std::collections::HashSet as Set; #[cfg(not(feature = "std"))] use sp_std::collections::btree_set::BTreeSet as Set; -use sp_std::collections::btree_map::BTreeMap; -use sp_std::collections::btree_set::BTreeSet; +use sp_std::collections::{btree_map::BTreeMap, btree_set::BTreeSet}; use smallvec::SmallVec; use crate::warn; @@ -112,7 +111,7 @@ impl OverlayedValue { /// Unique list of extrinsic indices which modified the value. pub fn extrinsics(&self) -> BTreeSet { let mut set = BTreeSet::new(); - self.transactions.iter().for_each(|t| t.extrinsics.flush_content(&mut set)); + self.transactions.iter().for_each(|t| t.extrinsics.copy_extrinsics_into(&mut set)); set } @@ -372,7 +371,7 @@ impl OverlayedChangeSet { if has_predecessor { let dropped_tx = overlayed.pop_transaction(); *overlayed.value_mut() = dropped_tx.value; - overlayed.transaction_extrinsics_mut().merge(dropped_tx.extrinsics); + overlayed.transaction_extrinsics_mut().extend(dropped_tx.extrinsics); } } } diff --git a/primitives/state-machine/src/overlayed_changes/mod.rs b/primitives/state-machine/src/overlayed_changes/mod.rs index e81a2c1be11aa..a2ebde47ee544 100644 --- a/primitives/state-machine/src/overlayed_changes/mod.rs +++ b/primitives/state-machine/src/overlayed_changes/mod.rs @@ -69,7 +69,7 @@ pub struct Extrinsics(Vec); impl Extrinsics { /// Extracts extrinsics into a `BTreeSets`. - fn flush_content(&self, dest: &mut BTreeSet) { + fn copy_extrinsics_into(&self, dest: &mut BTreeSet) { dest.extend(self.0.iter()) } @@ -80,8 +80,8 @@ impl Extrinsics { } } - ///Merge two extrinsics sets. - fn merge(&mut self, other: Self) { + /// Merge two extrinsics sets. + fn extend(&mut self, other: Self) { self.0.extend(other.0.into_iter()); } } @@ -90,7 +90,7 @@ impl Extrinsics { /// /// It allows changes to be modified using nestable transactions. #[derive(Debug, Default, Clone)] -pub struct OverlayedChanges { +pub struct OverlayedChanges { /// Top level storage changes. top: OverlayedChangeSet, /// Child storage changes. The map key is the child storage key without the common prefix. diff --git a/primitives/state-machine/src/stats.rs b/primitives/state-machine/src/stats.rs index 2b5557620b8c8..f84de6a5bad07 100644 --- a/primitives/state-machine/src/stats.rs +++ b/primitives/state-machine/src/stats.rs @@ -22,7 +22,6 @@ use std::time::{Instant, Duration}; use sp_std::cell::RefCell; /// Measured count of operations and total bytes. -#[cfg(feature = "std")] #[derive(Clone, Debug, Default)] pub struct UsageUnit { /// Number of operations. @@ -31,13 +30,7 @@ pub struct UsageUnit { pub bytes: u64, } -/// Ignore stat for no_std. -#[cfg(not(feature = "std"))] -#[derive(Clone, Debug, Default)] -pub struct UsageUnit; - /// Usage statistics for state backend. -#[cfg(feature = "std")] #[derive(Clone, Debug)] pub struct UsageInfo { /// Read statistics (total). @@ -58,19 +51,16 @@ pub struct UsageInfo { /// Memory used. pub memory: usize, + #[cfg(feature = "std")] /// Moment at which current statistics has been started being collected. pub started: Instant, + #[cfg(feature = "std")] /// Timespan of the statistics. pub span: Duration, } -#[cfg(not(feature = "std"))] -#[derive(Clone, Debug, Default)] -pub struct UsageInfo; - /// Accumulated usage statistics specific to state machine /// crate. -#[cfg(feature = "std")] #[derive(Debug, Default, Clone)] pub struct StateMachineStats { /// Number of read query from runtime @@ -88,11 +78,6 @@ pub struct StateMachineStats { pub bytes_writes_overlay: RefCell, } -#[cfg(not(feature = "std"))] -#[derive(Clone, Debug, Default)] -pub struct StateMachineStats; - -#[cfg(feature = "std")] impl StateMachineStats { /// Accumulates some registered stats. pub fn add(&self, other: &StateMachineStats) { @@ -103,7 +88,6 @@ impl StateMachineStats { } } -#[cfg(feature = "std")] impl UsageInfo { /// Empty statistics. /// @@ -118,6 +102,7 @@ impl UsageInfo { cache_reads: UsageUnit::default(), modified_reads: UsageUnit::default(), memory: 0, + #[cfg(feature = "std")] started: Instant::now(), #[cfg(feature = "std")] span: Default::default(), @@ -132,19 +117,6 @@ impl UsageInfo { } } -/// Ignore stat for no_std. -#[cfg(not(feature = "std"))] -impl UsageInfo { - /// Ignore stat for no_std. - pub fn empty() -> Self { - UsageInfo - } - /// Ignore stat for no_std. - pub fn include_state_machine_states(&mut self, _count: &StateMachineStats) { - } -} - -#[cfg(feature = "std")] impl StateMachineStats { /// Tally one read modified operation, of some length. pub fn tally_read_modified(&self, data_bytes: u64) { @@ -157,13 +129,3 @@ impl StateMachineStats { *self.bytes_writes_overlay.borrow_mut() += data_bytes; } } - -#[cfg(not(feature = "std"))] -impl StateMachineStats { - /// Ignore stat for no_std. - pub fn tally_read_modified(&self, _data_bytes: u64) { - } - /// Ignore stat for no_std. - pub fn tally_write_overlay(&self, _data_bytes: u64) { - } -} diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 39d5f611e0e4a..4eaa0870baed0 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -27,8 +27,7 @@ use crate::{ StorageKey, StorageValue, Backend, trie_backend_essence::{TrieBackendEssence, TrieBackendStorage, Ephemeral}, }; -use sp_std::boxed::Box; -use sp_std::vec::Vec; +use sp_std::{boxed::Box, vec::Vec}; /// Patricia trie-based backend. Transaction type is an overlay of changes to commit. pub struct TrieBackend, H: Hasher> { From 9b3146d4201f5ebaa719b8d564b861c5417f25d6 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 8 Sep 2020 12:43:40 +0200 Subject: [PATCH 16/21] include cache, exclude change trie cache with individual temporary bad looking no_std check --- primitives/state-machine/src/ext.rs | 312 +++++++++--------- primitives/state-machine/src/lib.rs | 11 +- .../src/overlayed_changes/mod.rs | 59 ++-- test-utils/runtime/src/system.rs | 3 +- 4 files changed, 192 insertions(+), 193 deletions(-) diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index bac464f7aee74..7108350b28dc0 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -37,8 +37,7 @@ use crate::{warn, trace, log_error}; use sp_core::offchain::storage::OffchainOverlayedChanges; #[cfg(feature = "std")] use crate::changes_trie::State as ChangesTrieState; -#[cfg(feature = "std")] -use crate:: StorageTransactionCache; +use crate::StorageTransactionCache; #[cfg(feature = "std")] use std::error; @@ -100,11 +99,9 @@ pub struct Ext<'a, H, N, B> N: crate::changes_trie::BlockNumber, { /// Inner Ext (change overlay and backend). - inner: ExtInner<'a, H, B>, + inner: ExtInner<'a, H, B, N>, /// The overlayed changes destined for the Offchain DB. offchain_overlay: &'a mut OffchainOverlayedChanges, - /// The cache for the storage transactions. - storage_transaction_cache: &'a mut StorageTransactionCache, /// Changes trie state to read from. changes_trie_state: Option>, /// Dummy usage of N arg. @@ -114,10 +111,11 @@ pub struct Ext<'a, H, N, B> } /// Basis implementation for `Externalities` trait. -pub struct ExtInner<'a, H, B> +pub struct ExtInner<'a, H, B, N> where H: Hasher, B: Backend, + N: crate::changes_trie::BlockNumber, { /// The overlayed changes to write to. overlay: &'a mut OverlayedChanges, @@ -125,14 +123,17 @@ pub struct ExtInner<'a, H, B> backend: &'a B, /// Pseudo-unique id used for tracing. id: u16, + /// The cache for the storage transactions. + storage_transaction_cache: &'a mut StorageTransactionCache, /// Dummy usage of H arg. _phantom: sp_std::marker::PhantomData, } -impl<'a, H, B> ExtInner<'a, H, B> +impl<'a, H, B, N> ExtInner<'a, H, B, N> where H: Hasher, B: Backend, + N: crate::changes_trie::BlockNumber, { /// Create a new `ExtInner`. Warning, this /// do not init its inner id with a random @@ -140,14 +141,23 @@ impl<'a, H, B> ExtInner<'a, H, B> pub fn new( overlay: &'a mut OverlayedChanges, backend: &'a B, + storage_transaction_cache: &'a mut StorageTransactionCache, ) -> Self { ExtInner { overlay, backend, id: 0, + storage_transaction_cache, _phantom: Default::default(), } } + + /// Invalidates the currently cached storage root and the db transaction. + /// + /// Called when there are changes that likely will invalidate the storage root. + fn mark_dirty(&mut self) { + self.storage_transaction_cache.reset(); + } } #[cfg(feature = "std")] @@ -172,23 +182,16 @@ where overlay, backend, id: rand::random(), + storage_transaction_cache, _phantom: Default::default(), }, offchain_overlay, changes_trie_state, - storage_transaction_cache, _phantom: Default::default(), extensions, } } - /// Invalidates the currently cached storage root and the db transaction. - /// - /// Called when there are changes that likely will invalidate the storage root. - fn mark_dirty(&mut self) { - self.storage_transaction_cache.reset(); - } - /// Read only accessor for the scheduled overlay changes. pub fn get_offchain_storage_changes(&self) -> &OffchainOverlayedChanges { &*self.offchain_overlay @@ -221,12 +224,15 @@ where } } -impl<'a, H, B> Externalities for ExtInner<'a, H, B> +impl<'a, H, B, N> Externalities for ExtInner<'a, H, B, N> where H: Hasher, H::Out: Ord + 'static + codec::Codec, B: Backend, + N: crate::changes_trie::BlockNumber, { + fn set_offchain_storage(&mut self, _key: &[u8], _value: Option<&[u8]>) {} + fn storage(&self, key: &[u8]) -> Option { let _guard = guard(); let result = self.overlay.storage(key).map(|x| x.map(|x| x.to_vec())).unwrap_or_else(|| @@ -383,10 +389,6 @@ where } } - fn read_write_count(&self) -> (u32, u32, u32, u32) { - self.backend.read_write_count() - } - fn place_storage(&mut self, key: StorageKey, value: Option) { trace!(target: "state", "{:04x}: Put {}={:?}", self.id, @@ -399,6 +401,7 @@ where return; } + self.mark_dirty(); self.overlay.set_storage(key, value); } @@ -416,6 +419,7 @@ where ); let _guard = guard(); + self.mark_dirty(); self.overlay.set_child_storage(child_info, key, value); } @@ -429,6 +433,7 @@ where ); let _guard = guard(); + self.mark_dirty(); self.overlay.clear_child_storage(child_info); self.backend.for_keys_in_child_storage(child_info, |key| { self.overlay.set_child_storage(child_info, key.to_vec(), None); @@ -446,6 +451,7 @@ where return; } + self.mark_dirty(); self.overlay.clear_prefix(prefix); self.backend.for_keys_with_prefix(prefix, |key| { self.overlay.set_storage(key.to_vec(), None); @@ -464,6 +470,7 @@ where ); let _guard = guard(); + self.mark_dirty(); self.overlay.clear_child_prefix(child_info, prefix); self.backend.for_child_keys_with_prefix(child_info, prefix, |key| { self.overlay.set_child_storage(child_info, key.to_vec(), None); @@ -482,6 +489,7 @@ where ); let _guard = guard(); + self.mark_dirty(); let backend = &mut self.backend; let current_value = self.overlay.value_mut_or_insert_with( @@ -491,9 +499,21 @@ where StorageAppend::new(current_value).append(value); } + fn chain_id(&self) -> u64 { + 42 + } + fn storage_root(&mut self) -> Vec { let _guard = guard(); - let root = self.overlay.storage_root_no_cache(self.backend); + if let Some(ref root) = self.storage_transaction_cache.transaction_storage_root { + trace!(target: "state", "{:04x}: Root(cached) {}", + self.id, + HexDisplay::from(&root.as_ref()), + ); + return root.encode(); + } + + let root = self.overlay.storage_root(self.backend, self.storage_transaction_cache); trace!(target: "state", "{:04x}: Root {}", self.id, HexDisplay::from(&root.as_ref())); root.encode() } @@ -505,55 +525,74 @@ where let _guard = guard(); let storage_key = child_info.storage_key(); let prefixed_storage_key = child_info.prefixed_storage_key(); - - let root = if let Some((changes, info)) = self.overlay.child_changes(storage_key) { - let delta = changes.map(|(k, v)| (k.as_ref(), v.value().map(AsRef::as_ref))); - Some(self.backend.child_storage_root(info, delta)) - } else { - None - }; - - if let Some((root, is_empty, _)) = root { - let root = root.encode(); - // We store update in the overlay in order to be able to use 'self.storage_transaction' - // cache. This is brittle as it rely on Ext only querying the trie backend for - // storage root. - // A better design would be to manage 'child_storage_transaction' in a - // similar way as 'storage_transaction' but for each child trie. - if is_empty { - self.overlay.set_storage(prefixed_storage_key.into_inner(), None); - } else { - self.overlay.set_storage(prefixed_storage_key.into_inner(), Some(root.clone())); - } - - trace!(target: "state", "{:04x}: ChildRoot({}) {}", - self.id, - HexDisplay::from(&storage_key.as_ref()), - HexDisplay::from(&root.as_ref()), - ); - root - } else { - // empty overlay + if self.storage_transaction_cache.transaction_storage_root.is_some() { let root = self .storage(prefixed_storage_key.as_slice()) .and_then(|k| Decode::decode(&mut &k[..]).ok()) .unwrap_or_else( || empty_child_trie_root::>() ); - trace!(target: "state", "{:04x}: ChildRoot({})(no_change) {}", + trace!(target: "state", "{:04x}: ChildRoot({})(cached) {}", self.id, - HexDisplay::from(&storage_key.as_ref()), + HexDisplay::from(&storage_key), HexDisplay::from(&root.as_ref()), ); root.encode() + } else { + let root = if let Some((changes, info)) = self.overlay.child_changes(storage_key) { + let delta = changes.map(|(k, v)| (k.as_ref(), v.value().map(AsRef::as_ref))); + Some(self.backend.child_storage_root(info, delta)) + } else { + None + }; + + if let Some((root, is_empty, _)) = root { + let root = root.encode(); + // We store update in the overlay in order to be able to use 'self.storage_transaction' + // cache. This is brittle as it rely on Ext only querying the trie backend for + // storage root. + // A better design would be to manage 'child_storage_transaction' in a + // similar way as 'storage_transaction' but for each child trie. + if is_empty { + self.overlay.set_storage(prefixed_storage_key.into_inner(), None); + } else { + self.overlay.set_storage(prefixed_storage_key.into_inner(), Some(root.clone())); + } + + trace!(target: "state", "{:04x}: ChildRoot({}) {}", + self.id, + HexDisplay::from(&storage_key.as_ref()), + HexDisplay::from(&root.as_ref()), + ); + root + } else { + // empty overlay + let root = self + .storage(prefixed_storage_key.as_slice()) + .and_then(|k| Decode::decode(&mut &k[..]).ok()) + .unwrap_or_else( + || empty_child_trie_root::>() + ); + trace!(target: "state", "{:04x}: ChildRoot({})(no_change) {}", + self.id, + HexDisplay::from(&storage_key.as_ref()), + HexDisplay::from(&root.as_ref()), + ); + root.encode() + } } } + fn storage_changes_root(&mut self, _parent_hash: &[u8]) -> Result>, ()> { + Ok(None) + } + fn storage_start_transaction(&mut self) { self.overlay.start_transaction() } fn storage_rollback_transaction(&mut self) -> Result<(), ()> { + self.mark_dirty(); self.overlay.rollback_transaction().map_err(|_| ()) } @@ -561,43 +600,70 @@ where self.overlay.commit_transaction().map_err(|_| ()) } - fn reset_read_write_count(&mut self) { - self.backend.reset_read_write_count() - } - - fn chain_id(&self) -> u64 { - 42 - } - - fn set_offchain_storage(&mut self, _key: &[u8], _value: Option<&[u8]>) { + fn wipe(&mut self) { + for _ in 0..self.overlay.transaction_depth() { + self.overlay.rollback_transaction().expect(BENCHMARKING_FN); + } + self.overlay.drain_storage_changes( + &self.backend, + #[cfg(feature = "std")] + None, + Default::default(), + self.storage_transaction_cache, + ).expect(EXT_NOT_ALLOWED_TO_FAIL); + self.backend.wipe().expect(EXT_NOT_ALLOWED_TO_FAIL); + self.mark_dirty(); + self.overlay + .enter_runtime() + .expect("We have reset the overlay above, so we can not be in the runtime; qed"); } - fn storage_changes_root(&mut self, _parent_hash: &[u8]) -> Result>, ()> { - Ok(None) + fn commit(&mut self) { + for _ in 0..self.overlay.transaction_depth() { + self.overlay.commit_transaction().expect(BENCHMARKING_FN); + } + let changes = self.overlay.drain_storage_changes( + &self.backend, + #[cfg(feature = "std")] + None, + Default::default(), + self.storage_transaction_cache, + ).expect(EXT_NOT_ALLOWED_TO_FAIL); + self.backend.commit( + changes.transaction_storage_root, + changes.transaction, + changes.main_storage_changes, + changes.child_storage_changes, + ).expect(EXT_NOT_ALLOWED_TO_FAIL); + self.mark_dirty(); + self.overlay + .enter_runtime() + .expect("We have reset the overlay above, so we can not be in the runtime; qed"); } - fn wipe(&mut self) { - unimplemented!("Unsupported") + fn read_write_count(&self) -> (u32, u32, u32, u32) { + self.backend.read_write_count() } - fn commit(&mut self) { - unimplemented!("Unsupported") + fn reset_read_write_count(&mut self) { + self.backend.reset_read_write_count() } fn get_whitelist(&self) -> Vec { - unimplemented!("Unsupported") + self.backend.get_whitelist() } - fn set_whitelist(&mut self, _new: Vec) { - unimplemented!("Unsupported") + fn set_whitelist(&mut self, new: Vec) { + self.backend.set_whitelist(new) } } -impl<'a, H, B> ExtensionStore for ExtInner<'a, H, B> +impl<'a, H, B, N> ExtensionStore for ExtInner<'a, H, B, N> where H: Hasher, H::Out: Ord + 'static + codec::Codec, B: Backend, + N: crate::changes_trie::BlockNumber, { fn extension_by_type_id(&mut self, _type_id: TypeId) -> Option<&mut dyn Any> { None @@ -627,7 +693,6 @@ where B: 'a + Backend, N: crate::changes_trie::BlockNumber, { - fn set_offchain_storage(&mut self, key: &[u8], value: Option<&[u8]>) { use ::sp_core::offchain::STORAGE_PREFIX; match value { @@ -685,9 +750,7 @@ where } fn place_storage(&mut self, key: StorageKey, value: Option) { - let result = self.inner.place_storage(key, value); - self.mark_dirty(); - result + self.inner.place_storage(key, value) } fn place_child_storage( @@ -696,24 +759,18 @@ where key: StorageKey, value: Option, ) { - let result = self.inner.place_child_storage(child_info, key, value); - self.mark_dirty(); - result + self.inner.place_child_storage(child_info, key, value) } fn kill_child_storage( &mut self, child_info: &ChildInfo, ) { - let result = self.inner.kill_child_storage(child_info); - self.mark_dirty(); - result + self.inner.kill_child_storage(child_info) } fn clear_prefix(&mut self, prefix: &[u8]) { - let result = self.inner.clear_prefix(prefix); - self.mark_dirty(); - result + self.inner.clear_prefix(prefix) } fn clear_child_prefix( @@ -721,9 +778,7 @@ where child_info: &ChildInfo, prefix: &[u8], ) { - let result = self.inner.clear_child_prefix(child_info, prefix); - self.mark_dirty(); - result + self.inner.clear_child_prefix(child_info, prefix) } fn storage_append( @@ -731,9 +786,7 @@ where key: Vec, value: Vec, ) { - let result = self.inner.storage_append(key, value); - self.mark_dirty(); - result + self.inner.storage_append(key, value) } fn chain_id(&self) -> u64 { @@ -741,43 +794,14 @@ where } fn storage_root(&mut self) -> Vec { - let _guard = guard(); - if let Some(ref root) = self.storage_transaction_cache.transaction_storage_root { - trace!(target: "state", "{:04x}: Root(cached) {}", - self.inner.id, - HexDisplay::from(&root.as_ref()), - ); - return root.encode(); - } - - let root = self.inner.overlay.storage_root(self.inner.backend, self.storage_transaction_cache); - trace!(target: "state", "{:04x}: Root {}", self.inner.id, HexDisplay::from(&root.as_ref())); - root.encode() + self.inner.storage_root() } fn child_storage_root( &mut self, child_info: &ChildInfo, ) -> Vec { - let _guard = guard(); - let storage_key = child_info.storage_key(); - let prefixed_storage_key = child_info.prefixed_storage_key(); - if self.storage_transaction_cache.transaction_storage_root.is_some() { - let root = self - .storage(prefixed_storage_key.as_slice()) - .and_then(|k| Decode::decode(&mut &k[..]).ok()) - .unwrap_or_else( - || empty_child_trie_root::>() - ); - trace!(target: "state", "{:04x}: ChildRoot({})(cached) {}", - self.inner.id, - HexDisplay::from(&storage_key), - HexDisplay::from(&root.as_ref()), - ); - root.encode() - } else { - self.inner.child_storage_root(child_info) - } + self.inner.child_storage_root(child_info) } fn storage_changes_root(&mut self, parent_hash: &[u8]) -> Result>, ()> { @@ -793,7 +817,7 @@ where ) )?, true, - self.storage_transaction_cache, + self.inner.storage_transaction_cache, ); trace!(target: "state", "{:04x}: ChangesRoot({}) {:?}", @@ -810,7 +834,6 @@ where } fn storage_rollback_transaction(&mut self) -> Result<(), ()> { - self.mark_dirty(); self.inner.storage_rollback_transaction() } @@ -819,58 +842,27 @@ where } fn wipe(&mut self) { - for _ in 0..self.inner.overlay.transaction_depth() { - self.inner.overlay.rollback_transaction().expect(BENCHMARKING_FN); - } - self.inner.overlay.drain_storage_changes( - &self.inner.backend, - None, - Default::default(), - self.storage_transaction_cache, - ).expect(EXT_NOT_ALLOWED_TO_FAIL); - self.inner.backend.wipe().expect(EXT_NOT_ALLOWED_TO_FAIL); - self.mark_dirty(); - self.inner.overlay - .enter_runtime() - .expect("We have reset the overlay above, so we can not be in the runtime; qed"); + self.inner.wipe() } fn commit(&mut self) { - for _ in 0..self.inner.overlay.transaction_depth() { - self.inner.overlay.commit_transaction().expect(BENCHMARKING_FN); - } - let changes = self.inner.overlay.drain_storage_changes( - &self.inner.backend, - None, - Default::default(), - self.storage_transaction_cache, - ).expect(EXT_NOT_ALLOWED_TO_FAIL); - self.inner.backend.commit( - changes.transaction_storage_root, - changes.transaction, - changes.main_storage_changes, - changes.child_storage_changes, - ).expect(EXT_NOT_ALLOWED_TO_FAIL); - self.mark_dirty(); - self.inner.overlay - .enter_runtime() - .expect("We have reset the overlay above, so we can not be in the runtime; qed"); + self.inner.commit() } fn read_write_count(&self) -> (u32, u32, u32, u32) { - self.inner.backend.read_write_count() + self.inner.read_write_count() } fn reset_read_write_count(&mut self) { - self.inner.backend.reset_read_write_count() + self.inner.reset_read_write_count() } fn get_whitelist(&self) -> Vec { - self.inner.backend.get_whitelist() + self.inner.get_whitelist() } fn set_whitelist(&mut self, new: Vec) { - self.inner.backend.set_whitelist(new) + self.inner.set_whitelist(new) } } diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index ac0c35f71f81f..edd6a03bd7798 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -120,6 +120,7 @@ impl sp_std::fmt::Display for DefaultError { pub use crate::overlayed_changes::{ OverlayedChanges, StorageKey, StorageValue, StorageCollection, ChildStorageCollection, + StorageChanges, StorageTransactionCache, }; pub use crate::backend::Backend; pub use crate::trie_backend_essence::{TrieBackendStorage, Storage}; @@ -128,9 +129,17 @@ pub use crate::stats::{UsageInfo, UsageUnit, StateMachineStats}; pub use error::{Error, ExecutionError}; pub use crate::ext::ExtInner; +#[cfg(not(feature = "std"))] +mod changes_trie { + /// Stub for change trie block number until + /// change trie move to no_std. + pub trait BlockNumber {} + + impl BlockNumber for N {} +} + #[cfg(feature = "std")] mod std_reexport { - pub use crate::overlayed_changes::{StorageChanges, StorageTransactionCache}; pub use sp_trie::{trie_types::{Layout, TrieDBMut}, StorageProof, TrieMut, DBValue, MemoryDB}; pub use crate::testing::TestExternalities; pub use crate::basic::BasicExternalities; diff --git a/primitives/state-machine/src/overlayed_changes/mod.rs b/primitives/state-machine/src/overlayed_changes/mod.rs index a2ebde47ee544..ddd305aaa360b 100644 --- a/primitives/state-machine/src/overlayed_changes/mod.rs +++ b/primitives/state-machine/src/overlayed_changes/mod.rs @@ -30,10 +30,11 @@ use self::changeset::OverlayedChangeSet; use crate::{ ChangesTrieTransaction, changes_trie::{ - BlockNumber, build_changes_trie, + build_changes_trie, State as ChangesTrieState, }, }; +use crate::changes_trie::BlockNumber; #[cfg(feature = "std")] use std::collections::HashMap as Map; #[cfg(not(feature = "std"))] @@ -105,7 +106,6 @@ pub struct OverlayedChanges { /// /// This contains all the changes to the storage and transactions to apply theses changes to the /// backend. -#[cfg(feature = "std")] pub struct StorageChanges { /// All changes to the main storage. /// @@ -114,6 +114,7 @@ pub struct StorageChanges { /// All changes to the child storages. pub child_storage_changes: ChildStorageCollection, /// Offchain state changes to write to the offchain database. + #[cfg(feature = "std")] pub offchain_storage_changes: OffchainOverlayedChanges, /// A transaction for the backend that contains all changes from /// [`main_storage_changes`](StorageChanges::main_storage_changes) and from @@ -125,7 +126,11 @@ pub struct StorageChanges { /// Contains the transaction for the backend for the changes trie. /// /// If changes trie is disabled the value is set to `None`. + #[cfg(feature = "std")] pub changes_trie_transaction: Option>, + /// Phantom data for block number until change trie support no_std. + #[cfg(not(feature = "std"))] + pub _ph: sp_std::marker::PhantomData, } #[cfg(feature = "std")] @@ -153,19 +158,22 @@ impl StorageChanges { /// The storage transaction are calculated as part of the `storage_root` and /// `changes_trie_storage_root`. These transactions can be reused for importing the block into the /// storage. So, we cache them to not require a recomputation of those transactions. -#[cfg(feature = "std")] pub struct StorageTransactionCache { /// Contains the changes for the main and the child storages as one transaction. pub(crate) transaction: Option, /// The storage root after applying the transaction. pub(crate) transaction_storage_root: Option, /// Contains the changes trie transaction. + #[cfg(feature = "std")] pub(crate) changes_trie_transaction: Option>>, /// The storage root after applying the changes trie transaction. + #[cfg(feature = "std")] pub(crate) changes_trie_transaction_storage_root: Option>, + /// Phantom data for block number until change trie support no_std. + #[cfg(not(feature = "std"))] + pub(crate) _ph: sp_std::marker::PhantomData, } -#[cfg(feature = "std")] impl StorageTransactionCache { /// Reset the cached transactions. pub fn reset(&mut self) { @@ -173,28 +181,34 @@ impl StorageTransactionCache Default for StorageTransactionCache { fn default() -> Self { Self { transaction: None, transaction_storage_root: None, + #[cfg(feature = "std")] changes_trie_transaction: None, + #[cfg(feature = "std")] changes_trie_transaction_storage_root: None, + #[cfg(not(feature = "std"))] + _ph: Default::default(), } } } -#[cfg(feature = "std")] impl Default for StorageChanges { fn default() -> Self { Self { main_storage_changes: Default::default(), child_storage_changes: Default::default(), + #[cfg(feature = "std")] offchain_storage_changes: Default::default(), transaction: Default::default(), transaction_storage_root: Default::default(), + #[cfg(feature = "std")] changes_trie_transaction: None, + #[cfg(not(feature = "std"))] + _ph: Default::default(), } } } @@ -467,10 +481,10 @@ impl OverlayedChanges { } /// Drain all changes into a [`StorageChanges`] instance. Leave empty overlay in place. - #[cfg(feature = "std")] pub fn drain_storage_changes, H: Hasher, N: BlockNumber>( &mut self, backend: &B, + #[cfg(feature = "std")] changes_trie_state: Option<&ChangesTrieState>, parent_hash: H::Out, mut cache: &mut StorageTransactionCache, @@ -486,6 +500,7 @@ impl OverlayedChanges { .expect("Transaction was be generated as part of `storage_root`; qed"); // If the transaction does not exist, we generate it. + #[cfg(feature = "std")] if cache.changes_trie_transaction.is_none() { self.changes_trie_root( backend, @@ -496,20 +511,24 @@ impl OverlayedChanges { ).map_err(|_| "Failed to generate changes trie transaction")?; } + #[cfg(feature = "std")] let changes_trie_transaction = cache.changes_trie_transaction .take() .expect("Changes trie transaction was generated by `changes_trie_root`; qed"); - let offchain_storage_changes = Default::default(); let (main_storage_changes, child_storage_changes) = self.drain_committed(); Ok(StorageChanges { main_storage_changes: main_storage_changes.collect(), child_storage_changes: child_storage_changes.map(|(sk, it)| (sk, it.0.collect())).collect(), - offchain_storage_changes, + #[cfg(feature = "std")] + offchain_storage_changes: Default::default(), transaction, transaction_storage_root, + #[cfg(feature = "std")] changes_trie_transaction, + #[cfg(not(feature = "std"))] + _ph: Default::default(), }) } @@ -539,7 +558,6 @@ impl OverlayedChanges { /// as seen by the current transaction. /// /// Returns the storage root and caches storage transaction in the given `cache`. - #[cfg(feature = "std")] pub fn storage_root>( &self, backend: &B, @@ -561,27 +579,6 @@ impl OverlayedChanges { root } - /// Generate the storage root using `backend` and all changes - /// as seen by the current transaction. - /// - /// Returns the storage root and do not cache. - pub fn storage_root_no_cache>( - &self, - backend: &B, - ) -> H::Out - where H::Out: Ord + Encode, - { - let delta = self.changes().map(|(k, v)| (&k[..], v.value().map(|v| &v[..]))); - let child_delta = self.children() - .map(|(changes, info)| (info, changes.map( - |(k, v)| (&k[..], v.value().map(|v| &v[..])) - ))); - - let (root, _transaction) = backend.full_storage_root(delta, child_delta); - - root - } - /// Generate the changes trie root. /// /// Returns the changes trie root and caches the storage transaction into the given `cache`. diff --git a/test-utils/runtime/src/system.rs b/test-utils/runtime/src/system.rs index 1cf4017f2a92c..d29925a7e3a4b 100644 --- a/test-utils/runtime/src/system.rs +++ b/test-utils/runtime/src/system.rs @@ -313,7 +313,8 @@ fn execute_witness_backend() -> ApplyExtrinsicResult { Default::default(), ); let mut overlay = sp_state_machine::OverlayedChanges::default(); - let mut ext = sp_state_machine::ExtInner::new(&mut overlay, &backend); + let mut cache = sp_state_machine::StorageTransactionCache::<_, _, BlockNumber>::default(); + let mut ext = sp_state_machine::ExtInner::new(&mut overlay, &backend, &mut cache); ext.place_storage(vec![0], Some(vec![1])); assert!(ext.storage(&[0]).is_some()); Ok(Ok(())) From bdbaefc87d785f0cf85789c502e7fb9d80785f77 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 8 Sep 2020 16:45:11 +0200 Subject: [PATCH 17/21] little test --- test-utils/runtime/src/lib.rs | 62 +++++++++++++++++++++++++++++--- test-utils/runtime/src/system.rs | 17 --------- 2 files changed, 58 insertions(+), 21 deletions(-) diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index 354e217c437dd..5287f16573b06 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -29,7 +29,7 @@ use codec::{Encode, Decode, Input, Error}; use sp_core::{offchain::KeyTypeId, ChangesTrieConfiguration, OpaqueMetadata, RuntimeDebug}; use sp_application_crypto::{ed25519, sr25519, ecdsa, RuntimeAppPublic}; use trie_db::{TrieMut, Trie}; -use sp_trie::PrefixedMemoryDB; +use sp_trie::{PrefixedMemoryDB, StorageProof}; use sp_trie::trie_types::{TrieDB, TrieDBMut}; use sp_api::{decl_runtime_apis, impl_runtime_apis}; @@ -146,7 +146,6 @@ pub enum Extrinsic { IncludeData(Vec), StorageChange(Vec, Option>), ChangesTrieConfigUpdate(Option), - WitnessExt, } parity_util_mem::malloc_size_of_is_0!(Extrinsic); // non-opaque extrinsic does not need this @@ -175,8 +174,6 @@ impl BlindCheckable for Extrinsic { Extrinsic::StorageChange(key, value) => Ok(Extrinsic::StorageChange(key, value)), Extrinsic::ChangesTrieConfigUpdate(new_config) => Ok(Extrinsic::ChangesTrieConfigUpdate(new_config)), - Extrinsic::WitnessExt => - Ok(Extrinsic::WitnessExt), } } } @@ -338,6 +335,8 @@ cfg_if! { fn test_ecdsa_crypto() -> (ecdsa::AppSignature, ecdsa::AppPublic); /// Run various tests against storage. fn test_storage(); + /// Check a witness. + fn test_witness(proof: StorageProof, root: crate::Hash); /// Test that ensures that we can call a function that takes multiple /// arguments. fn test_multiple_arguments(data: Vec, other: Vec, num: u32); @@ -387,6 +386,8 @@ cfg_if! { fn test_ecdsa_crypto() -> (ecdsa::AppSignature, ecdsa::AppPublic); /// Run various tests against storage. fn test_storage(); + /// Check a witness. + fn test_witness(proof: StorageProof, root: crate::Hash); /// Test that ensures that we can call a function that takes multiple /// arguments. fn test_multiple_arguments(data: Vec, other: Vec, num: u32); @@ -687,6 +688,10 @@ cfg_if! { test_read_child_storage(); } + fn test_witness(proof: StorageProof, root: crate::Hash) { + test_witness(proof, root); + } + fn test_multiple_arguments(data: Vec, other: Vec, num: u32) { assert_eq!(&data[..], &other[..]); assert_eq!(data.len(), num as usize); @@ -929,6 +934,10 @@ cfg_if! { test_read_child_storage(); } + fn test_witness(proof: StorageProof, root: crate::Hash) { + test_witness(proof, root); + } + fn test_multiple_arguments(data: Vec, other: Vec, num: u32) { assert_eq!(&data[..], &other[..]); assert_eq!(data.len(), num as usize); @@ -1102,6 +1111,22 @@ fn test_read_child_storage() { assert_eq!(&v, &[0, 0, 0, 0]); } +fn test_witness(proof: StorageProof, root: crate::Hash) { + use sp_externalities::Externalities; + let db: sp_trie::MemoryDB = proof.into_memory_db(); + let backend = sp_state_machine::TrieBackend::<_, crate::Hashing>::new( + db, + root, + ); + let mut overlay = sp_state_machine::OverlayedChanges::default(); + let mut cache = sp_state_machine::StorageTransactionCache::<_, _, BlockNumber>::default(); + let mut ext = sp_state_machine::ExtInner::new(&mut overlay, &backend, &mut cache); + assert!(ext.storage(b"value3").is_some()); + assert!(ext.storage_root().as_slice() == &root[..]); + ext.place_storage(vec![0], Some(vec![1])); + assert!(ext.storage_root().as_slice() != &root[..]); +} + #[cfg(test)] mod tests { use substrate_test_runtime_client::{ @@ -1160,4 +1185,33 @@ mod tests { runtime_api.test_storage(&block_id).unwrap(); } + + fn witness_backend() -> (sp_trie::MemoryDB, crate::Hash) { + use sp_trie::TrieMut; + let mut root = crate::Hash::default(); + let mut mdb = sp_trie::MemoryDB::::default(); + { + let mut trie = sp_trie::trie_types::TrieDBMut::new(&mut mdb, &mut root); + trie.insert(b"value3", &[142]).expect("insert failed"); + trie.insert(b"value4", &[124]).expect("insert failed"); + }; + (mdb, root) + } + + #[test] + fn witness_backend_works() { + let (db, root) = witness_backend(); + let backend = sp_state_machine::TrieBackend::<_, crate::Hashing>::new( + db, + root, + ); + let proof = sp_state_machine::prove_read(backend, vec![b"value3"]).unwrap(); + let client = TestClientBuilder::new() + .set_execution_strategy(ExecutionStrategy::Both) + .build(); + let runtime_api = client.runtime_api(); + let block_id = BlockId::Number(client.chain_info().best_number); + + runtime_api.test_witness(&block_id, proof, root).unwrap(); + } } diff --git a/test-utils/runtime/src/system.rs b/test-utils/runtime/src/system.rs index d29925a7e3a4b..818487a89e518 100644 --- a/test-utils/runtime/src/system.rs +++ b/test-utils/runtime/src/system.rs @@ -261,8 +261,6 @@ fn execute_transaction_backend(utx: &Extrinsic, extrinsic_index: u32) -> ApplyEx execute_storage_change(key, value.as_ref().map(|v| &**v)), Extrinsic::ChangesTrieConfigUpdate(ref new_config) => execute_changes_trie_config_update(new_config.clone()), - Extrinsic::WitnessExt => - execute_witness_backend(), } } @@ -305,21 +303,6 @@ fn execute_storage_change(key: &[u8], value: Option<&[u8]>) -> ApplyExtrinsicRes Ok(Ok(())) } -fn execute_witness_backend() -> ApplyExtrinsicResult { - use sp_externalities::Externalities; - let db = sp_trie::MemoryDB::::default(); - let backend = sp_state_machine::TrieBackend::<_, crate::Hashing>::new( - db, - Default::default(), - ); - let mut overlay = sp_state_machine::OverlayedChanges::default(); - let mut cache = sp_state_machine::StorageTransactionCache::<_, _, BlockNumber>::default(); - let mut ext = sp_state_machine::ExtInner::new(&mut overlay, &backend, &mut cache); - ext.place_storage(vec![0], Some(vec![1])); - assert!(ext.storage(&[0]).is_some()); - Ok(Ok(())) -} - fn execute_changes_trie_config_update(new_config: Option) -> ApplyExtrinsicResult { match new_config.clone() { Some(new_config) => storage::unhashed::put_raw( From d1b10c84fdcb6518abbeced6b159b5e746cae753 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 10 Sep 2020 09:16:55 +0200 Subject: [PATCH 18/21] fuse imports and filter_map prepare_extrinsics_input_inner fold. --- primitives/arithmetic/src/biguint.rs | 2 +- primitives/externalities/Cargo.toml | 2 +- primitives/externalities/src/extensions.rs | 4 +- primitives/externalities/src/lib.rs | 4 +- primitives/state-machine/Cargo.toml | 2 +- .../state-machine/src/changes_trie/build.rs | 75 ++++++++++--------- primitives/state-machine/src/lib.rs | 5 +- .../src/overlayed_changes/mod.rs | 2 +- .../state-machine/src/trie_backend_essence.rs | 4 +- 9 files changed, 50 insertions(+), 50 deletions(-) diff --git a/primitives/arithmetic/src/biguint.rs b/primitives/arithmetic/src/biguint.rs index 32edbfcd235d6..41e2c759a5967 100644 --- a/primitives/arithmetic/src/biguint.rs +++ b/primitives/arithmetic/src/biguint.rs @@ -18,7 +18,7 @@ //! Infinite precision unsigned integer for substrate runtime. use num_traits::Zero; -use sp_std::{cmp::Ordering, ops, prelude::*, cell::RefCell, convert::TryFrom, vec}; +use sp_std::{cmp::Ordering, ops, prelude::*, cell::RefCell, convert::TryFrom}; // A sensible value for this would be half of the dword size of the host machine. Since the // runtime is compiled to 32bit webassembly, using 32 and 64 for single and double respectively diff --git a/primitives/externalities/Cargo.toml b/primitives/externalities/Cargo.toml index 10f060a059594..952912bee592c 100644 --- a/primitives/externalities/Cargo.toml +++ b/primitives/externalities/Cargo.toml @@ -14,7 +14,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] sp-storage = { version = "2.0.0-rc6", path = "../storage", default-features = false } -sp-std = { version = "2.0.0-rc5", path = "../std", default-features = false } +sp-std = { version = "2.0.0-rc6", path = "../std", default-features = false } environmental = { version = "1.1.1", default-features = false } codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } diff --git a/primitives/externalities/src/extensions.rs b/primitives/externalities/src/extensions.rs index bb611952521ba..c4b34de6f6d07 100644 --- a/primitives/externalities/src/extensions.rs +++ b/primitives/externalities/src/extensions.rs @@ -22,8 +22,8 @@ //! //! It is required that each extension implements the [`Extension`] trait. -use sp_std::collections::btree_map::{BTreeMap, Entry}; -use sp_std::{any::{Any, TypeId}, ops::DerefMut, boxed::Box}; +use sp_std::{collections::btree_map::{BTreeMap, Entry}, any::{Any, TypeId}, + ops::DerefMut, boxed::Box}; use crate::Error; /// Marker trait for types that should be registered as [`Externalities`](crate::Externalities) extension. diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index 7bd1a00c9bc4c..388482964f18c 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -25,9 +25,7 @@ //! //! This crate exposes the main [`Externalities`] trait. -use sp_std::any::{Any, TypeId}; -use sp_std::vec::Vec; -use sp_std::boxed::Box; +use sp_std::{any::{Any, TypeId}, vec::Vec, boxed::Box}; use sp_storage::{ChildInfo, TrackedStorageKey}; diff --git a/primitives/state-machine/Cargo.toml b/primitives/state-machine/Cargo.toml index 998720d4ecb59..8ae342835ba98 100644 --- a/primitives/state-machine/Cargo.toml +++ b/primitives/state-machine/Cargo.toml @@ -26,7 +26,7 @@ num-traits = { version = "0.2.8", default-features = false } rand = { version = "0.7.2", optional = true } sp-externalities = { version = "0.8.0-rc6", path = "../externalities", default-features = false } smallvec = "1.4.1" -sp-std = { version = "2.0.0-rc5", default-features = false, path = "../std" } +sp-std = { version = "2.0.0-rc6", default-features = false, path = "../std" } [dev-dependencies] hex-literal = "0.3.1" diff --git a/primitives/state-machine/src/changes_trie/build.rs b/primitives/state-machine/src/changes_trie/build.rs index 2628d8d15358c..b23481411ae27 100644 --- a/primitives/state-machine/src/changes_trie/build.rs +++ b/primitives/state-machine/src/changes_trie/build.rs @@ -132,7 +132,7 @@ fn prepare_extrinsics_input_inner<'a, B, H, Number>( block: &Number, overlay: &'a OverlayedChanges, child_info: Option, - mut changes: impl Iterator + changes: impl Iterator ) -> Result> + 'a, String> where B: Backend, @@ -140,44 +140,49 @@ fn prepare_extrinsics_input_inner<'a, B, H, Number>( Number: BlockNumber, { changes - .try_fold(BTreeMap::new(), |mut map: BTreeMap<&[u8], (ExtrinsicIndex, Vec)>, (k, v)| { + .filter_map(|(k, v)| { let extrinsics = v.extrinsics(); if !extrinsics.is_empty() { - match map.entry(k) { - Entry::Vacant(entry) => { - // ignore temporary values (values that have null value at the end of operation - // AND are not in storage at the beginning of operation - if let Some(child_info) = child_info.as_ref() { - if !overlay.child_storage(child_info, k).map(|v| v.is_some()).unwrap_or_default() { - if !backend.exists_child_storage(&child_info, k) - .map_err(|e| format!("{}", e))? { - return Ok(map); - } + Some((k, extrinsics)) + } else { + None + } + }) + .try_fold(BTreeMap::new(), |mut map: BTreeMap<&[u8], (ExtrinsicIndex, Vec)>, (k, extrinsics)| { + match map.entry(k) { + Entry::Vacant(entry) => { + // ignore temporary values (values that have null value at the end of operation + // AND are not in storage at the beginning of operation + if let Some(child_info) = child_info.as_ref() { + if !overlay.child_storage(child_info, k).map(|v| v.is_some()).unwrap_or_default() { + if !backend.exists_child_storage(&child_info, k) + .map_err(|e| format!("{}", e))? { + return Ok(map); } - } else { - if !overlay.storage(k).map(|v| v.is_some()).unwrap_or_default() { - if !backend.exists_storage(k).map_err(|e| format!("{}", e))? { - return Ok(map); - } + } + } else { + if !overlay.storage(k).map(|v| v.is_some()).unwrap_or_default() { + if !backend.exists_storage(k).map_err(|e| format!("{}", e))? { + return Ok(map); } - }; - - let extrinsics = extrinsics.into_iter().collect(); - entry.insert((ExtrinsicIndex { - block: block.clone(), - key: k.to_vec(), - }, extrinsics)); - }, - Entry::Occupied(mut entry) => { - // we do not need to check for temporary values here, because entry is Occupied - // AND we are checking it before insertion - let entry_extrinsics = &mut entry.get_mut().1; - entry_extrinsics.extend( - extrinsics.into_iter() - ); - entry_extrinsics.sort(); - }, - } + } + }; + + let extrinsics = extrinsics.into_iter().collect(); + entry.insert((ExtrinsicIndex { + block: block.clone(), + key: k.to_vec(), + }, extrinsics)); + }, + Entry::Occupied(mut entry) => { + // we do not need to check for temporary values here, because entry is Occupied + // AND we are checking it before insertion + let entry_extrinsics = &mut entry.get_mut().1; + entry_extrinsics.extend( + extrinsics.into_iter() + ); + entry_extrinsics.sort(); + }, } Ok(map) diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index edd6a03bd7798..0ef563d0a6251 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -46,7 +46,6 @@ pub use std_reexport::*; #[cfg(feature = "std")] pub use execution::*; - #[cfg(feature = "std")] pub use log::{debug, warn, trace, error as log_error}; @@ -102,7 +101,7 @@ macro_rules! log_error { ); } -/// Default rror type to use with state machine trie backend. +/// Default error type to use with state machine trie backend. #[cfg(feature = "std")] pub type DefaultError = String; /// Error type to use with state machine trie backend. @@ -113,7 +112,7 @@ pub struct DefaultError; #[cfg(not(feature = "std"))] impl sp_std::fmt::Display for DefaultError { fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { - write!(f, "Default Error") + write!(f, "DefaultError") } } diff --git a/primitives/state-machine/src/overlayed_changes/mod.rs b/primitives/state-machine/src/overlayed_changes/mod.rs index ddd305aaa360b..992f7b3519299 100644 --- a/primitives/state-machine/src/overlayed_changes/mod.rs +++ b/primitives/state-machine/src/overlayed_changes/mod.rs @@ -81,7 +81,7 @@ impl Extrinsics { } } - /// Merge two extrinsics sets. + /// Extend `self` with `other`. fn extend(&mut self, other: Self) { self.0.extend(other.0.into_iter()); } diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index a270ac857677c..37bbbb7cf9822 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -20,7 +20,7 @@ #[cfg(feature = "std")] use std::sync::Arc; -use sp_std::ops::Deref; +use sp_std::{ops::Deref, boxed::Box, vec::Vec}; use crate::{warn, debug}; use hash_db::{self, Hasher, Prefix}; use sp_trie::{Trie, MemoryDB, PrefixedMemoryDB, DBValue, @@ -30,8 +30,6 @@ use sp_trie::trie_types::{TrieDB, TrieError, Layout}; use crate::{backend::Consolidate, StorageKey, StorageValue}; use sp_core::storage::ChildInfo; use codec::Encode; -use sp_std::boxed::Box; -use sp_std::vec::Vec; #[cfg(not(feature = "std"))] macro_rules! format { From 14e40b60746d4d2a787bc7193ed2328ba26611cb Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 10 Sep 2020 09:58:31 +0200 Subject: [PATCH 19/21] put back ExtInner into Ext, awkward double proto for new function. --- primitives/state-machine/src/ext.rs | 367 ++++++++-------------------- primitives/state-machine/src/lib.rs | 5 +- test-utils/runtime/src/lib.rs | 14 +- 3 files changed, 115 insertions(+), 271 deletions(-) diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 7108350b28dc0..0cc2d2fcc112a 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -28,7 +28,7 @@ use sp_core::{ }; use sp_trie::{trie_types::Layout, empty_child_trie_root}; use sp_externalities::{Externalities, Extensions, Extension, - ExtensionStore, Error as ExtensionError}; + ExtensionStore}; use codec::{Decode, Encode, EncodeAppend}; use sp_std::{fmt, any::{Any, TypeId}, vec::Vec, vec, boxed::Box}; @@ -91,59 +91,50 @@ impl error::Error for Error { } /// Wraps a read-only backend, call executor, and current overlayed changes. -#[cfg(feature = "std")] pub struct Ext<'a, H, N, B> where H: Hasher, B: 'a + Backend, N: crate::changes_trie::BlockNumber, { - /// Inner Ext (change overlay and backend). - inner: ExtInner<'a, H, B, N>, + /// The overlayed changes to write to. + overlay: &'a mut OverlayedChanges, /// The overlayed changes destined for the Offchain DB. + #[cfg(feature = "std")] offchain_overlay: &'a mut OffchainOverlayedChanges, + /// The storage backend to read from. + backend: &'a B, + /// The cache for the storage transactions. + storage_transaction_cache: &'a mut StorageTransactionCache, /// Changes trie state to read from. + #[cfg(feature = "std")] changes_trie_state: Option>, + /// Pseudo-unique id used for tracing. + pub id: u16, /// Dummy usage of N arg. - _phantom: std::marker::PhantomData, + _phantom: sp_std::marker::PhantomData, /// Extensions registered with this instance. + #[cfg(feature = "std")] extensions: Option<&'a mut Extensions>, } -/// Basis implementation for `Externalities` trait. -pub struct ExtInner<'a, H, B, N> - where - H: Hasher, - B: Backend, - N: crate::changes_trie::BlockNumber, -{ - /// The overlayed changes to write to. - overlay: &'a mut OverlayedChanges, - /// The storage backend to read from. - backend: &'a B, - /// Pseudo-unique id used for tracing. - id: u16, - /// The cache for the storage transactions. - storage_transaction_cache: &'a mut StorageTransactionCache, - /// Dummy usage of H arg. - _phantom: sp_std::marker::PhantomData, -} -impl<'a, H, B, N> ExtInner<'a, H, B, N> +impl<'a, H, N, B> Ext<'a, H, N, B> where H: Hasher, B: Backend, N: crate::changes_trie::BlockNumber, { - /// Create a new `ExtInner`. Warning, this + /// Create a new `Ext`. Warning, this /// do not init its inner id with a random /// value. + #[cfg(not(feature = "std"))] pub fn new( overlay: &'a mut OverlayedChanges, - backend: &'a B, storage_transaction_cache: &'a mut StorageTransactionCache, + backend: &'a B, ) -> Self { - ExtInner { + Ext { overlay, backend, id: 0, @@ -152,23 +143,8 @@ impl<'a, H, B, N> ExtInner<'a, H, B, N> } } - /// Invalidates the currently cached storage root and the db transaction. - /// - /// Called when there are changes that likely will invalidate the storage root. - fn mark_dirty(&mut self) { - self.storage_transaction_cache.reset(); - } -} - -#[cfg(feature = "std")] -impl<'a, H, N, B> Ext<'a, H, N, B> -where - H: Hasher, - H::Out: Ord + 'static + codec::Codec, - B: 'a + Backend, - N: crate::changes_trie::BlockNumber, -{ /// Create a new `Ext` from overlayed changes and read-only backend + #[cfg(feature = "std")] pub fn new( overlay: &'a mut OverlayedChanges, offchain_overlay: &'a mut OffchainOverlayedChanges, @@ -178,29 +154,29 @@ where extensions: Option<&'a mut Extensions>, ) -> Self { Self { - inner: ExtInner { - overlay, - backend, - id: rand::random(), - storage_transaction_cache, - _phantom: Default::default(), - }, + overlay, offchain_overlay, + backend, changes_trie_state, + storage_transaction_cache, + id: rand::random(), _phantom: Default::default(), extensions, } } + /// Invalidates the currently cached storage root and the db transaction. + /// + /// Called when there are changes that likely will invalidate the storage root. + fn mark_dirty(&mut self) { + self.storage_transaction_cache.reset(); + } + /// Read only accessor for the scheduled overlay changes. + #[cfg(feature = "std")] pub fn get_offchain_storage_changes(&self) -> &OffchainOverlayedChanges { &*self.offchain_overlay } - - /// Access internal random id. - pub fn id(&self) -> u16 { - self.inner.id - } } #[cfg(test)] @@ -214,9 +190,9 @@ where pub fn storage_pairs(&self) -> Vec<(StorageKey, StorageValue)> { use std::collections::HashMap; - self.inner.backend.pairs().iter() + self.backend.pairs().iter() .map(|&(ref k, ref v)| (k.to_vec(), Some(v.to_vec()))) - .chain(self.inner.overlay.changes().map(|(k, v)| (k.clone(), v.value().cloned()))) + .chain(self.overlay.changes().map(|(k, v)| (k.clone(), v.value().cloned()))) .collect::>() .into_iter() .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) @@ -224,13 +200,23 @@ where } } -impl<'a, H, B, N> Externalities for ExtInner<'a, H, B, N> +impl<'a, H, N, B> Externalities for Ext<'a, H, N, B> where H: Hasher, H::Out: Ord + 'static + codec::Codec, B: Backend, N: crate::changes_trie::BlockNumber, { + #[cfg(feature = "std")] + fn set_offchain_storage(&mut self, key: &[u8], value: Option<&[u8]>) { + use ::sp_core::offchain::STORAGE_PREFIX; + match value { + Some(value) => self.offchain_overlay.set(STORAGE_PREFIX, key, value), + None => self.offchain_overlay.remove(STORAGE_PREFIX, key), + } + } + + #[cfg(not(feature = "std"))] fn set_offchain_storage(&mut self, _key: &[u8], _value: Option<&[u8]>) {} fn storage(&self, key: &[u8]) -> Option { @@ -583,10 +569,37 @@ where } } + #[cfg(not(feature = "std"))] fn storage_changes_root(&mut self, _parent_hash: &[u8]) -> Result>, ()> { Ok(None) } + #[cfg(feature = "std")] + fn storage_changes_root(&mut self, parent_hash: &[u8]) -> Result>, ()> { + let _guard = guard(); + let root = self.overlay.changes_trie_root( + self.backend, + self.changes_trie_state.as_ref(), + Decode::decode(&mut &parent_hash[..]).map_err(|e| + trace!( + target: "state", + "Failed to decode changes root parent hash: {}", + e, + ) + )?, + true, + self.storage_transaction_cache, + ); + + trace!(target: "state", "{:04x}: ChangesRoot({}) {:?}", + self.id, + HexDisplay::from(&parent_hash), + root, + ); + + root.map(|r| r.map(|o| o.encode())) + } + fn storage_start_transaction(&mut self) { self.overlay.start_transaction() } @@ -658,214 +671,6 @@ where } } -impl<'a, H, B, N> ExtensionStore for ExtInner<'a, H, B, N> -where - H: Hasher, - H::Out: Ord + 'static + codec::Codec, - B: Backend, - N: crate::changes_trie::BlockNumber, -{ - fn extension_by_type_id(&mut self, _type_id: TypeId) -> Option<&mut dyn Any> { - None - } - - fn register_extension_with_type_id( - &mut self, - _type_id: TypeId, - _extension: Box, - ) -> Result<(), ExtensionError> { - Err(ExtensionError::ExtensionsAreNotSupported) - } - - fn deregister_extension_by_type_id( - &mut self, - _type_id: TypeId, - ) -> Result<(), ExtensionError> { - Err(ExtensionError::ExtensionsAreNotSupported) - } -} - -#[cfg(feature = "std")] -impl<'a, H, B, N> Externalities for Ext<'a, H, N, B> -where - H: Hasher, - H::Out: Ord + 'static + codec::Codec, - B: 'a + Backend, - N: crate::changes_trie::BlockNumber, -{ - fn set_offchain_storage(&mut self, key: &[u8], value: Option<&[u8]>) { - use ::sp_core::offchain::STORAGE_PREFIX; - match value { - Some(value) => self.offchain_overlay.set(STORAGE_PREFIX, key, value), - None => self.offchain_overlay.remove(STORAGE_PREFIX, key), - } - } - - fn storage(&self, key: &[u8]) -> Option { - self.inner.storage(key) - } - - fn storage_hash(&self, key: &[u8]) -> Option> { - self.inner.storage_hash(key) - } - - fn child_storage( - &self, - child_info: &ChildInfo, - key: &[u8], - ) -> Option { - self.inner.child_storage(child_info, key) - } - - fn child_storage_hash( - &self, - child_info: &ChildInfo, - key: &[u8], - ) -> Option> { - self.inner.child_storage_hash(child_info, key) - } - - fn exists_storage(&self, key: &[u8]) -> bool { - self.inner.exists_storage(key) - } - - fn exists_child_storage( - &self, - child_info: &ChildInfo, - key: &[u8], - ) -> bool { - self.inner.exists_child_storage(child_info, key) - } - - fn next_storage_key(&self, key: &[u8]) -> Option { - self.inner.next_storage_key(key) - } - - fn next_child_storage_key( - &self, - child_info: &ChildInfo, - key: &[u8], - ) -> Option { - self.inner.next_child_storage_key(child_info, key) - } - - fn place_storage(&mut self, key: StorageKey, value: Option) { - self.inner.place_storage(key, value) - } - - fn place_child_storage( - &mut self, - child_info: &ChildInfo, - key: StorageKey, - value: Option, - ) { - self.inner.place_child_storage(child_info, key, value) - } - - fn kill_child_storage( - &mut self, - child_info: &ChildInfo, - ) { - self.inner.kill_child_storage(child_info) - } - - fn clear_prefix(&mut self, prefix: &[u8]) { - self.inner.clear_prefix(prefix) - } - - fn clear_child_prefix( - &mut self, - child_info: &ChildInfo, - prefix: &[u8], - ) { - self.inner.clear_child_prefix(child_info, prefix) - } - - fn storage_append( - &mut self, - key: Vec, - value: Vec, - ) { - self.inner.storage_append(key, value) - } - - fn chain_id(&self) -> u64 { - 42 - } - - fn storage_root(&mut self) -> Vec { - self.inner.storage_root() - } - - fn child_storage_root( - &mut self, - child_info: &ChildInfo, - ) -> Vec { - self.inner.child_storage_root(child_info) - } - - fn storage_changes_root(&mut self, parent_hash: &[u8]) -> Result>, ()> { - let _guard = guard(); - let root = self.inner.overlay.changes_trie_root( - self.inner.backend, - self.changes_trie_state.as_ref(), - Decode::decode(&mut &parent_hash[..]).map_err(|e| - trace!( - target: "state", - "Failed to decode changes root parent hash: {}", - e, - ) - )?, - true, - self.inner.storage_transaction_cache, - ); - - trace!(target: "state", "{:04x}: ChangesRoot({}) {:?}", - self.inner.id, - HexDisplay::from(&parent_hash), - root, - ); - - root.map(|r| r.map(|o| o.encode())) - } - - fn storage_start_transaction(&mut self) { - self.inner.storage_start_transaction() - } - - fn storage_rollback_transaction(&mut self) -> Result<(), ()> { - self.inner.storage_rollback_transaction() - } - - fn storage_commit_transaction(&mut self) -> Result<(), ()> { - self.inner.storage_commit_transaction() - } - - fn wipe(&mut self) { - self.inner.wipe() - } - - fn commit(&mut self) { - self.inner.commit() - } - - fn read_write_count(&self) -> (u32, u32, u32, u32) { - self.inner.read_write_count() - } - - fn reset_read_write_count(&mut self) { - self.inner.reset_read_write_count() - } - - fn get_whitelist(&self) -> Vec { - self.inner.get_whitelist() - } - - fn set_whitelist(&mut self, new: Vec) { - self.inner.set_whitelist(new) - } -} - /// Implement `Encode` by forwarding the stored raw vec. struct EncodeOpaqueValue(Vec); @@ -905,8 +710,36 @@ impl<'a> StorageAppend<'a> { } } +#[cfg(not(feature = "std"))] +impl<'a, H, N, B> ExtensionStore for Ext<'a, H, N, B> +where + H: Hasher, + H::Out: Ord + 'static + codec::Codec, + B: Backend, + N: crate::changes_trie::BlockNumber, +{ + fn extension_by_type_id(&mut self, _type_id: TypeId) -> Option<&mut dyn Any> { + None + } + + fn register_extension_with_type_id( + &mut self, + _type_id: TypeId, + _extension: Box, + ) -> Result<(), sp_externalities::Error> { + Err(sp_externalities::Error::ExtensionsAreNotSupported) + } + + fn deregister_extension_by_type_id( + &mut self, + _type_id: TypeId, + ) -> Result<(), sp_externalities::Error> { + Err(sp_externalities::Error::ExtensionsAreNotSupported) + } +} + #[cfg(feature = "std")] -impl<'a, H, B, N> sp_externalities::ExtensionStore for Ext<'a, H, N, B> +impl<'a, H, N, B> ExtensionStore for Ext<'a, H, N, B> where H: Hasher, B: 'a + Backend, diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 0ef563d0a6251..89a509127b29d 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -126,7 +126,7 @@ pub use crate::trie_backend_essence::{TrieBackendStorage, Storage}; pub use crate::trie_backend::TrieBackend; pub use crate::stats::{UsageInfo, UsageUnit, StateMachineStats}; pub use error::{Error, ExecutionError}; -pub use crate::ext::ExtInner; +pub use crate::ext::Ext; #[cfg(not(feature = "std"))] mod changes_trie { @@ -143,7 +143,6 @@ mod std_reexport { pub use crate::testing::TestExternalities; pub use crate::basic::BasicExternalities; pub use crate::read_only::{ReadOnlyExternalities, InspectState}; - pub use crate::ext::Ext; pub use crate::changes_trie::{ AnchorBlockId as ChangesTrieAnchorBlockId, State as ChangesTrieState, @@ -416,7 +415,7 @@ mod execution { Some(&mut self.extensions), ); - let id = ext.id(); + let id = ext.id; trace!( target: "state", "{:04x}: Call {} at {:?}. Input={:?}", id, diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index 5287f16573b06..a7c1c261b5541 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -1119,8 +1119,20 @@ fn test_witness(proof: StorageProof, root: crate::Hash) { root, ); let mut overlay = sp_state_machine::OverlayedChanges::default(); + #[cfg(feature = "std")] + let mut offchain_overlay = Default::default(); let mut cache = sp_state_machine::StorageTransactionCache::<_, _, BlockNumber>::default(); - let mut ext = sp_state_machine::ExtInner::new(&mut overlay, &backend, &mut cache); + let mut ext = sp_state_machine::Ext::new( + &mut overlay, + #[cfg(feature = "std")] + &mut offchain_overlay, + &mut cache, + &backend, + #[cfg(feature = "std")] + None, + #[cfg(feature = "std")] + None, + ); assert!(ext.storage(b"value3").is_some()); assert!(ext.storage_root().as_slice() == &root[..]); ext.place_storage(vec![0], Some(vec![1])); From e459698712725de5a8a97182145cf607c339bca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 11 Sep 2020 11:36:33 +0200 Subject: [PATCH 20/21] Apply suggestions from code review --- primitives/externalities/src/extensions.rs | 5 +++-- primitives/state-machine/src/ext.rs | 4 +--- primitives/state-machine/src/lib.rs | 1 - 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/primitives/externalities/src/extensions.rs b/primitives/externalities/src/extensions.rs index c4b34de6f6d07..d79f99d3344ea 100644 --- a/primitives/externalities/src/extensions.rs +++ b/primitives/externalities/src/extensions.rs @@ -22,8 +22,9 @@ //! //! It is required that each extension implements the [`Extension`] trait. -use sp_std::{collections::btree_map::{BTreeMap, Entry}, any::{Any, TypeId}, - ops::DerefMut, boxed::Box}; +use sp_std::{ + collections::btree_map::{BTreeMap, Entry}, any::{Any, TypeId}, ops::DerefMut, boxed::Box, +}; use crate::Error; /// Marker trait for types that should be registered as [`Externalities`](crate::Externalities) extension. diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 0cc2d2fcc112a..e9259f9a10bc1 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -125,9 +125,7 @@ impl<'a, H, N, B> Ext<'a, H, N, B> B: Backend, N: crate::changes_trie::BlockNumber, { - /// Create a new `Ext`. Warning, this - /// do not init its inner id with a random - /// value. + /// Create a new `Ext`. #[cfg(not(feature = "std"))] pub fn new( overlay: &'a mut OverlayedChanges, diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 89a509127b29d..5b86640aa7d0e 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -45,7 +45,6 @@ pub use std_reexport::*; #[cfg(feature = "std")] pub use execution::*; - #[cfg(feature = "std")] pub use log::{debug, warn, trace, error as log_error}; From bef5997f5e323cdbf5411cb5793e9c557008cb26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 11 Sep 2020 11:37:07 +0200 Subject: [PATCH 21/21] Update primitives/state-machine/Cargo.toml --- primitives/state-machine/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/primitives/state-machine/Cargo.toml b/primitives/state-machine/Cargo.toml index 8ae342835ba98..f34fabdd88908 100644 --- a/primitives/state-machine/Cargo.toml +++ b/primitives/state-machine/Cargo.toml @@ -45,7 +45,6 @@ std = [ "sp-trie/std", "trie-db/std", "trie-root/std", - "log", "parking_lot", "rand",