Skip to content
This repository has been archived by the owner on Nov 15, 2023. It is now read-only.

Some changes tries optimizations #2840

Merged
merged 3 commits into from
Jul 16, 2019
Merged

Some changes tries optimizations #2840

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
652a449
changes tries initial optimizations
svyatonik Jun 11, 2019
b2ed0f5
line width
svyatonik Jun 11, 2019
2c509fa
Merge branch 'master' into changes_tries_initial_optimization
svyatonik Jun 12, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
194 changes: 115 additions & 79 deletions core/state-machine/src/changes_trie/build.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@

//! Structures and functions required to build changes trie for given block.

use std::collections::{BTreeMap, BTreeSet};
use std::collections::BTreeMap;
use std::collections::btree_map::Entry;
use parity_codec::Decode;
use hash_db::Hasher;
use num_traits::One;
Expand All @@ -31,75 +32,89 @@ use crate::changes_trie::{AnchorBlockId, Configuration, Storage, BlockNumber};
///
/// Returns Err if storage error has occured OR if storage haven't returned
/// required data.
/// Returns Ok(None) data required to prepare input pairs is not collected
/// or storage is not provided.
pub fn prepare_input<'a, B, S, H, Number>(
backend: &B,
backend: &'a B,
storage: &'a S,
config: &'a Configuration,
changes: &OverlayedChanges,
changes: &'a OverlayedChanges,
parent: &'a AnchorBlockId<H::Out, Number>,
) -> Result<Option<Vec<InputPair<Number>>>, String>
) -> Result<impl Iterator<Item=InputPair<Number>> + 'a, String>
where
B: Backend<H>,
S: Storage<H, Number>,
H: Hasher,
H: Hasher + 'a,
Number: BlockNumber,
{
let mut input = Vec::new();
input.extend(prepare_extrinsics_input(
let number = parent.number.clone() + One::one();
let extrinsics_input = prepare_extrinsics_input(
backend,
parent.number.clone() + 1.into(),
changes)?);
input.extend(prepare_digest_input::<_, H, Number>(
&number,
changes)?;
let digest_input = prepare_digest_input::<_, H, Number>(
parent,
config,
storage)?);

Ok(Some(input))
number,
storage)?;
Ok(extrinsics_input.chain(digest_input))
}

/// Prepare ExtrinsicIndex input pairs.
fn prepare_extrinsics_input<B, H, Number>(
backend: &B,
block: Number,
changes: &OverlayedChanges,
) -> Result<impl Iterator<Item=InputPair<Number>>, String>
fn prepare_extrinsics_input<'a, B, H, Number>(
backend: &'a B,
block: &Number,
changes: &'a OverlayedChanges,
) -> Result<impl Iterator<Item=InputPair<Number>> + 'a, String>
where
B: Backend<H>,
H: Hasher,
Number: BlockNumber,
{
let mut extrinsic_map = BTreeMap::<Vec<u8>, BTreeSet<u32>>::new();
for (key, val) in changes.prospective.top.iter().chain(changes.committed.top.iter()) {
let extrinsics = match val.extrinsics {
Some(ref extrinsics) => extrinsics,
None => continue,
};
changes.committed.top.iter()
.chain(changes.prospective.top.iter())
.filter(|( _, v)| v.extrinsics.is_some())
.try_fold(BTreeMap::new(), |mut map: BTreeMap<&[u8], (ExtrinsicIndex<Number>, Vec<u32>)>, (k, v)| {
match map.entry(k) {
Entry::Vacant(entry) => {
// ignore temporary values (values that have null value at the end of operation
// AND are not in storage at the beginning of operation
if !changes.storage(k).map(|v| v.is_some()).unwrap_or_default() {
if !backend.exists_storage(k).map_err(|e| format!("{}", e))? {
return Ok(map);
}
}

// ignore values that have null value at the end of operation AND are not in storage
// at the beginning of operation
if !changes.storage(key).map(|v| v.is_some()).unwrap_or_default() {
if !backend.exists_storage(key).map_err(|e| format!("{}", e))? {
continue;
let extrinsics = v.extrinsics.as_ref()
.expect("filtered by filter() call above; qed")
.iter().cloned().collect();
entry.insert((ExtrinsicIndex {
block: block.clone(),
key: k.to_vec(),
}, extrinsics));
},
Entry::Occupied(mut entry) => {
// we do not need to check for temporary values here, because entry is Occupied
// AND we are checking it before insertion
let extrinsics = &mut entry.get_mut().1;
extrinsics.extend(
v.extrinsics.as_ref()
.expect("filtered by filter() call above; qed")
.iter()
.cloned()
);
extrinsics.sort_unstable();
},
}
}

extrinsic_map.entry(key.clone()).or_default()
.extend(extrinsics.iter().cloned());
}

Ok(extrinsic_map.into_iter()
.map(move |(key, extrinsics)| InputPair::ExtrinsicIndex(ExtrinsicIndex {
block: block.clone(),
key,
}, extrinsics.iter().cloned().collect())))
Ok(map)
})
.map(|pairs| pairs.into_iter().map(|(_, (k, v))| InputPair::ExtrinsicIndex(k, v)))
}

/// Prepare DigestIndex input pairs.
fn prepare_digest_input<'a, S, H, Number>(
parent: &'a AnchorBlockId<H::Out, Number>,
config: &Configuration,
block: Number,
storage: &'a S
) -> Result<impl Iterator<Item=InputPair<Number>> + 'a, String>
where
Expand All @@ -108,35 +123,52 @@ fn prepare_digest_input<'a, S, H, Number>(
H::Out: 'a,
Number: BlockNumber,
{
let mut digest_map = BTreeMap::<Vec<u8>, BTreeSet<Number>>::new();
for digest_build_block in digest_build_iterator(config, parent.number.clone() + One::one()) {
let trie_root = storage.root(parent, digest_build_block.clone())?;
let trie_root = trie_root.ok_or_else(|| format!("No changes trie root for block {}", digest_build_block.clone()))?;
let trie_storage = TrieBackendEssence::<_, H>::new(
crate::changes_trie::TrieBackendStorageAdapter(storage),
trie_root,
);
digest_build_iterator(config, block.clone())
.try_fold(BTreeMap::new(), move |mut map, digest_build_block| {
let trie_root = storage.root(parent, digest_build_block.clone())?;
let trie_root = trie_root.ok_or_else(|| format!("No changes trie root for block {}", digest_build_block.clone()))?;
let trie_storage = TrieBackendEssence::<_, H>::new(
crate::changes_trie::TrieBackendStorageAdapter(storage),
trie_root,
);

let extrinsic_prefix = ExtrinsicIndex::key_neutral_prefix(digest_build_block.clone());
trie_storage.for_keys_with_prefix(&extrinsic_prefix, |key|
if let Some(InputKey::ExtrinsicIndex::<Number>(trie_key)) = Decode::decode(&mut &key[..]) {
digest_map.entry(trie_key.key).or_default()
.insert(digest_build_block.clone());
});
let mut insert_to_map = |key: Vec<u8>| {
match map.entry(key.clone()) {
Entry::Vacant(entry) => {
entry.insert((DigestIndex {
block: block.clone(),
key,
}, vec![digest_build_block.clone()]));
},
Entry::Occupied(mut entry) => {
// DigestIndexValue must be sorted. Here we are relying on the fact that digest_build_iterator()
// returns blocks in ascending order => we only need to check for duplicates
//
// is_dup_block could be true when key has been changed in both digest block
// AND other blocks that it covers
let is_dup_block = entry.get().1.last() == Some(&digest_build_block);
if !is_dup_block {
entry.get_mut().1.push(digest_build_block.clone());
}
},
}
};

let digest_prefix = DigestIndex::key_neutral_prefix(digest_build_block.clone());
trie_storage.for_keys_with_prefix(&digest_prefix, |key|
if let Some(InputKey::DigestIndex::<Number>(trie_key)) = Decode::decode(&mut &key[..]) {
digest_map.entry(trie_key.key).or_default()
.insert(digest_build_block.clone());
});
}
let extrinsic_prefix = ExtrinsicIndex::key_neutral_prefix(digest_build_block.clone());
trie_storage.for_keys_with_prefix(&extrinsic_prefix, |key|
if let Some(InputKey::ExtrinsicIndex::<Number>(trie_key)) = Decode::decode(&mut &key[..]) {
insert_to_map(trie_key.key);
});

Ok(digest_map.into_iter()
.map(move |(key, set)| InputPair::DigestIndex(DigestIndex {
block: parent.number.clone() + One::one(),
key
}, set.into_iter().collect())))
let digest_prefix = DigestIndex::key_neutral_prefix(digest_build_block.clone());
trie_storage.for_keys_with_prefix(&digest_prefix, |key|
if let Some(InputKey::DigestIndex::<Number>(trie_key)) = Decode::decode(&mut &key[..]) {
insert_to_map(trie_key.key);
});

Ok(map)
})
.map(|pairs| pairs.into_iter().map(|(_, (k, v))| InputPair::DigestIndex(k, v)))
}

#[cfg(test)]
Expand Down Expand Up @@ -227,32 +259,34 @@ mod test {
fn build_changes_trie_nodes_on_non_digest_block() {
let (backend, storage, changes) = prepare_for_build();
let config = changes.changes_trie_config.as_ref().unwrap();
let parent = AnchorBlockId { hash: Default::default(), number: 4 };
let changes_trie_nodes = prepare_input(
&backend,
&storage,
config,
&changes,
&AnchorBlockId { hash: Default::default(), number: 4 },
&parent,
).unwrap();
assert_eq!(changes_trie_nodes, Some(vec![
assert_eq!(changes_trie_nodes.collect::<Vec<InputPair<u64>>>(), vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 5, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 5, key: vec![101] }, vec![1]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 5, key: vec![103] }, vec![0, 1]),
]));
]);
}

#[test]
fn build_changes_trie_nodes_on_digest_block_l1() {
let (backend, storage, changes) = prepare_for_build();
let config = changes.changes_trie_config.as_ref().unwrap();
let parent = AnchorBlockId { hash: Default::default(), number: 3 };
let changes_trie_nodes = prepare_input(
&backend,
&storage,
config,
&changes,
&AnchorBlockId { hash: Default::default(), number: 3 },
&parent,
).unwrap();
assert_eq!(changes_trie_nodes, Some(vec![
assert_eq!(changes_trie_nodes.collect::<Vec<InputPair<u64>>>(), vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![101] }, vec![1]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![103] }, vec![0, 1]),
Expand All @@ -261,21 +295,22 @@ mod test {
InputPair::DigestIndex(DigestIndex { block: 4, key: vec![101] }, vec![1]),
InputPair::DigestIndex(DigestIndex { block: 4, key: vec![102] }, vec![2]),
InputPair::DigestIndex(DigestIndex { block: 4, key: vec![105] }, vec![1, 3]),
]));
]);
}

#[test]
fn build_changes_trie_nodes_on_digest_block_l2() {
let (backend, storage, changes) = prepare_for_build();
let config = changes.changes_trie_config.as_ref().unwrap();
let parent = AnchorBlockId { hash: Default::default(), number: 15 };
let changes_trie_nodes = prepare_input(
&backend,
&storage,
config,
&changes,
&AnchorBlockId { hash: Default::default(), number: 15 },
&parent,
).unwrap();
assert_eq!(changes_trie_nodes, Some(vec![
assert_eq!(changes_trie_nodes.collect::<Vec<InputPair<u64>>>(), vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![101] }, vec![1]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![103] }, vec![0, 1]),
Expand All @@ -285,7 +320,7 @@ mod test {
InputPair::DigestIndex(DigestIndex { block: 16, key: vec![102] }, vec![4]),
InputPair::DigestIndex(DigestIndex { block: 16, key: vec![103] }, vec![4]),
InputPair::DigestIndex(DigestIndex { block: 16, key: vec![105] }, vec![4, 8]),
]));
]);
}

#[test]
Expand All @@ -299,14 +334,15 @@ mod test {
});

let config = changes.changes_trie_config.as_ref().unwrap();
let parent = AnchorBlockId { hash: Default::default(), number: 3 };
let changes_trie_nodes = prepare_input(
&backend,
&storage,
config,
&changes,
&AnchorBlockId { hash: Default::default(), number: 3 },
&parent,
).unwrap();
assert_eq!(changes_trie_nodes, Some(vec![
assert_eq!(changes_trie_nodes.collect::<Vec<InputPair<u64>>>(), vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![101] }, vec![1]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![103] }, vec![0, 1]),
Expand All @@ -315,6 +351,6 @@ mod test {
InputPair::DigestIndex(DigestIndex { block: 4, key: vec![101] }, vec![1]),
InputPair::DigestIndex(DigestIndex { block: 4, key: vec![102] }, vec![2]),
InputPair::DigestIndex(DigestIndex { block: 4, key: vec![105] }, vec![1, 3]),
]));
]);
}
}
Loading