Skip to content
This repository has been archived by the owner on Nov 15, 2023. It is now read-only.

Commit

Permalink
Some changes tries optimizations (#2840)
Browse files Browse the repository at this point in the history 
* changes tries initial optimizations

* line width
  • Loading branch information
@svyatonik @gavofyork
svyatonik authored and gavofyork committed Jul 16, 2019
1 parent ca9f9e5 commit f7489f0
Show file tree
Hide file tree
Showing 6 changed files with 166 additions and 131 deletions.
194 changes: 115 additions & 79 deletions core/state-machine/src/changes_trie/build.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@

//! Structures and functions required to build changes trie for given block.
use std::collections::{BTreeMap, BTreeSet};
use std::collections::BTreeMap;
use std::collections::btree_map::Entry;
use parity_codec::Decode;
use hash_db::Hasher;
use num_traits::One;
Expand All @@ -31,75 +32,89 @@ use crate::changes_trie::{AnchorBlockId, Configuration, Storage, BlockNumber};
///
/// Returns Err if storage error has occurred OR if storage haven't returned
/// required data.
/// Returns Ok(None) data required to prepare input pairs is not collected
/// or storage is not provided.
pub fn prepare_input<'a, B, S, H, Number>(
backend: &B,
backend: &'a B,
storage: &'a S,
config: &'a Configuration,
changes: &OverlayedChanges,
changes: &'a OverlayedChanges,
parent: &'a AnchorBlockId<H::Out, Number>,
) -> Result<Option<Vec<InputPair<Number>>>, String>
) -> Result<impl Iterator<Item=InputPair<Number>> + 'a, String>
where
B: Backend<H>,
S: Storage<H, Number>,
H: Hasher,
H: Hasher + 'a,
Number: BlockNumber,
{
let mut input = Vec::new();
input.extend(prepare_extrinsics_input(
let number = parent.number.clone() + One::one();
let extrinsics_input = prepare_extrinsics_input(
backend,
parent.number.clone() + 1.into(),
changes)?);
input.extend(prepare_digest_input::<_, H, Number>(
&number,
changes)?;
let digest_input = prepare_digest_input::<_, H, Number>(
parent,
config,
storage)?);

Ok(Some(input))
number,
storage)?;
Ok(extrinsics_input.chain(digest_input))
}

/// Prepare ExtrinsicIndex input pairs.
fn prepare_extrinsics_input<B, H, Number>(
backend: &B,
block: Number,
changes: &OverlayedChanges,
) -> Result<impl Iterator<Item=InputPair<Number>>, String>
fn prepare_extrinsics_input<'a, B, H, Number>(
backend: &'a B,
block: &Number,
changes: &'a OverlayedChanges,
) -> Result<impl Iterator<Item=InputPair<Number>> + 'a, String>
where
B: Backend<H>,
H: Hasher,
Number: BlockNumber,
{
let mut extrinsic_map = BTreeMap::<Vec<u8>, BTreeSet<u32>>::new();
for (key, val) in changes.prospective.top.iter().chain(changes.committed.top.iter()) {
let extrinsics = match val.extrinsics {
Some(ref extrinsics) => extrinsics,
None => continue,
};
changes.committed.top.iter()
.chain(changes.prospective.top.iter())
.filter(|( _, v)| v.extrinsics.is_some())
.try_fold(BTreeMap::new(), |mut map: BTreeMap<&[u8], (ExtrinsicIndex<Number>, Vec<u32>)>, (k, v)| {
match map.entry(k) {
Entry::Vacant(entry) => {
// ignore temporary values (values that have null value at the end of operation
// AND are not in storage at the beginning of operation
if !changes.storage(k).map(|v| v.is_some()).unwrap_or_default() {
if !backend.exists_storage(k).map_err(|e| format!("{}", e))? {
return Ok(map);
}
}

// ignore values that have null value at the end of operation AND are not in storage
// at the beginning of operation
if !changes.storage(key).map(|v| v.is_some()).unwrap_or_default() {
if !backend.exists_storage(key).map_err(|e| format!("{}", e))? {
continue;
let extrinsics = v.extrinsics.as_ref()
.expect("filtered by filter() call above; qed")
.iter().cloned().collect();
entry.insert((ExtrinsicIndex {
block: block.clone(),
key: k.to_vec(),
}, extrinsics));
},
Entry::Occupied(mut entry) => {
// we do not need to check for temporary values here, because entry is Occupied
// AND we are checking it before insertion
let extrinsics = &mut entry.get_mut().1;
extrinsics.extend(
v.extrinsics.as_ref()
.expect("filtered by filter() call above; qed")
.iter()
.cloned()
);
extrinsics.sort_unstable();
},
}
}

extrinsic_map.entry(key.clone()).or_default()
.extend(extrinsics.iter().cloned());
}

Ok(extrinsic_map.into_iter()
.map(move |(key, extrinsics)| InputPair::ExtrinsicIndex(ExtrinsicIndex {
block: block.clone(),
key,
}, extrinsics.iter().cloned().collect())))
Ok(map)
})
.map(|pairs| pairs.into_iter().map(|(_, (k, v))| InputPair::ExtrinsicIndex(k, v)))
}

/// Prepare DigestIndex input pairs.
fn prepare_digest_input<'a, S, H, Number>(
parent: &'a AnchorBlockId<H::Out, Number>,
config: &Configuration,
block: Number,
storage: &'a S
) -> Result<impl Iterator<Item=InputPair<Number>> + 'a, String>
where
Expand All @@ -108,35 +123,52 @@ fn prepare_digest_input<'a, S, H, Number>(
H::Out: 'a,
Number: BlockNumber,
{
let mut digest_map = BTreeMap::<Vec<u8>, BTreeSet<Number>>::new();
for digest_build_block in digest_build_iterator(config, parent.number.clone() + One::one()) {
let trie_root = storage.root(parent, digest_build_block.clone())?;
let trie_root = trie_root.ok_or_else(|| format!("No changes trie root for block {}", digest_build_block.clone()))?;
let trie_storage = TrieBackendEssence::<_, H>::new(
crate::changes_trie::TrieBackendStorageAdapter(storage),
trie_root,
);
digest_build_iterator(config, block.clone())
.try_fold(BTreeMap::new(), move |mut map, digest_build_block| {
let trie_root = storage.root(parent, digest_build_block.clone())?;
let trie_root = trie_root.ok_or_else(|| format!("No changes trie root for block {}", digest_build_block.clone()))?;
let trie_storage = TrieBackendEssence::<_, H>::new(
crate::changes_trie::TrieBackendStorageAdapter(storage),
trie_root,
);

let extrinsic_prefix = ExtrinsicIndex::key_neutral_prefix(digest_build_block.clone());
trie_storage.for_keys_with_prefix(&extrinsic_prefix, |key|
if let Some(InputKey::ExtrinsicIndex::<Number>(trie_key)) = Decode::decode(&mut &key[..]) {
digest_map.entry(trie_key.key).or_default()
.insert(digest_build_block.clone());
});
let mut insert_to_map = |key: Vec<u8>| {
match map.entry(key.clone()) {
Entry::Vacant(entry) => {
entry.insert((DigestIndex {
block: block.clone(),
key,
}, vec![digest_build_block.clone()]));
},
Entry::Occupied(mut entry) => {
// DigestIndexValue must be sorted. Here we are relying on the fact that digest_build_iterator()
// returns blocks in ascending order => we only need to check for duplicates
//
// is_dup_block could be true when key has been changed in both digest block
// AND other blocks that it covers
let is_dup_block = entry.get().1.last() == Some(&digest_build_block);
if !is_dup_block {
entry.get_mut().1.push(digest_build_block.clone());
}
},
}
};

let digest_prefix = DigestIndex::key_neutral_prefix(digest_build_block.clone());
trie_storage.for_keys_with_prefix(&digest_prefix, |key|
if let Some(InputKey::DigestIndex::<Number>(trie_key)) = Decode::decode(&mut &key[..]) {
digest_map.entry(trie_key.key).or_default()
.insert(digest_build_block.clone());
});
}
let extrinsic_prefix = ExtrinsicIndex::key_neutral_prefix(digest_build_block.clone());
trie_storage.for_keys_with_prefix(&extrinsic_prefix, |key|
if let Some(InputKey::ExtrinsicIndex::<Number>(trie_key)) = Decode::decode(&mut &key[..]) {
insert_to_map(trie_key.key);
});

Ok(digest_map.into_iter()
.map(move |(key, set)| InputPair::DigestIndex(DigestIndex {
block: parent.number.clone() + One::one(),
key
}, set.into_iter().collect())))
let digest_prefix = DigestIndex::key_neutral_prefix(digest_build_block.clone());
trie_storage.for_keys_with_prefix(&digest_prefix, |key|
if let Some(InputKey::DigestIndex::<Number>(trie_key)) = Decode::decode(&mut &key[..]) {
insert_to_map(trie_key.key);
});

Ok(map)
})
.map(|pairs| pairs.into_iter().map(|(_, (k, v))| InputPair::DigestIndex(k, v)))
}

#[cfg(test)]
Expand Down Expand Up @@ -227,32 +259,34 @@ mod test {
fn build_changes_trie_nodes_on_non_digest_block() {
let (backend, storage, changes) = prepare_for_build();
let config = changes.changes_trie_config.as_ref().unwrap();
let parent = AnchorBlockId { hash: Default::default(), number: 4 };
let changes_trie_nodes = prepare_input(
&backend,
&storage,
config,
&changes,
&AnchorBlockId { hash: Default::default(), number: 4 },
&parent,
).unwrap();
assert_eq!(changes_trie_nodes, Some(vec![
assert_eq!(changes_trie_nodes.collect::<Vec<InputPair<u64>>>(), vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 5, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 5, key: vec![101] }, vec![1]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 5, key: vec![103] }, vec![0, 1]),
]));
]);
}

#[test]
fn build_changes_trie_nodes_on_digest_block_l1() {
let (backend, storage, changes) = prepare_for_build();
let config = changes.changes_trie_config.as_ref().unwrap();
let parent = AnchorBlockId { hash: Default::default(), number: 3 };
let changes_trie_nodes = prepare_input(
&backend,
&storage,
config,
&changes,
&AnchorBlockId { hash: Default::default(), number: 3 },
&parent,
).unwrap();
assert_eq!(changes_trie_nodes, Some(vec![
assert_eq!(changes_trie_nodes.collect::<Vec<InputPair<u64>>>(), vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![101] }, vec![1]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![103] }, vec![0, 1]),
Expand All @@ -261,21 +295,22 @@ mod test {
InputPair::DigestIndex(DigestIndex { block: 4, key: vec![101] }, vec![1]),
InputPair::DigestIndex(DigestIndex { block: 4, key: vec![102] }, vec![2]),
InputPair::DigestIndex(DigestIndex { block: 4, key: vec![105] }, vec![1, 3]),
]));
]);
}

#[test]
fn build_changes_trie_nodes_on_digest_block_l2() {
let (backend, storage, changes) = prepare_for_build();
let config = changes.changes_trie_config.as_ref().unwrap();
let parent = AnchorBlockId { hash: Default::default(), number: 15 };
let changes_trie_nodes = prepare_input(
&backend,
&storage,
config,
&changes,
&AnchorBlockId { hash: Default::default(), number: 15 },
&parent,
).unwrap();
assert_eq!(changes_trie_nodes, Some(vec![
assert_eq!(changes_trie_nodes.collect::<Vec<InputPair<u64>>>(), vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![101] }, vec![1]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![103] }, vec![0, 1]),
Expand All @@ -285,7 +320,7 @@ mod test {
InputPair::DigestIndex(DigestIndex { block: 16, key: vec![102] }, vec![4]),
InputPair::DigestIndex(DigestIndex { block: 16, key: vec![103] }, vec![4]),
InputPair::DigestIndex(DigestIndex { block: 16, key: vec![105] }, vec![4, 8]),
]));
]);
}

#[test]
Expand All @@ -299,14 +334,15 @@ mod test {
});

let config = changes.changes_trie_config.as_ref().unwrap();
let parent = AnchorBlockId { hash: Default::default(), number: 3 };
let changes_trie_nodes = prepare_input(
&backend,
&storage,
config,
&changes,
&AnchorBlockId { hash: Default::default(), number: 3 },
&parent,
).unwrap();
assert_eq!(changes_trie_nodes, Some(vec![
assert_eq!(changes_trie_nodes.collect::<Vec<InputPair<u64>>>(), vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![101] }, vec![1]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![103] }, vec![0, 1]),
Expand All @@ -315,6 +351,6 @@ mod test {
InputPair::DigestIndex(DigestIndex { block: 4, key: vec![101] }, vec![1]),
InputPair::DigestIndex(DigestIndex { block: 4, key: vec![102] }, vec![2]),
InputPair::DigestIndex(DigestIndex { block: 4, key: vec![105] }, vec![1, 3]),
]));
]);
}
}
Loading

0 comments on commit f7489f0

Please sign in to comment.