From 3f35f2769c907a08dea48a3a15f3da2a60e7dfd8 Mon Sep 17 00:00:00 2001 From: firatNEAR Date: Tue, 16 Aug 2022 20:46:11 +0200 Subject: [PATCH 01/37] Introduce Storage Metrics for Prometheus --- core/store/src/metrics.rs | 64 ++++++++++++++++++++++++++++- core/store/src/trie/shard_tries.rs | 5 ++- core/store/src/trie/trie_storage.rs | 47 ++++++++++++++++++--- core/store/src/trie/trie_tests.rs | 15 ++++--- 4 files changed, 118 insertions(+), 13 deletions(-) diff --git a/core/store/src/metrics.rs b/core/store/src/metrics.rs index e677530af21..da2973151bb 100644 --- a/core/store/src/metrics.rs +++ b/core/store/src/metrics.rs @@ -1,4 +1,7 @@ -use near_metrics::{try_create_histogram_vec, HistogramVec}; +use near_metrics::{ + try_create_histogram_vec, try_create_int_counter_vec, try_create_int_gauge_vec, HistogramVec, + IntCounterVec, IntGaugeVec, +}; use once_cell::sync::Lazy; pub(crate) static DATABASE_OP_LATENCY_HIST: Lazy = Lazy::new(|| { @@ -10,3 +13,62 @@ pub(crate) static DATABASE_OP_LATENCY_HIST: Lazy = Lazy::new(|| { ) .unwrap() }); + +pub static CHUNK_CACHE_HITS: Lazy = Lazy::new(|| { + try_create_int_counter_vec( + "near_chunk_cache_hits", + "Chunk cache hits", + &["shard_id", "is_view"], + ) + .unwrap() +}); + +pub static CHUNK_CACHE_MISSES: Lazy = Lazy::new(|| { + try_create_int_counter_vec( + "near_chunk_cache_misses", + "Chunk cache misses", + &["shard_id", "is_view"], + ) + .unwrap() +}); + +pub static SHARD_CACHE_HITS: Lazy = Lazy::new(|| { + try_create_int_counter_vec( + "near_shard_cache_hits", + "Shard cache hits", + &["shard_id", "is_view"], + ) + .unwrap() +}); + +pub static SHARD_CACHE_MISSES: Lazy = Lazy::new(|| { + try_create_int_counter_vec( + "near_shard_cache_misses", + "Shard cache misses", + &["shard_id", "is_view"], + ) + .unwrap() +}); + +pub static SHARD_CACHE_TOO_LARGE: Lazy = Lazy::new(|| { + try_create_int_counter_vec( + "near_shard_cache_too_large", + "Shard cache too large", + &["shard_id", "is_view"], + ) + .unwrap() +}); + +pub static SHARD_CACHE_SIZE: Lazy = Lazy::new(|| { + try_create_int_gauge_vec("near_shard_cache_size", "Shard cache size", &["shard_id", "is_view"]) + .unwrap() +}); + +pub static CHUNK_CACHE_SIZE: Lazy = Lazy::new(|| { + try_create_int_gauge_vec("near_chunk_cache_size", "Chunk cache size", &["shard_id", "is_view"]) + .unwrap() +}); + +pub static SHARD_CACHE_POPS: Lazy = Lazy::new(|| { + try_create_int_counter_vec("near_shard_cache_pops", "Shard cache pops", &["shard_id"]).unwrap() +}); diff --git a/core/store/src/trie/shard_tries.rs b/core/store/src/trie/shard_tries.rs index 25ec14beeef..24f656665bd 100644 --- a/core/store/src/trie/shard_tries.rs +++ b/core/store/src/trie/shard_tries.rs @@ -115,7 +115,8 @@ impl ShardTries { .or_insert_with(|| self.0.trie_cache_factory.create_cache(&shard_uid)) .clone() }; - let storage = Box::new(TrieCachingStorage::new(self.0.store.clone(), cache, shard_uid)); + let storage = + Box::new(TrieCachingStorage::new(self.0.store.clone(), cache, shard_uid, is_view)); let flat_state = { #[cfg(feature = "protocol_feature_flat_state")] if use_flat_state { @@ -179,7 +180,7 @@ impl ShardTries { .entry(shard_uid) .or_insert_with(|| self.0.trie_cache_factory.create_cache(&shard_uid)) .clone(); - cache.update_cache(ops); + cache.update_cache(ops, shard_uid); } Ok(()) } diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index 04069072cbc..55f2f30949f 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -6,7 +6,7 @@ use near_primitives::hash::CryptoHash; use crate::db::refcount::decode_value_with_rc; use crate::trie::POISONED_LOCK_ERR; -use crate::{DBCol, StorageError, Store}; +use crate::{metrics, DBCol, StorageError, Store}; use lru::LruCache; use near_primitives::shard_layout::ShardUId; use near_primitives::types::{TrieCacheMode, TrieNodesCount}; @@ -34,7 +34,10 @@ impl TrieCache { self.0.lock().expect(POISONED_LOCK_ERR).clear() } - pub fn update_cache(&self, ops: Vec<(CryptoHash, Option<&Vec>)>) { + pub fn update_cache(&self, ops: Vec<(CryptoHash, Option<&Vec>)>, shard_uid: ShardUId) { + let shard_id_str = format!("{}", shard_uid.shard_id); + let labels: [&str; 1] = [&shard_id_str]; + let mut guard = self.0.lock().expect(POISONED_LOCK_ERR); for (hash, opt_value_rc) in ops { if let Some(value_rc) = opt_value_rc { @@ -43,10 +46,20 @@ impl TrieCache { guard.put(hash, value.into()); } } else { - guard.pop(&hash); + match guard.pop(&hash) { + Some(_) => { + metrics::SHARD_CACHE_POPS.with_label_values(&labels).inc(); + } + _ => {} + }; } } else { - guard.pop(&hash); + match guard.pop(&hash) { + Some(_) => { + metrics::SHARD_CACHE_POPS.with_label_values(&labels).inc(); + } + _ => {} + }; } } } @@ -179,10 +192,17 @@ pub struct TrieCachingStorage { pub(crate) db_read_nodes: Cell, /// Counts trie nodes retrieved from the chunk cache. pub(crate) mem_read_nodes: Cell, + /// Boolean for determining if the cache is a view cache or not + is_view: bool, } impl TrieCachingStorage { - pub fn new(store: Store, shard_cache: TrieCache, shard_uid: ShardUId) -> TrieCachingStorage { + pub fn new( + store: Store, + shard_cache: TrieCache, + shard_uid: ShardUId, + is_view: bool, + ) -> TrieCachingStorage { TrieCachingStorage { store, shard_uid, @@ -191,6 +211,7 @@ impl TrieCachingStorage { chunk_cache: RefCell::new(Default::default()), db_read_nodes: Cell::new(0), mem_read_nodes: Cell::new(0), + is_view, } } @@ -231,21 +252,36 @@ impl TrieCachingStorage { impl TrieStorage for TrieCachingStorage { fn retrieve_raw_bytes(&self, hash: &CryptoHash) -> Result, StorageError> { + let shard_id_str = format!("{}", self.shard_uid.shard_id); + let is_view_str = format!("{}", self.is_view as u8); + let labels: [&str; 2] = [&shard_id_str, &is_view_str]; + { + metrics::CHUNK_CACHE_SIZE + .with_label_values(&labels) + .set(self.chunk_cache.borrow().len() as i64); + metrics::SHARD_CACHE_SIZE + .with_label_values(&labels) + .set(self.shard_cache.0.lock().expect(POISONED_LOCK_ERR).len() as i64); + } // Try to get value from chunk cache containing nodes with cheaper access. We can do it for any `TrieCacheMode`, // because we charge for reading nodes only when `CachingChunk` mode is enabled anyway. if let Some(val) = self.chunk_cache.borrow_mut().get(hash) { + metrics::CHUNK_CACHE_HITS.with_label_values(&labels).inc(); self.inc_mem_read_nodes(); return Ok(val.clone()); } + metrics::CHUNK_CACHE_MISSES.with_label_values(&labels).inc(); // Try to get value from shard cache containing most recently touched nodes. let mut guard = self.shard_cache.0.lock().expect(POISONED_LOCK_ERR); let val = match guard.get(hash) { Some(val) => { + metrics::SHARD_CACHE_HITS.with_label_values(&labels).inc(); near_o11y::io_trace!(count: "shard_cache_hit"); val.clone() } None => { + metrics::SHARD_CACHE_MISSES.with_label_values(&labels).inc(); near_o11y::io_trace!(count: "shard_cache_miss"); // If value is not present in cache, get it from the storage. let key = Self::get_key_from_shard_uid_and_hash(self.shard_uid, hash); @@ -265,6 +301,7 @@ impl TrieStorage for TrieCachingStorage { if val.len() < TRIE_LIMIT_CACHED_VALUE_SIZE { guard.put(*hash, val.clone()); } else { + metrics::SHARD_CACHE_TOO_LARGE.with_label_values(&labels).inc(); near_o11y::io_trace!(count: "shard_cache_too_large"); } diff --git a/core/store/src/trie/trie_tests.rs b/core/store/src/trie/trie_tests.rs index 9abe817a0c8..03f9782b348 100644 --- a/core/store/src/trie/trie_tests.rs +++ b/core/store/src/trie/trie_tests.rs @@ -235,7 +235,8 @@ mod caching_storage_tests { let shard_uid = ShardUId::single_shard(); let store = create_store_with_values(&values, shard_uid); let trie_cache = TrieCache::new(); - let trie_caching_storage = TrieCachingStorage::new(store, trie_cache.clone(), shard_uid); + let trie_caching_storage = + TrieCachingStorage::new(store, trie_cache.clone(), shard_uid, false); let key = hash(&value); assert_eq!(trie_cache.get(&key), None); @@ -255,7 +256,8 @@ mod caching_storage_tests { fn test_retrieve_error() { let shard_uid = ShardUId::single_shard(); let store = create_test_store(); - let trie_caching_storage = TrieCachingStorage::new(store, TrieCache::new(), shard_uid); + let trie_caching_storage = + TrieCachingStorage::new(store, TrieCache::new(), shard_uid, false); let value = vec![1u8]; let key = hash(&value); @@ -271,7 +273,8 @@ mod caching_storage_tests { let shard_uid = ShardUId::single_shard(); let store = create_store_with_values(&values, shard_uid); let trie_cache = TrieCache::new(); - let trie_caching_storage = TrieCachingStorage::new(store, trie_cache.clone(), shard_uid); + let trie_caching_storage = + TrieCachingStorage::new(store, trie_cache.clone(), shard_uid, false); let key = hash(&value); trie_caching_storage.set_mode(TrieCacheMode::CachingChunk); @@ -293,7 +296,8 @@ mod caching_storage_tests { let shard_uid = ShardUId::single_shard(); let store = create_store_with_values(&values, shard_uid); let trie_cache = TrieCache::new(); - let trie_caching_storage = TrieCachingStorage::new(store, trie_cache.clone(), shard_uid); + let trie_caching_storage = + TrieCachingStorage::new(store, trie_cache.clone(), shard_uid, false); let value = &values[0]; let key = hash(&value); @@ -341,7 +345,8 @@ mod caching_storage_tests { let shard_uid = ShardUId::single_shard(); let store = create_store_with_values(&values, shard_uid); let trie_cache = TrieCache::with_capacity(shard_cache_size); - let trie_caching_storage = TrieCachingStorage::new(store, trie_cache.clone(), shard_uid); + let trie_caching_storage = + TrieCachingStorage::new(store, trie_cache.clone(), shard_uid, false); let value = &values[0]; let key = hash(&value); From 9aa35a7827a20de6bfb5564b40f97f7a3fa79187 Mon Sep 17 00:00:00 2001 From: Longarithm Date: Wed, 17 Aug 2022 20:43:50 +0400 Subject: [PATCH 02/37] new shard cache --- chain/chain/src/store.rs | 10 +++- core/store/src/lib.rs | 19 ++++--- core/store/src/trie/shard_tries.rs | 4 ++ core/store/src/trie/trie_storage.rs | 87 +++++++++++++++++++++++++++-- 4 files changed, 107 insertions(+), 13 deletions(-) diff --git a/chain/chain/src/store.rs b/chain/chain/src/store.rs index 8b8e3b49ab1..9ea30e8a0dd 100644 --- a/chain/chain/src/store.rs +++ b/chain/chain/src/store.rs @@ -1818,8 +1818,16 @@ impl<'a> ChainStoreUpdate<'a> { self.chain_store_cache_update.outcome_ids.insert((*block_hash, shard_id), outcome_ids); } - pub fn save_trie_changes(&mut self, trie_changes: WrappedTrieChanges) { + // fix me: catch error everywhere + pub fn save_trie_changes(&mut self, trie_changes: WrappedTrieChanges) -> Result<(), Error> { + // Pop deleted trie nodes from the shard cache. + // TODO: get rid of creating `StoreUpdate` + let mut store_update = self.store().store_update(); + trie_changes.deletions_into(&mut store_update); + store_update.update_cache()?; + self.trie_changes.push(trie_changes); + Ok(()) } pub fn add_state_changes_for_split_states( diff --git a/core/store/src/lib.rs b/core/store/src/lib.rs index eed1aab1cd5..8768690fb04 100644 --- a/core/store/src/lib.rs +++ b/core/store/src/lib.rs @@ -356,6 +356,17 @@ impl StoreUpdate { self.transaction.merge(other.transaction) } + pub fn update_cache(&self) -> io::Result<()> { + if let Some(tries) = &self.shard_tries { + // Note: avoid comparing wide pointers here to work-around + // https://github.com/rust-lang/rust/issues/69757 + let addr = |arc| Arc::as_ptr(arc) as *const u8; + assert_eq!(addr(&tries.get_store().storage), addr(&self.storage),); + tries.update_cache(&self.transaction)?; + } + Ok(()) + } + pub fn commit(self) -> io::Result<()> { debug_assert!( { @@ -376,13 +387,7 @@ impl StoreUpdate { "Transaction overwrites itself: {:?}", self ); - if let Some(tries) = self.shard_tries { - // Note: avoid comparing wide pointers here to work-around - // https://github.com/rust-lang/rust/issues/69757 - let addr = |arc| Arc::as_ptr(arc) as *const u8; - assert_eq!(addr(&tries.get_store().storage), addr(&self.storage),); - tries.update_cache(&self.transaction)?; - } + self.update_cache()?; let _span = tracing::trace_span!(target: "store", "commit").entered(); for op in &self.transaction.ops { match op { diff --git a/core/store/src/trie/shard_tries.rs b/core/store/src/trie/shard_tries.rs index 25ec14beeef..b2e46278599 100644 --- a/core/store/src/trie/shard_tries.rs +++ b/core/store/src/trie/shard_tries.rs @@ -325,6 +325,10 @@ impl WrappedTrieChanges { self.tries.apply_insertions(&self.trie_changes, self.shard_uid, store_update) } + pub fn deletions_into(&self, store_update: &mut StoreUpdate) { + self.tries.apply_deletions(&self.trie_changes, self.shard_uid, store_update) + } + /// Save state changes into Store. /// /// NOTE: the changes are drained from `self`. diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index 04069072cbc..c86df6bbf58 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -1,5 +1,5 @@ use std::borrow::Borrow; -use std::collections::{HashMap, HashSet}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::{Arc, Mutex}; use near_primitives::hash::CryptoHash; @@ -13,9 +13,86 @@ use near_primitives::types::{TrieCacheMode, TrieNodesCount}; use std::cell::{Cell, RefCell}; use std::io::ErrorKind; -/// Wrapper over LruCache which doesn't hold too large elements. +pub const DELETIONS_QUEUE_CAPACITY: usize = 100_000; + +/// In-memory cache for trie items - nodes and values. All nodes are stored in the LRU cache with three modifications. +/// 1) Size of each value must not exceed `TRIE_LIMIT_CACHED_VALUE_SIZE`. +/// Needed to avoid caching large values like contract codes. +/// 2) Summary size of all values should not exceed `total_sizes_capacity`. If new value doesn't fit, LRU values are +/// evicted until it fits. +/// Needed to bound cache capacity more precisely, because value sizes generally vary from 1 B to 500 B. +/// 3) If value is popped, it is put to the `deletions` queue with `DELETIONS_CACHE_CAPACITY` first. If popped value +/// doesn't fit in the queue, the last value is removed from the queue and LRU cache, and newly popped value is inserted +/// to the queue. +/// Needed to delay deletions when we have forks. In such case, many blocks can share same parent, and we want to keep +/// old nodes in cache for a while to process all new roots. For example, it helps to read old state root. +pub struct SyncTrieCache { + /// LRU cache keeping mapping from keys to values. + cache: LruCache>, + /// Queue of items which were popped, which postpones deletion of old nodes. + deletions: VecDeque, + /// Total size of all values in the cache. + total_size: u64, + /// Upper bound for the total size. + total_size_capacity: u64, +} + +impl SyncTrieCache { + pub fn new(cap: usize) -> Self { + Self { + cache: LruCache::new(cap), + deletions: VecDeque::with_capacity(DELETIONS_QUEUE_CAPACITY), + total_size: 0, + total_size_capacity: 4_500_000_000, + } + } + + pub fn get(&mut self, key: &CryptoHash) -> Option> { + self.cache.get(key).cloned() + } + + pub fn clear(&mut self) { + self.total_size = 0; + self.deletions.clear(); + self.cache.clear(); + } + + pub fn put(&mut self, key: CryptoHash, value: Arc<[u8]>) { + // We assume that value.len() is less than total size capacity. + // If adding new value exceeds capacity, we pop LRU values until it fits. + while self.total_size + value.len() as u64 > self.total_size_capacity { + let (_, evicted_value) = + self.cache.pop_lru().expect("Cannot fail because cap_lengths is > 0"); + self.total_size -= evicted_value.len() as u64; + } + // Add value to the cache. + self.total_size += value.len() as u64; + self.cache.put(key, value); + } + + pub fn pop(&mut self, key: &CryptoHash) { + // Do nothing if key was removed before. + if self.cache.contains(key) { + // Put key to the queue of deletions. If it exceeds queue capacity, key from queue tail is popped and the + // value from the LRU cache is removed. + if self.deletions.len() == DELETIONS_QUEUE_CAPACITY { + let key_to_remove = self.deletions.pop_front().expect("Deletions cannot be empty"); + match self.cache.pop(&key_to_remove) { + Some(evicted_value) => { + self.total_size -= evicted_value.len() as u64; + } + None => {} + } + } + + self.deletions.push_back(key.clone()); + } + } +} + +/// Wrapper over LruCache to handle concurrent access. #[derive(Clone)] -pub struct TrieCache(Arc>>>); +pub struct TrieCache(Arc>); impl TrieCache { pub fn new() -> Self { @@ -23,11 +100,11 @@ impl TrieCache { } pub fn with_capacity(cap: usize) -> Self { - Self(Arc::new(Mutex::new(LruCache::new(cap)))) + Self(Arc::new(Mutex::new(SyncTrieCache::new(cap)))) } pub fn get(&self, key: &CryptoHash) -> Option> { - self.0.lock().expect(POISONED_LOCK_ERR).get(key).cloned() + self.0.lock().expect(POISONED_LOCK_ERR).get(key) } pub fn clear(&self) { From a1e4a62c0e5229e56434e71d086aa97313fa0f25 Mon Sep 17 00:00:00 2001 From: Longarithm Date: Wed, 17 Aug 2022 20:46:16 +0400 Subject: [PATCH 03/37] minor fix --- core/store/src/trie/trie_storage.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index c86df6bbf58..7b34f281701 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -88,6 +88,11 @@ impl SyncTrieCache { self.deletions.push_back(key.clone()); } } + + #[cfg(test)] + pub fn len(&self) -> usize { + self.cache.len() + } } /// Wrapper over LruCache to handle concurrent access. From 8080aa39e39fa2c52fadbb766585ccec597b7a99 Mon Sep 17 00:00:00 2001 From: firatNEAR Date: Thu, 18 Aug 2022 14:40:29 +0200 Subject: [PATCH 04/37] Fix estimator --- runtime/runtime-params-estimator/src/estimator_context.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/runtime-params-estimator/src/estimator_context.rs b/runtime/runtime-params-estimator/src/estimator_context.rs index da06897003b..bf6d509aa21 100644 --- a/runtime/runtime-params-estimator/src/estimator_context.rs +++ b/runtime/runtime-params-estimator/src/estimator_context.rs @@ -124,7 +124,7 @@ impl<'c> Testbed<'c> { pub(crate) fn trie_caching_storage(&mut self) -> TrieCachingStorage { let store = self.inner.store(); let caching_storage = - TrieCachingStorage::new(store, TrieCache::new(), ShardUId::single_shard()); + TrieCachingStorage::new(store, TrieCache::new(), ShardUId::single_shard(), false); caching_storage } From adf8034724283815e8cb1f869b20cf8fcaece4d0 Mon Sep 17 00:00:00 2001 From: Longarithm Date: Fri, 19 Aug 2022 00:15:14 +0400 Subject: [PATCH 05/37] warning fixes --- chain/chain/src/chain.rs | 10 +++++----- chain/chain/src/tests/gc.rs | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/chain/chain/src/chain.rs b/chain/chain/src/chain.rs index c1e2791d2bd..24b0069243c 100644 --- a/chain/chain/src/chain.rs +++ b/chain/chain/src/chain.rs @@ -4533,7 +4533,7 @@ impl<'a> ChainUpdate<'a> { &result.shard_uid, new_chunk_extra, ); - self.chain_store_update.save_trie_changes(result.trie_changes); + self.chain_store_update.save_trie_changes(result.trie_changes)?; } assert_eq!(sum_gas_used, total_gas_used); assert_eq!(sum_balance_burnt, total_balance_burnt); @@ -4580,7 +4580,7 @@ impl<'a> ChainUpdate<'a> { apply_result.total_balance_burnt, ), ); - self.chain_store_update.save_trie_changes(apply_result.trie_changes); + self.chain_store_update.save_trie_changes(apply_result.trie_changes)?; self.chain_store_update.save_outgoing_receipt( &block_hash, shard_id, @@ -4614,7 +4614,7 @@ impl<'a> ChainUpdate<'a> { *new_extra.state_root_mut() = apply_result.new_root; self.chain_store_update.save_chunk_extra(&block_hash, &shard_uid, new_extra); - self.chain_store_update.save_trie_changes(apply_result.trie_changes); + self.chain_store_update.save_trie_changes(apply_result.trie_changes)?; if let Some(apply_results_or_state_changes) = apply_split_result_or_state_changes { self.process_split_state( @@ -4966,7 +4966,7 @@ impl<'a> ChainUpdate<'a> { self.chain_store_update.save_chunk(chunk); - self.chain_store_update.save_trie_changes(apply_result.trie_changes); + self.chain_store_update.save_trie_changes(apply_result.trie_changes)?; let chunk_extra = ChunkExtra::new( &apply_result.new_root, outcome_root, @@ -5045,7 +5045,7 @@ impl<'a> ChainUpdate<'a> { Default::default(), )?; - self.chain_store_update.save_trie_changes(apply_result.trie_changes); + self.chain_store_update.save_trie_changes(apply_result.trie_changes)?; let mut new_chunk_extra = ChunkExtra::clone(&chunk_extra); *new_chunk_extra.state_root_mut() = apply_result.new_root; diff --git a/chain/chain/src/tests/gc.rs b/chain/chain/src/tests/gc.rs index 2c347916579..62b061a5a68 100644 --- a/chain/chain/src/tests/gc.rs +++ b/chain/chain/src/tests/gc.rs @@ -137,7 +137,7 @@ fn do_fork( Default::default(), *block.hash(), ); - store_update.save_trie_changes(wrapped_trie_changes); + store_update.save_trie_changes(wrapped_trie_changes).unwrap(); prev_state_roots[shard_id as usize] = new_root; trie_changes_shards.push(trie_changes_data); From a139decceffcdfe6245f9c33df4a77b38bbc7cc8 Mon Sep 17 00:00:00 2001 From: Longarithm Date: Fri, 19 Aug 2022 01:52:16 +0400 Subject: [PATCH 06/37] separate deletions queue struct, fix push bug --- core/store/src/trie/shard_tries.rs | 2 +- core/store/src/trie/trie_storage.rs | 133 ++++++++++++++++++++-------- core/store/src/trie/trie_tests.rs | 6 +- 3 files changed, 100 insertions(+), 41 deletions(-) diff --git a/core/store/src/trie/shard_tries.rs b/core/store/src/trie/shard_tries.rs index b2e46278599..4168dba49fd 100644 --- a/core/store/src/trie/shard_tries.rs +++ b/core/store/src/trie/shard_tries.rs @@ -44,7 +44,7 @@ impl TrieCacheFactory { /// Create new cache for the given shard uid. pub fn create_cache(&self, shard_uid: &ShardUId) -> TrieCache { match self.capacities.get(shard_uid) { - Some(capacity) => TrieCache::with_capacity(*capacity), + Some(capacity) => TrieCache::with_capacities(*capacity), None => TrieCache::new(), } } diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index 7b34f281701..5d4759a2393 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -13,15 +13,45 @@ use near_primitives::types::{TrieCacheMode, TrieNodesCount}; use std::cell::{Cell, RefCell}; use std::io::ErrorKind; -pub const DELETIONS_QUEUE_CAPACITY: usize = 100_000; +pub struct TrieDeletionsQueue { + queue: VecDeque, + /// Upper bound for the deletions queue size. + capacity: usize, +} + +impl TrieDeletionsQueue { + pub fn new(capacity: usize) -> Self { + Self { queue: VecDeque::with_capacity(capacity), capacity } + } + + pub fn clear(&mut self) { + self.queue.clear(); + } + + pub fn pop(&mut self) -> Option { + self.queue.pop_front() + } + + pub fn put(&mut self, key: CryptoHash) -> Option { + let result = if self.queue.len() == self.capacity { + let key_to_remove = self.pop().expect("Queue cannot be empty"); + Some(key_to_remove) + } else { + None + }; + self.queue.push_back(key); + result + } +} /// In-memory cache for trie items - nodes and values. All nodes are stored in the LRU cache with three modifications. -/// 1) Size of each value must not exceed `TRIE_LIMIT_CACHED_VALUE_SIZE`. +/// 1) Size of each value must not exceed `SHARD_CACHE_VALUE_SIZE_LIMIT`. /// Needed to avoid caching large values like contract codes. -/// 2) Summary size of all values should not exceed `total_sizes_capacity`. If new value doesn't fit, LRU values are -/// evicted until it fits. -/// Needed to bound cache capacity more precisely, because value sizes generally vary from 1 B to 500 B. -/// 3) If value is popped, it is put to the `deletions` queue with `DELETIONS_CACHE_CAPACITY` first. If popped value +/// 2) If we put new value to LRU cache and total size of existing values exceeds `total_sizes_capacity`, we evict +/// values from it until that is no longer the case. So the actual total size should never exceed +/// `total_size_limit` + `SHARD_CACHE_VALUE_SIZE_LIMIT`. +/// Needed because value sizes generally vary from 1 B to 500 B and we want to count cache size precisely. +/// 3) If value is popped, it is put to the `deletions` queue with `deletions_queue_capacity` first. If popped value /// doesn't fit in the queue, the last value is removed from the queue and LRU cache, and newly popped value is inserted /// to the queue. /// Needed to delay deletions when we have forks. In such case, many blocks can share same parent, and we want to keep @@ -30,20 +60,24 @@ pub struct SyncTrieCache { /// LRU cache keeping mapping from keys to values. cache: LruCache>, /// Queue of items which were popped, which postpones deletion of old nodes. - deletions: VecDeque, - /// Total size of all values in the cache. + deletions: TrieDeletionsQueue, + /// Current total size of all values in the cache. total_size: u64, /// Upper bound for the total size. - total_size_capacity: u64, + total_size_limit: u64, } impl SyncTrieCache { - pub fn new(cap: usize) -> Self { + pub fn new( + cache_capacity: usize, + deletions_queue_capacity: usize, + total_size_limit: u64, + ) -> Self { Self { - cache: LruCache::new(cap), - deletions: VecDeque::with_capacity(DELETIONS_QUEUE_CAPACITY), + cache: LruCache::new(cache_capacity), + deletions: TrieDeletionsQueue::new(deletions_queue_capacity), total_size: 0, - total_size_capacity: 4_500_000_000, + total_size_limit, } } @@ -58,34 +92,45 @@ impl SyncTrieCache { } pub fn put(&mut self, key: CryptoHash, value: Arc<[u8]>) { - // We assume that value.len() is less than total size capacity. - // If adding new value exceeds capacity, we pop LRU values until it fits. - while self.total_size + value.len() as u64 > self.total_size_capacity { - let (_, evicted_value) = - self.cache.pop_lru().expect("Cannot fail because cap_lengths is > 0"); - self.total_size -= evicted_value.len() as u64; + while self.total_size > self.total_size_limit { + let evicted_value = match self.deletions.pop() { + // First, try to evict value using the key from deletions queue. + Some(key) => self.cache.pop(&key), + // Second, pop LRU value. + None => Some( + self.cache.pop_lru().expect("Cannot fail because total size capacity is > 0").1, + ), + }; + match evicted_value { + Some(value) => { + self.total_size -= value.len() as u64; + } + None => {} + }; } // Add value to the cache. self.total_size += value.len() as u64; - self.cache.put(key, value); + match self.cache.push(key, value) { + Some((_, evicted_value)) => { + self.total_size -= evicted_value.len() as u64; + } + None => {} + }; } pub fn pop(&mut self, key: &CryptoHash) { // Do nothing if key was removed before. if self.cache.contains(key) { - // Put key to the queue of deletions. If it exceeds queue capacity, key from queue tail is popped and the - // value from the LRU cache is removed. - if self.deletions.len() == DELETIONS_QUEUE_CAPACITY { - let key_to_remove = self.deletions.pop_front().expect("Deletions cannot be empty"); - match self.cache.pop(&key_to_remove) { + // Put key to the queue of deletions and possibly remove another key we have to delete. + match self.deletions.put(key.clone()) { + Some(key_to_delete) => match self.cache.pop(&key_to_delete) { Some(evicted_value) => { self.total_size -= evicted_value.len() as u64; } None => {} - } + }, + None => {} } - - self.deletions.push_back(key.clone()); } } @@ -101,11 +146,15 @@ pub struct TrieCache(Arc>); impl TrieCache { pub fn new() -> Self { - Self::with_capacity(TRIE_DEFAULT_SHARD_CACHE_SIZE) + Self::with_capacities(DEFAULT_SHARD_CACHE_CAPACITY) } - pub fn with_capacity(cap: usize) -> Self { - Self(Arc::new(Mutex::new(SyncTrieCache::new(cap)))) + pub fn with_capacities(cap: usize) -> Self { + Self(Arc::new(Mutex::new(SyncTrieCache::new( + cap, + DEFAULT_SHARD_CACHE_DELETIONS_QUEUE_CAPACITY, + DEFAULT_SHARD_CACHE_TOTAL_SIZE_LIMIT, + )))) } pub fn get(&self, key: &CryptoHash) -> Option> { @@ -121,7 +170,7 @@ impl TrieCache { for (hash, opt_value_rc) in ops { if let Some(value_rc) = opt_value_rc { if let (Some(value), _rc) = decode_value_with_rc(&value_rc) { - if value.len() < TRIE_LIMIT_CACHED_VALUE_SIZE { + if value.len() < SHARD_CACHE_VALUE_SIZE_LIMIT { guard.put(hash, value.into()); } } else { @@ -227,18 +276,28 @@ impl TrieStorage for TrieMemoryPartialStorage { /// Default number of cache entries. /// It was chosen to fit into RAM well. RAM spend on trie cache should not exceed 50_000 * 4 (number of shards) * -/// TRIE_LIMIT_CACHED_VALUE_SIZE * 2 (number of caches - for regular and view client) = 0.4 GB. +/// SHARD_CACHE_VALUE_SIZE_LIMIT * 2 (number of caches - for regular and view client) = 0.4 GB. /// In our tests on a single shard, it barely occupied 40 MB, which is dominated by state cache size /// with 512 MB limit. The total RAM usage for a single shard was 1 GB. #[cfg(not(feature = "no_cache"))] -const TRIE_DEFAULT_SHARD_CACHE_SIZE: usize = 50000; +const DEFAULT_SHARD_CACHE_CAPACITY: usize = 50_000; +#[cfg(feature = "no_cache")] +const DEFAULT_SHARD_CACHE_CAPACITY: usize = 1; + +#[cfg(not(feature = "no_cache"))] +const DEFAULT_SHARD_CACHE_TOTAL_SIZE_LIMIT: u64 = 50_000_000; // consider 4_500_000_000 +#[cfg(feature = "no_cache")] +const DEFAULT_SHARD_CACHE_TOTAL_SIZE_LIMIT: u64 = 1; +/// Capacity for the deletions queue. +#[cfg(feature = "cache")] +pub const DEFAULT_SHARD_CACHE_DELETIONS_QUEUE_CAPACITY: usize = 100_000; #[cfg(feature = "no_cache")] -const TRIE_DEFAULT_SHARD_CACHE_SIZE: usize = 1; +pub const DEFAULT_SHARD_CACHE_DELETIONS_QUEUE_CAPACITY: usize = 1; /// Values above this size (in bytes) are never cached. /// Note that most of Trie inner nodes are smaller than this - e.g. branches use around 32 * 16 = 512 bytes. -pub(crate) const TRIE_LIMIT_CACHED_VALUE_SIZE: usize = 1000; +pub(crate) const SHARD_CACHE_VALUE_SIZE_LIMIT: usize = 1000; pub struct TrieCachingStorage { pub(crate) store: Store, @@ -344,7 +403,7 @@ impl TrieStorage for TrieCachingStorage { // It is fine to have a size limit for shard cache and **not** have a limit for chunk cache, because key // is always a value hash, so for each key there could be only one value, and it is impossible to have // **different** values for the given key in shard and chunk caches. - if val.len() < TRIE_LIMIT_CACHED_VALUE_SIZE { + if val.len() < SHARD_CACHE_VALUE_SIZE_LIMIT { guard.put(*hash, val.clone()); } else { near_o11y::io_trace!(count: "shard_cache_too_large"); diff --git a/core/store/src/trie/trie_tests.rs b/core/store/src/trie/trie_tests.rs index 9abe817a0c8..322cd0c3e70 100644 --- a/core/store/src/trie/trie_tests.rs +++ b/core/store/src/trie/trie_tests.rs @@ -204,7 +204,7 @@ mod nodes_counter_tests { mod caching_storage_tests { use super::*; use crate::test_utils::{create_test_store, create_tries}; - use crate::trie::trie_storage::{TrieCache, TrieCachingStorage, TRIE_LIMIT_CACHED_VALUE_SIZE}; + use crate::trie::trie_storage::{TrieCache, TrieCachingStorage, SHARD_CACHE_VALUE_SIZE_LIMIT}; use crate::trie::TrieRefcountChange; use crate::{Store, TrieChanges}; use assert_matches::assert_matches; @@ -266,7 +266,7 @@ mod caching_storage_tests { /// Check that large values does not fall into shard cache, but fall into chunk cache. #[test] fn test_large_value() { - let value = vec![1u8].repeat(TRIE_LIMIT_CACHED_VALUE_SIZE + 1); + let value = vec![1u8].repeat(SHARD_CACHE_VALUE_SIZE_LIMIT + 1); let values = vec![value.clone()]; let shard_uid = ShardUId::single_shard(); let store = create_store_with_values(&values, shard_uid); @@ -340,7 +340,7 @@ mod caching_storage_tests { let values: Vec> = (0..shard_cache_size as u8 + 1).map(|i| vec![i]).collect(); let shard_uid = ShardUId::single_shard(); let store = create_store_with_values(&values, shard_uid); - let trie_cache = TrieCache::with_capacity(shard_cache_size); + let trie_cache = TrieCache::with_capacities(shard_cache_size); let trie_caching_storage = TrieCachingStorage::new(store, trie_cache.clone(), shard_uid); let value = &values[0]; From 158db1a675c7b606449563b3c25225d24c240950 Mon Sep 17 00:00:00 2001 From: Longarithm Date: Fri, 19 Aug 2022 15:52:27 +0400 Subject: [PATCH 07/37] bounded queue tests --- core/store/src/trie/trie_storage.rs | 94 +++++++++++++++++++---------- core/store/src/trie/trie_tests.rs | 4 +- 2 files changed, 65 insertions(+), 33 deletions(-) diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index 5d4759a2393..3862a7770de 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -13,43 +13,42 @@ use near_primitives::types::{TrieCacheMode, TrieNodesCount}; use std::cell::{Cell, RefCell}; use std::io::ErrorKind; -pub struct TrieDeletionsQueue { - queue: VecDeque, - /// Upper bound for the deletions queue size. +pub(crate) struct BoundedQueue { + queue: VecDeque, + /// If queue size exceeds capacity, item from the tail is removed. capacity: usize, } -impl TrieDeletionsQueue { - pub fn new(capacity: usize) -> Self { - Self { queue: VecDeque::with_capacity(capacity), capacity } +impl BoundedQueue { + pub(crate) fn new(capacity: usize) -> Self { + // Reserve space for one extra element to simplify `put`. + Self { queue: VecDeque::with_capacity(capacity + 1), capacity } } - pub fn clear(&mut self) { + pub(crate) fn clear(&mut self) { self.queue.clear(); } - pub fn pop(&mut self) -> Option { + pub(crate) fn pop(&mut self) -> Option { self.queue.pop_front() } - pub fn put(&mut self, key: CryptoHash) -> Option { - let result = if self.queue.len() == self.capacity { - let key_to_remove = self.pop().expect("Queue cannot be empty"); - Some(key_to_remove) + pub(crate) fn put(&mut self, key: T) -> Option { + self.queue.push_back(key); + if self.queue.len() > self.capacity { + Some(self.pop().expect("Queue cannot be empty")) } else { None - }; - self.queue.push_back(key); - result + } } } /// In-memory cache for trie items - nodes and values. All nodes are stored in the LRU cache with three modifications. -/// 1) Size of each value must not exceed `SHARD_CACHE_VALUE_SIZE_LIMIT`. +/// 1) Size of each value must not exceed `TRIE_LIMIT_CACHED_VALUE_SIZE`. /// Needed to avoid caching large values like contract codes. /// 2) If we put new value to LRU cache and total size of existing values exceeds `total_sizes_capacity`, we evict /// values from it until that is no longer the case. So the actual total size should never exceed -/// `total_size_limit` + `SHARD_CACHE_VALUE_SIZE_LIMIT`. +/// `total_size_limit` + `TRIE_LIMIT_CACHED_VALUE_SIZE`. /// Needed because value sizes generally vary from 1 B to 500 B and we want to count cache size precisely. /// 3) If value is popped, it is put to the `deletions` queue with `deletions_queue_capacity` first. If popped value /// doesn't fit in the queue, the last value is removed from the queue and LRU cache, and newly popped value is inserted @@ -60,7 +59,7 @@ pub struct SyncTrieCache { /// LRU cache keeping mapping from keys to values. cache: LruCache>, /// Queue of items which were popped, which postpones deletion of old nodes. - deletions: TrieDeletionsQueue, + deletions: BoundedQueue, /// Current total size of all values in the cache. total_size: u64, /// Upper bound for the total size. @@ -68,30 +67,30 @@ pub struct SyncTrieCache { } impl SyncTrieCache { - pub fn new( + pub(crate) fn new( cache_capacity: usize, deletions_queue_capacity: usize, total_size_limit: u64, ) -> Self { Self { cache: LruCache::new(cache_capacity), - deletions: TrieDeletionsQueue::new(deletions_queue_capacity), + deletions: BoundedQueue::new(deletions_queue_capacity), total_size: 0, total_size_limit, } } - pub fn get(&mut self, key: &CryptoHash) -> Option> { + pub(crate) fn get(&mut self, key: &CryptoHash) -> Option> { self.cache.get(key).cloned() } - pub fn clear(&mut self) { + pub(crate) fn clear(&mut self) { self.total_size = 0; self.deletions.clear(); self.cache.clear(); } - pub fn put(&mut self, key: CryptoHash, value: Arc<[u8]>) { + pub(crate) fn put(&mut self, key: CryptoHash, value: Arc<[u8]>) { while self.total_size > self.total_size_limit { let evicted_value = match self.deletions.pop() { // First, try to evict value using the key from deletions queue. @@ -118,7 +117,7 @@ impl SyncTrieCache { }; } - pub fn pop(&mut self, key: &CryptoHash) { + pub(crate) fn pop(&mut self, key: &CryptoHash) { // Do nothing if key was removed before. if self.cache.contains(key) { // Put key to the queue of deletions and possibly remove another key we have to delete. @@ -146,7 +145,7 @@ pub struct TrieCache(Arc>); impl TrieCache { pub fn new() -> Self { - Self::with_capacities(DEFAULT_SHARD_CACHE_CAPACITY) + Self::with_capacities(TRIE_DEFAULT_SHARD_CACHE_SIZE) } pub fn with_capacities(cap: usize) -> Self { @@ -170,7 +169,7 @@ impl TrieCache { for (hash, opt_value_rc) in ops { if let Some(value_rc) = opt_value_rc { if let (Some(value), _rc) = decode_value_with_rc(&value_rc) { - if value.len() < SHARD_CACHE_VALUE_SIZE_LIMIT { + if value.len() < TRIE_LIMIT_CACHED_VALUE_SIZE { guard.put(hash, value.into()); } } else { @@ -276,13 +275,13 @@ impl TrieStorage for TrieMemoryPartialStorage { /// Default number of cache entries. /// It was chosen to fit into RAM well. RAM spend on trie cache should not exceed 50_000 * 4 (number of shards) * -/// SHARD_CACHE_VALUE_SIZE_LIMIT * 2 (number of caches - for regular and view client) = 0.4 GB. +/// TRIE_LIMIT_CACHED_VALUE_SIZE * 2 (number of caches - for regular and view client) = 0.4 GB. /// In our tests on a single shard, it barely occupied 40 MB, which is dominated by state cache size /// with 512 MB limit. The total RAM usage for a single shard was 1 GB. #[cfg(not(feature = "no_cache"))] -const DEFAULT_SHARD_CACHE_CAPACITY: usize = 50_000; +const TRIE_DEFAULT_SHARD_CACHE_SIZE: usize = 50000; #[cfg(feature = "no_cache")] -const DEFAULT_SHARD_CACHE_CAPACITY: usize = 1; +const TRIE_DEFAULT_SHARD_CACHE_SIZE: usize = 1; #[cfg(not(feature = "no_cache"))] const DEFAULT_SHARD_CACHE_TOTAL_SIZE_LIMIT: u64 = 50_000_000; // consider 4_500_000_000 @@ -297,7 +296,7 @@ pub const DEFAULT_SHARD_CACHE_DELETIONS_QUEUE_CAPACITY: usize = 1; /// Values above this size (in bytes) are never cached. /// Note that most of Trie inner nodes are smaller than this - e.g. branches use around 32 * 16 = 512 bytes. -pub(crate) const SHARD_CACHE_VALUE_SIZE_LIMIT: usize = 1000; +pub(crate) const TRIE_LIMIT_CACHED_VALUE_SIZE: usize = 1000; pub struct TrieCachingStorage { pub(crate) store: Store, @@ -403,7 +402,7 @@ impl TrieStorage for TrieCachingStorage { // It is fine to have a size limit for shard cache and **not** have a limit for chunk cache, because key // is always a value hash, so for each key there could be only one value, and it is impossible to have // **different** values for the given key in shard and chunk caches. - if val.len() < SHARD_CACHE_VALUE_SIZE_LIMIT { + if val.len() < TRIE_LIMIT_CACHED_VALUE_SIZE { guard.put(*hash, val.clone()); } else { near_o11y::io_trace!(count: "shard_cache_too_large"); @@ -439,3 +438,36 @@ impl TrieStorage for TrieCachingStorage { TrieNodesCount { db_reads: self.db_read_nodes.get(), mem_reads: self.mem_read_nodes.get() } } } + +#[cfg(test)] +mod tests { + use crate::trie::trie_storage::BoundedQueue; + + #[test] + fn test_put_pop() { + let mut queue = BoundedQueue::new(2); + assert_eq!(queue.put(1), None); + assert_eq!(queue.put(2), None); + assert_eq!(queue.put(3), Some(1)); + + assert_eq!(queue.pop(), Some(2)); + assert_eq!(queue.pop(), Some(3)); + assert_eq!(queue.pop(), None); + } + + #[test] + fn test_clear() { + let mut queue = BoundedQueue::new(2); + queue.put(1); + assert_eq!(queue.queue.get(0), Some(&1)); + queue.clear(); + assert!(queue.queue.is_empty()); + } + + #[test] + fn test_zero_capacity() { + let mut queue = BoundedQueue::new(0); + assert_eq!(queue.put(1), Some(1)); + assert!(queue.queue.is_empty()); + } +} diff --git a/core/store/src/trie/trie_tests.rs b/core/store/src/trie/trie_tests.rs index 322cd0c3e70..b9772713dfd 100644 --- a/core/store/src/trie/trie_tests.rs +++ b/core/store/src/trie/trie_tests.rs @@ -204,7 +204,7 @@ mod nodes_counter_tests { mod caching_storage_tests { use super::*; use crate::test_utils::{create_test_store, create_tries}; - use crate::trie::trie_storage::{TrieCache, TrieCachingStorage, SHARD_CACHE_VALUE_SIZE_LIMIT}; + use crate::trie::trie_storage::{TrieCache, TrieCachingStorage, TRIE_LIMIT_CACHED_VALUE_SIZE}; use crate::trie::TrieRefcountChange; use crate::{Store, TrieChanges}; use assert_matches::assert_matches; @@ -266,7 +266,7 @@ mod caching_storage_tests { /// Check that large values does not fall into shard cache, but fall into chunk cache. #[test] fn test_large_value() { - let value = vec![1u8].repeat(SHARD_CACHE_VALUE_SIZE_LIMIT + 1); + let value = vec![1u8].repeat(TRIE_LIMIT_CACHED_VALUE_SIZE + 1); let values = vec![value.clone()]; let shard_uid = ShardUId::single_shard(); let store = create_store_with_values(&values, shard_uid); From 8534a85dc3f506a4c1cf4538b20a22b67615c5b2 Mon Sep 17 00:00:00 2001 From: Longarithm Date: Fri, 19 Aug 2022 15:54:32 +0400 Subject: [PATCH 08/37] fix pub --- core/store/src/trie/trie_storage.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index 3862a7770de..ef820932e0e 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -290,9 +290,9 @@ const DEFAULT_SHARD_CACHE_TOTAL_SIZE_LIMIT: u64 = 1; /// Capacity for the deletions queue. #[cfg(feature = "cache")] -pub const DEFAULT_SHARD_CACHE_DELETIONS_QUEUE_CAPACITY: usize = 100_000; +const DEFAULT_SHARD_CACHE_DELETIONS_QUEUE_CAPACITY: usize = 100_000; #[cfg(feature = "no_cache")] -pub const DEFAULT_SHARD_CACHE_DELETIONS_QUEUE_CAPACITY: usize = 1; +const DEFAULT_SHARD_CACHE_DELETIONS_QUEUE_CAPACITY: usize = 1; /// Values above this size (in bytes) are never cached. /// Note that most of Trie inner nodes are smaller than this - e.g. branches use around 32 * 16 = 512 bytes. From bf9b8142f62aa4b90f19cf826f6e29ec45062719 Mon Sep 17 00:00:00 2001 From: Longarithm Date: Fri, 19 Aug 2022 15:58:07 +0400 Subject: [PATCH 09/37] move const --- core/store/src/trie/trie_storage.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index ef820932e0e..ce7d29bbb23 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -13,6 +13,12 @@ use near_primitives::types::{TrieCacheMode, TrieNodesCount}; use std::cell::{Cell, RefCell}; use std::io::ErrorKind; +/// Capacity for the deletions queue. +#[cfg(feature = "cache")] +const DEFAULT_SHARD_CACHE_DELETIONS_QUEUE_CAPACITY: usize = 100_000; +#[cfg(feature = "no_cache")] +const DEFAULT_SHARD_CACHE_DELETIONS_QUEUE_CAPACITY: usize = 1; + pub(crate) struct BoundedQueue { queue: VecDeque, /// If queue size exceeds capacity, item from the tail is removed. @@ -288,12 +294,6 @@ const DEFAULT_SHARD_CACHE_TOTAL_SIZE_LIMIT: u64 = 50_000_000; // consider 4_500_ #[cfg(feature = "no_cache")] const DEFAULT_SHARD_CACHE_TOTAL_SIZE_LIMIT: u64 = 1; -/// Capacity for the deletions queue. -#[cfg(feature = "cache")] -const DEFAULT_SHARD_CACHE_DELETIONS_QUEUE_CAPACITY: usize = 100_000; -#[cfg(feature = "no_cache")] -const DEFAULT_SHARD_CACHE_DELETIONS_QUEUE_CAPACITY: usize = 1; - /// Values above this size (in bytes) are never cached. /// Note that most of Trie inner nodes are smaller than this - e.g. branches use around 32 * 16 = 512 bytes. pub(crate) const TRIE_LIMIT_CACHED_VALUE_SIZE: usize = 1000; From 0451f050b68e6eb640d8c864237a31c097cde47a Mon Sep 17 00:00:00 2001 From: Longarithm Date: Fri, 19 Aug 2022 15:59:06 +0400 Subject: [PATCH 10/37] move const --- core/store/src/trie/trie_storage.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index ce7d29bbb23..c27e0395db0 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -13,12 +13,6 @@ use near_primitives::types::{TrieCacheMode, TrieNodesCount}; use std::cell::{Cell, RefCell}; use std::io::ErrorKind; -/// Capacity for the deletions queue. -#[cfg(feature = "cache")] -const DEFAULT_SHARD_CACHE_DELETIONS_QUEUE_CAPACITY: usize = 100_000; -#[cfg(feature = "no_cache")] -const DEFAULT_SHARD_CACHE_DELETIONS_QUEUE_CAPACITY: usize = 1; - pub(crate) struct BoundedQueue { queue: VecDeque, /// If queue size exceeds capacity, item from the tail is removed. @@ -294,6 +288,12 @@ const DEFAULT_SHARD_CACHE_TOTAL_SIZE_LIMIT: u64 = 50_000_000; // consider 4_500_ #[cfg(feature = "no_cache")] const DEFAULT_SHARD_CACHE_TOTAL_SIZE_LIMIT: u64 = 1; +/// Capacity for the deletions queue. +#[cfg(not(feature = "no_cache"))] +const DEFAULT_SHARD_CACHE_DELETIONS_QUEUE_CAPACITY: usize = 100_000; +#[cfg(feature = "no_cache")] +const DEFAULT_SHARD_CACHE_DELETIONS_QUEUE_CAPACITY: usize = 1; + /// Values above this size (in bytes) are never cached. /// Note that most of Trie inner nodes are smaller than this - e.g. branches use around 32 * 16 = 512 bytes. pub(crate) const TRIE_LIMIT_CACHED_VALUE_SIZE: usize = 1000; From 2e530812b4612d56c57111c3244acf94b726b4fc Mon Sep 17 00:00:00 2001 From: Longarithm Date: Fri, 19 Aug 2022 16:30:02 +0400 Subject: [PATCH 11/37] move const --- core/store/src/trie/trie_storage.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index c27e0395db0..e24060eb725 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -440,7 +440,7 @@ impl TrieStorage for TrieCachingStorage { } #[cfg(test)] -mod tests { +mod bounded_queue_tests { use crate::trie::trie_storage::BoundedQueue; #[test] @@ -471,3 +471,14 @@ mod tests { assert!(queue.queue.is_empty()); } } + +// #[cfg(test)] +// mod trie_cache_tests { +// use crate::trie::trie_storage::SyncTrieCache; +// +// #[test] +// fn test_put_pop() { +// let mut queue = SyncTrieCache::new() +// assert_eq!(queue.put(1), None); +// } +// } From 9cd170c1d34fb1a476bddfa48819d9c4873e59a5 Mon Sep 17 00:00:00 2001 From: Longarithm Date: Fri, 19 Aug 2022 17:24:26 +0400 Subject: [PATCH 12/37] cache size limit --- core/store/src/trie/trie_storage.rs | 34 ++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index e24060eb725..93b4f6fee1c 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -472,13 +472,27 @@ mod bounded_queue_tests { } } -// #[cfg(test)] -// mod trie_cache_tests { -// use crate::trie::trie_storage::SyncTrieCache; -// -// #[test] -// fn test_put_pop() { -// let mut queue = SyncTrieCache::new() -// assert_eq!(queue.put(1), None); -// } -// } +#[cfg(test)] +mod trie_cache_tests { + use crate::trie::trie_storage::SyncTrieCache; + use near_primitives::hash::hash; + + fn put_value(cache: &mut SyncTrieCache, value: &[u8]) { + cache.put(hash(value), value.into()); + } + + #[test] + fn test_size_limit() { + let mut cache = SyncTrieCache::new(100, 100, 5); + put_value(&mut cache, &[1, 1]); + assert_eq!(cache.total_size, 2); + put_value(&mut cache, &[1, 1, 1]); + assert_eq!(cache.total_size, 5); + put_value(&mut cache, &[1]); + assert_eq!(cache.total_size, 6); + put_value(&mut cache, &[1, 1]); + assert_eq!(cache.total_size, 6); + put_value(&mut cache, &[2]); + assert_eq!(cache.total_size, 4); + } +} From 81f6648be91afd0b50fd2539f69a57387b297bf1 Mon Sep 17 00:00:00 2001 From: Longarithm Date: Fri, 19 Aug 2022 17:34:11 +0400 Subject: [PATCH 13/37] synctriecache test --- core/store/src/trie/trie_storage.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index 93b4f6fee1c..d3fdac9b074 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -476,6 +476,7 @@ mod bounded_queue_tests { mod trie_cache_tests { use crate::trie::trie_storage::SyncTrieCache; use near_primitives::hash::hash; + use std::sync::Arc; fn put_value(cache: &mut SyncTrieCache, value: &[u8]) { cache.put(hash(value), value.into()); @@ -484,15 +485,25 @@ mod trie_cache_tests { #[test] fn test_size_limit() { let mut cache = SyncTrieCache::new(100, 100, 5); + // Add three values. Before each put, condition on total size should not be triggered. put_value(&mut cache, &[1, 1]); assert_eq!(cache.total_size, 2); put_value(&mut cache, &[1, 1, 1]); assert_eq!(cache.total_size, 5); put_value(&mut cache, &[1]); assert_eq!(cache.total_size, 6); - put_value(&mut cache, &[1, 1]); - assert_eq!(cache.total_size, 6); - put_value(&mut cache, &[2]); + + // Add one of previous values. LRU value should be evicted. + put_value(&mut cache, &[1, 1, 1]); assert_eq!(cache.total_size, 4); + assert_eq!(cache.cache.pop_lru(), Some((hash(&[1]), vec![1].into()))); + assert_eq!(cache.cache.pop_lru(), Some((hash(&[1, 1, 1]), vec![1, 1, 1].into()))); + } + + #[test] + fn test_deletions_queue() { + let mut cache = SyncTrieCache::new(100, 100, 5); + // Add three values. Before each put, condition on total size should not be triggered. + put_value(&mut cache, &[1, 1]); } } From 01db7a984b8bf6f846bf3bcd6973ff395b00afb5 Mon Sep 17 00:00:00 2001 From: Longarithm Date: Fri, 19 Aug 2022 17:34:53 +0400 Subject: [PATCH 14/37] synctriecache test --- core/store/src/trie/trie_storage.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index d3fdac9b074..dcfd44a6d89 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -476,7 +476,6 @@ mod bounded_queue_tests { mod trie_cache_tests { use crate::trie::trie_storage::SyncTrieCache; use near_primitives::hash::hash; - use std::sync::Arc; fn put_value(cache: &mut SyncTrieCache, value: &[u8]) { cache.put(hash(value), value.into()); From 2258f8df0c86164a8731bc44ab038cced7232618 Mon Sep 17 00:00:00 2001 From: Longarithm Date: Fri, 19 Aug 2022 17:44:34 +0400 Subject: [PATCH 15/37] test deletions queue removal --- core/store/src/trie/trie_storage.rs | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index dcfd44a6d89..298d1f016b7 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -117,7 +117,9 @@ impl SyncTrieCache { }; } - pub(crate) fn pop(&mut self, key: &CryptoHash) { + // Adds key to the deletions queue if it is present in cache. + // Returns key-value pair which are popped if deletions queue is full. + pub(crate) fn pop(&mut self, key: &CryptoHash) -> Option<(CryptoHash, Arc<[u8]>)> { // Do nothing if key was removed before. if self.cache.contains(key) { // Put key to the queue of deletions and possibly remove another key we have to delete. @@ -125,11 +127,14 @@ impl SyncTrieCache { Some(key_to_delete) => match self.cache.pop(&key_to_delete) { Some(evicted_value) => { self.total_size -= evicted_value.len() as u64; + Some((key_to_delete, evicted_value)) } - None => {} + None => None, }, - None => {} + None => None, } + } else { + None } } @@ -495,14 +500,23 @@ mod trie_cache_tests { // Add one of previous values. LRU value should be evicted. put_value(&mut cache, &[1, 1, 1]); assert_eq!(cache.total_size, 4); - assert_eq!(cache.cache.pop_lru(), Some((hash(&[1]), vec![1].into()))); + assert_eq!(cache.cache.pop_lru(), Some((hash(&[1, /* */ 1]), vec![1].into()))); assert_eq!(cache.cache.pop_lru(), Some((hash(&[1, 1, 1]), vec![1, 1, 1].into()))); } #[test] fn test_deletions_queue() { - let mut cache = SyncTrieCache::new(100, 100, 5); - // Add three values. Before each put, condition on total size should not be triggered. + let mut cache = SyncTrieCache::new(100, 2, 100); + // Add two values to the cache. + put_value(&mut cache, &[1]); put_value(&mut cache, &[1, 1]); + + // Call pop for inserted values. Because deletions queue is not full, no elements should be deleted. + assert_eq!(cache.pop(&hash(&[1, 1])), None); + assert_eq!(cache.pop(&hash(&[1])), None); + + // Call pop two times for a value existing in cache. Because queue is full, both elements should be deleted. + assert_eq!(cache.pop(&hash(&[1])), Some((hash(&[1, 1]), vec![1, 1].into()))); + assert_eq!(cache.pop(&hash(&[1])), Some((hash(&[1]), vec![1].into()))); } } From 54171d78304adffaa8211741cb9e9db5a29d894c Mon Sep 17 00:00:00 2001 From: Longarithm Date: Fri, 19 Aug 2022 17:48:03 +0400 Subject: [PATCH 16/37] test cache capacity --- core/store/src/trie/trie_storage.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index 298d1f016b7..7ce335a7de2 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -500,7 +500,7 @@ mod trie_cache_tests { // Add one of previous values. LRU value should be evicted. put_value(&mut cache, &[1, 1, 1]); assert_eq!(cache.total_size, 4); - assert_eq!(cache.cache.pop_lru(), Some((hash(&[1, /* */ 1]), vec![1].into()))); + assert_eq!(cache.cache.pop_lru(), Some((hash(&[1]), vec![1].into()))); assert_eq!(cache.cache.pop_lru(), Some((hash(&[1, 1, 1]), vec![1, 1, 1].into()))); } @@ -519,4 +519,16 @@ mod trie_cache_tests { assert_eq!(cache.pop(&hash(&[1])), Some((hash(&[1, 1]), vec![1, 1].into()))); assert_eq!(cache.pop(&hash(&[1])), Some((hash(&[1]), vec![1].into()))); } + + #[test] + fn test_cache_capacity() { + let mut cache = SyncTrieCache::new(2, 100, 100); + put_value(&mut cache, &[1]); + put_value(&mut cache, &[2]); + put_value(&mut cache, &[3]); + + assert!(!cache.cache.contains(&hash(&[1]))); + assert!(cache.cache.contains(&hash(&[2]))); + assert!(cache.cache.contains(&hash(&[3]))); + } } From 2b7f7bf64c7c6010e1a1d08b3f079e94fb5a6a57 Mon Sep 17 00:00:00 2001 From: firatNEAR Date: Fri, 19 Aug 2022 20:02:30 +0200 Subject: [PATCH 17/37] Fix comments --- core/store/src/metrics.rs | 2 +- core/store/src/trie/trie_storage.rs | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/core/store/src/metrics.rs b/core/store/src/metrics.rs index da2973151bb..e81321774e3 100644 --- a/core/store/src/metrics.rs +++ b/core/store/src/metrics.rs @@ -53,7 +53,7 @@ pub static SHARD_CACHE_MISSES: Lazy = Lazy::new(|| { pub static SHARD_CACHE_TOO_LARGE: Lazy = Lazy::new(|| { try_create_int_counter_vec( "near_shard_cache_too_large", - "Shard cache too large", + "Number of values to be inserted into shard cache is too large", &["shard_id", "is_view"], ) .unwrap() diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index 55f2f30949f..74a4f43b96c 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -44,6 +44,8 @@ impl TrieCache { if let (Some(value), _rc) = decode_value_with_rc(&value_rc) { if value.len() < TRIE_LIMIT_CACHED_VALUE_SIZE { guard.put(hash, value.into()); + } else { + metrics::SHARD_CACHE_TOO_LARGE.with_label_values(&labels).inc(); } } else { match guard.pop(&hash) { @@ -248,13 +250,8 @@ impl TrieCachingStorage { pub fn set_mode(&self, state: TrieCacheMode) { self.cache_mode.set(state); } -} -impl TrieStorage for TrieCachingStorage { - fn retrieve_raw_bytes(&self, hash: &CryptoHash) -> Result, StorageError> { - let shard_id_str = format!("{}", self.shard_uid.shard_id); - let is_view_str = format!("{}", self.is_view as u8); - let labels: [&str; 2] = [&shard_id_str, &is_view_str]; + fn update_cache_size_metrics(&self, labels: [&str; 2]) { { metrics::CHUNK_CACHE_SIZE .with_label_values(&labels) @@ -263,6 +260,16 @@ impl TrieStorage for TrieCachingStorage { .with_label_values(&labels) .set(self.shard_cache.0.lock().expect(POISONED_LOCK_ERR).len() as i64); } + } +} + +impl TrieStorage for TrieCachingStorage { + fn retrieve_raw_bytes(&self, hash: &CryptoHash) -> Result, StorageError> { + let shard_id_str = format!("{}", self.shard_uid.shard_id); + let is_view_str = format!("{}", self.is_view as u8); + let labels: [&str; 2] = [&shard_id_str, &is_view_str]; + + self.update_cache_size_metrics(labels); // Try to get value from chunk cache containing nodes with cheaper access. We can do it for any `TrieCacheMode`, // because we charge for reading nodes only when `CachingChunk` mode is enabled anyway. if let Some(val) = self.chunk_cache.borrow_mut().get(hash) { From 9e3108bbb501a594d60a385529a911b36d72007b Mon Sep 17 00:00:00 2001 From: Longarithm Date: Sat, 20 Aug 2022 03:54:15 +0400 Subject: [PATCH 18/37] temporary fixes --- core/store/src/trie/shard_tries.rs | 27 ++++---- core/store/src/trie/trie_storage.rs | 67 +++++++++---------- core/store/src/trie/trie_tests.rs | 22 +++--- .../src/estimator_context.rs | 2 +- 4 files changed, 55 insertions(+), 63 deletions(-) diff --git a/core/store/src/trie/shard_tries.rs b/core/store/src/trie/shard_tries.rs index b723a28ef39..f488c783e4a 100644 --- a/core/store/src/trie/shard_tries.rs +++ b/core/store/src/trie/shard_tries.rs @@ -42,20 +42,24 @@ impl TrieCacheFactory { } /// Create new cache for the given shard uid. - pub fn create_cache(&self, shard_uid: &ShardUId) -> TrieCache { - match self.capacities.get(shard_uid) { - Some(capacity) => TrieCache::with_capacities(*capacity), - None => TrieCache::new(), + pub fn create_cache(&self, shard_uid: &ShardUId, is_view: bool) -> TrieCache { + let capacity = if is_view { None } else { self.capacities.get(shard_uid) }; + match capacity { + Some(capacity) => TrieCache::with_capacities(*capacity, shard_uid.shard_id, is_view), + None => TrieCache::new(shard_uid.shard_id, is_view), } } /// Create caches on the initialization of storage structures. - pub fn create_initial_caches(&self) -> HashMap { + pub fn create_initial_caches(&self, is_view: bool) -> HashMap { assert_ne!(self.num_shards, 0); let shards: Vec<_> = (0..self.num_shards) .map(|shard_id| ShardUId { version: self.shard_version, shard_id: shard_id as u32 }) .collect(); - shards.iter().map(|&shard_uid| (shard_uid, self.create_cache(&shard_uid))).collect() + shards + .iter() + .map(|&shard_uid| (shard_uid, self.create_cache(&shard_uid, is_view))) + .collect() } } @@ -73,8 +77,8 @@ pub struct ShardTries(Arc); impl ShardTries { pub fn new(store: Store, trie_cache_factory: TrieCacheFactory) -> Self { - let caches = trie_cache_factory.create_initial_caches(); - let view_caches = trie_cache_factory.create_initial_caches(); + let caches = trie_cache_factory.create_initial_caches(false); + let view_caches = trie_cache_factory.create_initial_caches(true); ShardTries(Arc::new(ShardTriesInner { store, trie_cache_factory, @@ -112,11 +116,10 @@ impl ShardTries { let mut caches = caches_to_use.write().expect(POISONED_LOCK_ERR); caches .entry(shard_uid) - .or_insert_with(|| self.0.trie_cache_factory.create_cache(&shard_uid)) + .or_insert_with(|| self.0.trie_cache_factory.create_cache(&shard_uid, is_view)) .clone() }; - let storage = - Box::new(TrieCachingStorage::new(self.0.store.clone(), cache, shard_uid, is_view)); + let storage = Box::new(TrieCachingStorage::new(self.0.store.clone(), cache, shard_uid)); let flat_state = { #[cfg(feature = "protocol_feature_flat_state")] if use_flat_state { @@ -178,7 +181,7 @@ impl ShardTries { for (shard_uid, ops) in shards { let cache = caches .entry(shard_uid) - .or_insert_with(|| self.0.trie_cache_factory.create_cache(&shard_uid)) + .or_insert_with(|| self.0.trie_cache_factory.create_cache(&shard_uid, false)) .clone(); cache.update_cache(ops, shard_uid); } diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index a72a71f98fa..87e147e87da 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -64,6 +64,10 @@ pub struct SyncTrieCache { total_size: u64, /// Upper bound for the total size. total_size_limit: u64, + /// Shard id of the nodes being cached. + pub shard_id: u32, + /// Whether cache is used for view calls execution. + pub is_view: bool, } impl SyncTrieCache { @@ -71,12 +75,16 @@ impl SyncTrieCache { cache_capacity: usize, deletions_queue_capacity: usize, total_size_limit: u64, + shard_id: u32, + is_view: bool, ) -> Self { Self { cache: LruCache::new(cache_capacity), deletions: BoundedQueue::new(deletions_queue_capacity), total_size: 0, total_size_limit, + shard_id, + is_view, } } @@ -138,7 +146,6 @@ impl SyncTrieCache { } } - #[cfg(test)] pub fn len(&self) -> usize { self.cache.len() } @@ -149,15 +156,17 @@ impl SyncTrieCache { pub struct TrieCache(Arc>); impl TrieCache { - pub fn new() -> Self { - Self::with_capacities(TRIE_DEFAULT_SHARD_CACHE_SIZE) + pub fn new(shard_id: u32, is_view: bool) -> Self { + Self::with_capacities(TRIE_DEFAULT_SHARD_CACHE_SIZE, shard_id, is_view) } - pub fn with_capacities(cap: usize) -> Self { + pub fn with_capacities(cap: usize, shard_id: u32, is_view: bool) -> Self { Self(Arc::new(Mutex::new(SyncTrieCache::new( cap, DEFAULT_SHARD_CACHE_DELETIONS_QUEUE_CAPACITY, DEFAULT_SHARD_CACHE_TOTAL_SIZE_LIMIT, + shard_id, + is_view, )))) } @@ -339,17 +348,10 @@ pub struct TrieCachingStorage { pub(crate) db_read_nodes: Cell, /// Counts trie nodes retrieved from the chunk cache. pub(crate) mem_read_nodes: Cell, - /// Boolean for determining if the cache is a view cache or not - is_view: bool, } impl TrieCachingStorage { - pub fn new( - store: Store, - shard_cache: TrieCache, - shard_uid: ShardUId, - is_view: bool, - ) -> TrieCachingStorage { + pub fn new(store: Store, shard_cache: TrieCache, shard_uid: ShardUId) -> TrieCachingStorage { TrieCachingStorage { store, shard_uid, @@ -358,7 +360,6 @@ impl TrieCachingStorage { chunk_cache: RefCell::new(Default::default()), db_read_nodes: Cell::new(0), mem_read_nodes: Cell::new(0), - is_view, } } @@ -395,45 +396,37 @@ impl TrieCachingStorage { pub fn set_mode(&self, state: TrieCacheMode) { self.cache_mode.set(state); } - - fn update_cache_size_metrics(&self, labels: [&str; 2]) { - { - metrics::CHUNK_CACHE_SIZE - .with_label_values(&labels) - .set(self.chunk_cache.borrow().len() as i64); - metrics::SHARD_CACHE_SIZE - .with_label_values(&labels) - .set(self.shard_cache.0.lock().expect(POISONED_LOCK_ERR).len() as i64); - } - } } impl TrieStorage for TrieCachingStorage { fn retrieve_raw_bytes(&self, hash: &CryptoHash) -> Result, StorageError> { - let shard_id_str = format!("{}", self.shard_uid.shard_id); - let is_view_str = format!("{}", self.is_view as u8); - let labels: [&str; 2] = [&shard_id_str, &is_view_str]; + let mut guard = self.shard_cache.0.lock().expect(POISONED_LOCK_ERR); + + let metrics_labels: [&str; 2] = + [&format!("{}", self.shard_uid.shard_id), &format!("{}", guard.is_view as u8)]; + metrics::CHUNK_CACHE_SIZE + .with_label_values(&metrics_labels) + .set(self.chunk_cache.borrow().len() as i64); + metrics::SHARD_CACHE_SIZE.with_label_values(&metrics_labels).set(guard.len() as i64); - self.update_cache_size_metrics(labels); // Try to get value from chunk cache containing nodes with cheaper access. We can do it for any `TrieCacheMode`, // because we charge for reading nodes only when `CachingChunk` mode is enabled anyway. if let Some(val) = self.chunk_cache.borrow_mut().get(hash) { - metrics::CHUNK_CACHE_HITS.with_label_values(&labels).inc(); + metrics::CHUNK_CACHE_HITS.with_label_values(&metrics_labels).inc(); self.inc_mem_read_nodes(); return Ok(val.clone()); } - metrics::CHUNK_CACHE_MISSES.with_label_values(&labels).inc(); + metrics::CHUNK_CACHE_MISSES.with_label_values(&metrics_labels).inc(); // Try to get value from shard cache containing most recently touched nodes. - let mut guard = self.shard_cache.0.lock().expect(POISONED_LOCK_ERR); let val = match guard.get(hash) { Some(val) => { - metrics::SHARD_CACHE_HITS.with_label_values(&labels).inc(); + metrics::SHARD_CACHE_HITS.with_label_values(&metrics_labels).inc(); near_o11y::io_trace!(count: "shard_cache_hit"); val.clone() } None => { - metrics::SHARD_CACHE_MISSES.with_label_values(&labels).inc(); + metrics::SHARD_CACHE_MISSES.with_label_values(&metrics_labels).inc(); near_o11y::io_trace!(count: "shard_cache_miss"); // If value is not present in cache, get it from the storage. let key = Self::get_key_from_shard_uid_and_hash(self.shard_uid, hash); @@ -453,7 +446,7 @@ impl TrieStorage for TrieCachingStorage { if val.len() < TRIE_LIMIT_CACHED_VALUE_SIZE { guard.put(*hash, val.clone()); } else { - metrics::SHARD_CACHE_TOO_LARGE.with_label_values(&labels).inc(); + metrics::SHARD_CACHE_TOO_LARGE.with_label_values(&metrics_labels).inc(); near_o11y::io_trace!(count: "shard_cache_too_large"); } @@ -532,7 +525,7 @@ mod trie_cache_tests { #[test] fn test_size_limit() { - let mut cache = SyncTrieCache::new(100, 100, 5); + let mut cache = SyncTrieCache::new(100, 100, 5, 0, false); // Add three values. Before each put, condition on total size should not be triggered. put_value(&mut cache, &[1, 1]); assert_eq!(cache.total_size, 2); @@ -550,7 +543,7 @@ mod trie_cache_tests { #[test] fn test_deletions_queue() { - let mut cache = SyncTrieCache::new(100, 2, 100); + let mut cache = SyncTrieCache::new(100, 2, 100, 0, false); // Add two values to the cache. put_value(&mut cache, &[1]); put_value(&mut cache, &[1, 1]); @@ -566,7 +559,7 @@ mod trie_cache_tests { #[test] fn test_cache_capacity() { - let mut cache = SyncTrieCache::new(2, 100, 100); + let mut cache = SyncTrieCache::new(2, 100, 100, 0, false); put_value(&mut cache, &[1]); put_value(&mut cache, &[2]); put_value(&mut cache, &[3]); diff --git a/core/store/src/trie/trie_tests.rs b/core/store/src/trie/trie_tests.rs index 5b1d411bc17..377de774570 100644 --- a/core/store/src/trie/trie_tests.rs +++ b/core/store/src/trie/trie_tests.rs @@ -234,9 +234,8 @@ mod caching_storage_tests { let values = vec![value.clone()]; let shard_uid = ShardUId::single_shard(); let store = create_store_with_values(&values, shard_uid); - let trie_cache = TrieCache::new(); - let trie_caching_storage = - TrieCachingStorage::new(store, trie_cache.clone(), shard_uid, false); + let trie_cache = TrieCache::new(0, false); + let trie_caching_storage = TrieCachingStorage::new(store, trie_cache.clone(), shard_uid); let key = hash(&value); assert_eq!(trie_cache.get(&key), None); @@ -257,7 +256,7 @@ mod caching_storage_tests { let shard_uid = ShardUId::single_shard(); let store = create_test_store(); let trie_caching_storage = - TrieCachingStorage::new(store, TrieCache::new(), shard_uid, false); + TrieCachingStorage::new(store, TrieCache::new(0, false), shard_uid); let value = vec![1u8]; let key = hash(&value); @@ -272,9 +271,8 @@ mod caching_storage_tests { let values = vec![value.clone()]; let shard_uid = ShardUId::single_shard(); let store = create_store_with_values(&values, shard_uid); - let trie_cache = TrieCache::new(); - let trie_caching_storage = - TrieCachingStorage::new(store, trie_cache.clone(), shard_uid, false); + let trie_cache = TrieCache::new(0, false); + let trie_caching_storage = TrieCachingStorage::new(store, trie_cache.clone(), shard_uid); let key = hash(&value); trie_caching_storage.set_mode(TrieCacheMode::CachingChunk); @@ -295,9 +293,8 @@ mod caching_storage_tests { let values = vec![vec![1u8]]; let shard_uid = ShardUId::single_shard(); let store = create_store_with_values(&values, shard_uid); - let trie_cache = TrieCache::new(); - let trie_caching_storage = - TrieCachingStorage::new(store, trie_cache.clone(), shard_uid, false); + let trie_cache = TrieCache::new(0, false); + let trie_caching_storage = TrieCachingStorage::new(store, trie_cache.clone(), shard_uid); let value = &values[0]; let key = hash(&value); @@ -344,9 +341,8 @@ mod caching_storage_tests { let values: Vec> = (0..shard_cache_size as u8 + 1).map(|i| vec![i]).collect(); let shard_uid = ShardUId::single_shard(); let store = create_store_with_values(&values, shard_uid); - let trie_cache = TrieCache::with_capacities(shard_cache_size); - let trie_caching_storage = - TrieCachingStorage::new(store, trie_cache.clone(), shard_uid, false); + let trie_cache = TrieCache::with_capacities(shard_cache_size, 0, false); + let trie_caching_storage = TrieCachingStorage::new(store, trie_cache.clone(), shard_uid); let value = &values[0]; let key = hash(&value); diff --git a/runtime/runtime-params-estimator/src/estimator_context.rs b/runtime/runtime-params-estimator/src/estimator_context.rs index bf6d509aa21..fbfde1a3c3c 100644 --- a/runtime/runtime-params-estimator/src/estimator_context.rs +++ b/runtime/runtime-params-estimator/src/estimator_context.rs @@ -124,7 +124,7 @@ impl<'c> Testbed<'c> { pub(crate) fn trie_caching_storage(&mut self) -> TrieCachingStorage { let store = self.inner.store(); let caching_storage = - TrieCachingStorage::new(store, TrieCache::new(), ShardUId::single_shard(), false); + TrieCachingStorage::new(store, TrieCache::new(0, false), ShardUId::single_shard()); caching_storage } From e41e0c6d390cf887aa16d58e3ef5de68b7f4fd99 Mon Sep 17 00:00:00 2001 From: Longarithm Date: Sat, 20 Aug 2022 04:03:09 +0400 Subject: [PATCH 19/37] fix capacity for view cache --- core/store/src/trie/shard_tries.rs | 3 ++- core/store/src/trie/trie_storage.rs | 18 ++++++++++++------ core/store/src/trie/trie_tests.rs | 14 +++++++++----- .../src/estimator_context.rs | 8 ++++++-- 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/core/store/src/trie/shard_tries.rs b/core/store/src/trie/shard_tries.rs index f488c783e4a..0263f1166c9 100644 --- a/core/store/src/trie/shard_tries.rs +++ b/core/store/src/trie/shard_tries.rs @@ -119,7 +119,8 @@ impl ShardTries { .or_insert_with(|| self.0.trie_cache_factory.create_cache(&shard_uid, is_view)) .clone() }; - let storage = Box::new(TrieCachingStorage::new(self.0.store.clone(), cache, shard_uid)); + let storage = + Box::new(TrieCachingStorage::new(self.0.store.clone(), cache, shard_uid, is_view)); let flat_state = { #[cfg(feature = "protocol_feature_flat_state")] if use_flat_state { diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index 87e147e87da..579a376c271 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -330,6 +330,7 @@ pub(crate) const TRIE_LIMIT_CACHED_VALUE_SIZE: usize = 1000; pub struct TrieCachingStorage { pub(crate) store: Store, pub(crate) shard_uid: ShardUId, + is_view: bool, /// Caches ever requested items for the shard `shard_uid`. Used to speed up DB operations, presence of any item is /// not guaranteed. @@ -351,10 +352,16 @@ pub struct TrieCachingStorage { } impl TrieCachingStorage { - pub fn new(store: Store, shard_cache: TrieCache, shard_uid: ShardUId) -> TrieCachingStorage { + pub fn new( + store: Store, + shard_cache: TrieCache, + shard_uid: ShardUId, + is_view: bool, + ) -> TrieCachingStorage { TrieCachingStorage { store, shard_uid, + is_view, shard_cache, cache_mode: Cell::new(TrieCacheMode::CachingShard), chunk_cache: RefCell::new(Default::default()), @@ -400,15 +407,12 @@ impl TrieCachingStorage { impl TrieStorage for TrieCachingStorage { fn retrieve_raw_bytes(&self, hash: &CryptoHash) -> Result, StorageError> { - let mut guard = self.shard_cache.0.lock().expect(POISONED_LOCK_ERR); - let metrics_labels: [&str; 2] = - [&format!("{}", self.shard_uid.shard_id), &format!("{}", guard.is_view as u8)]; + [&format!("{}", self.shard_uid.shard_id), &format!("{}", self.is_view as u8)]; + metrics::CHUNK_CACHE_SIZE .with_label_values(&metrics_labels) .set(self.chunk_cache.borrow().len() as i64); - metrics::SHARD_CACHE_SIZE.with_label_values(&metrics_labels).set(guard.len() as i64); - // Try to get value from chunk cache containing nodes with cheaper access. We can do it for any `TrieCacheMode`, // because we charge for reading nodes only when `CachingChunk` mode is enabled anyway. if let Some(val) = self.chunk_cache.borrow_mut().get(hash) { @@ -419,6 +423,8 @@ impl TrieStorage for TrieCachingStorage { metrics::CHUNK_CACHE_MISSES.with_label_values(&metrics_labels).inc(); // Try to get value from shard cache containing most recently touched nodes. + let mut guard = self.shard_cache.0.lock().expect(POISONED_LOCK_ERR); + metrics::SHARD_CACHE_SIZE.with_label_values(&metrics_labels).set(guard.len() as i64); let val = match guard.get(hash) { Some(val) => { metrics::SHARD_CACHE_HITS.with_label_values(&metrics_labels).inc(); diff --git a/core/store/src/trie/trie_tests.rs b/core/store/src/trie/trie_tests.rs index 377de774570..5ffcf08b896 100644 --- a/core/store/src/trie/trie_tests.rs +++ b/core/store/src/trie/trie_tests.rs @@ -235,7 +235,8 @@ mod caching_storage_tests { let shard_uid = ShardUId::single_shard(); let store = create_store_with_values(&values, shard_uid); let trie_cache = TrieCache::new(0, false); - let trie_caching_storage = TrieCachingStorage::new(store, trie_cache.clone(), shard_uid); + let trie_caching_storage = + TrieCachingStorage::new(store, trie_cache.clone(), shard_uid, false); let key = hash(&value); assert_eq!(trie_cache.get(&key), None); @@ -256,7 +257,7 @@ mod caching_storage_tests { let shard_uid = ShardUId::single_shard(); let store = create_test_store(); let trie_caching_storage = - TrieCachingStorage::new(store, TrieCache::new(0, false), shard_uid); + TrieCachingStorage::new(store, TrieCache::new(0, false), shard_uid, false); let value = vec![1u8]; let key = hash(&value); @@ -272,7 +273,8 @@ mod caching_storage_tests { let shard_uid = ShardUId::single_shard(); let store = create_store_with_values(&values, shard_uid); let trie_cache = TrieCache::new(0, false); - let trie_caching_storage = TrieCachingStorage::new(store, trie_cache.clone(), shard_uid); + let trie_caching_storage = + TrieCachingStorage::new(store, trie_cache.clone(), shard_uid, false); let key = hash(&value); trie_caching_storage.set_mode(TrieCacheMode::CachingChunk); @@ -294,7 +296,8 @@ mod caching_storage_tests { let shard_uid = ShardUId::single_shard(); let store = create_store_with_values(&values, shard_uid); let trie_cache = TrieCache::new(0, false); - let trie_caching_storage = TrieCachingStorage::new(store, trie_cache.clone(), shard_uid); + let trie_caching_storage = + TrieCachingStorage::new(store, trie_cache.clone(), shard_uid, false); let value = &values[0]; let key = hash(&value); @@ -342,7 +345,8 @@ mod caching_storage_tests { let shard_uid = ShardUId::single_shard(); let store = create_store_with_values(&values, shard_uid); let trie_cache = TrieCache::with_capacities(shard_cache_size, 0, false); - let trie_caching_storage = TrieCachingStorage::new(store, trie_cache.clone(), shard_uid); + let trie_caching_storage = + TrieCachingStorage::new(store, trie_cache.clone(), shard_uid, false); let value = &values[0]; let key = hash(&value); diff --git a/runtime/runtime-params-estimator/src/estimator_context.rs b/runtime/runtime-params-estimator/src/estimator_context.rs index fbfde1a3c3c..2ddb8b2b76f 100644 --- a/runtime/runtime-params-estimator/src/estimator_context.rs +++ b/runtime/runtime-params-estimator/src/estimator_context.rs @@ -123,8 +123,12 @@ impl<'c> Testbed<'c> { pub(crate) fn trie_caching_storage(&mut self) -> TrieCachingStorage { let store = self.inner.store(); - let caching_storage = - TrieCachingStorage::new(store, TrieCache::new(0, false), ShardUId::single_shard()); + let caching_storage = TrieCachingStorage::new( + store, + TrieCache::new(0, false), + ShardUId::single_shard(), + false, + ); caching_storage } From 381f5f9ef3d99084a2813e5bef207094d47ed7fa Mon Sep 17 00:00:00 2001 From: Longarithm Date: Sat, 20 Aug 2022 04:24:48 +0400 Subject: [PATCH 20/37] add new metrics --- core/store/src/metrics.rs | 45 ++++++++++++++++- core/store/src/trie/shard_tries.rs | 10 +++- core/store/src/trie/trie_storage.rs | 75 +++++++++++++++++------------ 3 files changed, 94 insertions(+), 36 deletions(-) diff --git a/core/store/src/metrics.rs b/core/store/src/metrics.rs index e81321774e3..8297b6d3244 100644 --- a/core/store/src/metrics.rs +++ b/core/store/src/metrics.rs @@ -69,6 +69,47 @@ pub static CHUNK_CACHE_SIZE: Lazy = Lazy::new(|| { .unwrap() }); -pub static SHARD_CACHE_POPS: Lazy = Lazy::new(|| { - try_create_int_counter_vec("near_shard_cache_pops", "Shard cache pops", &["shard_id"]).unwrap() +pub static SHARD_CACHE_POP_HITS: Lazy = Lazy::new(|| { + try_create_int_counter_vec("near_shard_cache_pop_hits", "Shard cache pop hits", &["shard_id"]) + .unwrap() +}); +pub static SHARD_CACHE_POP_MISSES: Lazy = Lazy::new(|| { + try_create_int_counter_vec( + "near_shard_cache_pop_misses", + "Shard cache pop misses", + &["shard_id"], + ) + .unwrap() +}); +pub static SHARD_CACHE_GC_POP_MISSES: Lazy = Lazy::new(|| { + try_create_int_counter_vec( + "near_shard_cache_gc_pop_misses", + "Shard cache gc pop misses", + &["shard_id"], + ) + .unwrap() +}); +pub static SHARD_CACHE_DELETIONS_SIZE: Lazy = Lazy::new(|| { + try_create_int_gauge_vec( + "near_shard_cache_deletions_size", + "Shard cache deletions size", + &["shard_id"], + ) + .unwrap() +}); +pub static APPLIED_TRIE_DELETIONS: Lazy = Lazy::new(|| { + try_create_int_counter_vec( + "near_applied_trie_deletions", + "Applied deletions to trie", + &["shard_id"], + ) + .unwrap() +}); +pub static APPLIED_TRIE_INSERTIONS: Lazy = Lazy::new(|| { + try_create_int_counter_vec( + "near_applied_trie_insertions", + "Applied insertions to trie", + &["shard_id"], + ) + .unwrap() }); diff --git a/core/store/src/trie/shard_tries.rs b/core/store/src/trie/shard_tries.rs index 0263f1166c9..01cb8bd9e51 100644 --- a/core/store/src/trie/shard_tries.rs +++ b/core/store/src/trie/shard_tries.rs @@ -21,7 +21,7 @@ use near_primitives::state_record::is_delayed_receipt_key; use crate::flat_state::FlatState; use crate::trie::trie_storage::{TrieCache, TrieCachingStorage}; use crate::trie::{TrieRefcountChange, POISONED_LOCK_ERR}; -use crate::{DBCol, DBOp, DBTransaction}; +use crate::{metrics, DBCol, DBOp, DBTransaction}; use crate::{Store, StoreUpdate, Trie, TrieChanges, TrieUpdate}; /// Responsible for creation of trie caches, stores necessary configuration for it. @@ -184,7 +184,7 @@ impl ShardTries { .entry(shard_uid) .or_insert_with(|| self.0.trie_cache_factory.create_cache(&shard_uid, false)) .clone(); - cache.update_cache(ops, shard_uid); + cache.update_cache(ops); } Ok(()) } @@ -243,6 +243,9 @@ impl ShardTries { shard_uid: ShardUId, store_update: &mut StoreUpdate, ) { + metrics::APPLIED_TRIE_INSERTIONS + .with_label_values(&[&format!("{}", shard_uid.shard_id)]) + .inc_by(trie_changes.insertions.len() as u64); self.apply_insertions_inner(&trie_changes.insertions, shard_uid, store_update) } @@ -252,6 +255,9 @@ impl ShardTries { shard_uid: ShardUId, store_update: &mut StoreUpdate, ) { + metrics::APPLIED_TRIE_DELETIONS + .with_label_values(&[&format!("{}", shard_uid.shard_id)]) + .inc_by(trie_changes.deletions.len() as u64); self.apply_deletions_inner(&trie_changes.deletions, shard_uid, store_update) } diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index 579a376c271..e74a4432781 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -41,6 +41,10 @@ impl BoundedQueue { None } } + + pub(crate) fn len(&self) -> usize { + self.queue.len() + } } /// In-memory cache for trie items - nodes and values. All nodes are stored in the LRU cache with three modifications. @@ -99,26 +103,34 @@ impl SyncTrieCache { } pub(crate) fn put(&mut self, key: CryptoHash, value: Arc<[u8]>) { - while self.total_size > self.total_size_limit { - let evicted_value = match self.deletions.pop() { - // First, try to evict value using the key from deletions queue. - Some(key) => self.cache.pop(&key), - // Second, pop LRU value. - None => Some( - self.cache.pop_lru().expect("Cannot fail because total size capacity is > 0").1, - ), - }; - match evicted_value { - Some(value) => { - self.total_size -= value.len() as u64; + let metrics_labels: [&str; 2] = + [&format!("{}", self.shard_id), &format!("{}", self.is_view as u8)]; + while self.total_size > self.total_size_limit || self.cache.len() == self.cache.cap() { + // First, try to evict value using the key from deletions queue. + match self.deletions.pop() { + Some(key) => match self.cache.pop(&key) { + Some(value) => { + metrics::SHARD_CACHE_POP_HITS.with_label_values(&metrics_labels).inc(); + self.total_size -= value.len() as u64; + } + None => {} + }, + None => { + metrics::SHARD_CACHE_POP_MISSES.with_label_values(&metrics_labels).inc(); } - None => {} - }; + } + + // Second, pop LRU value. + let (_, value) = + self.cache.pop_lru().expect("Cannot fail because total size capacity is > 0"); + self.total_size -= value.len() as u64; } + // Add value to the cache. self.total_size += value.len() as u64; match self.cache.push(key, value) { Some((_, evicted_value)) => { + // TODO: warn that it should never happen self.total_size -= evicted_value.len() as u64; } None => {} @@ -128,20 +140,30 @@ impl SyncTrieCache { // Adds key to the deletions queue if it is present in cache. // Returns key-value pair which are popped if deletions queue is full. pub(crate) fn pop(&mut self, key: &CryptoHash) -> Option<(CryptoHash, Arc<[u8]>)> { + let metrics_labels: [&str; 2] = + [&format!("{}", self.shard_id), &format!("{}", self.is_view as u8)]; + metrics::SHARD_CACHE_DELETIONS_SIZE + .with_label_values(&metrics_labels) + .set(self.deletions.len() as i64); // Do nothing if key was removed before. if self.cache.contains(key) { // Put key to the queue of deletions and possibly remove another key we have to delete. match self.deletions.put(key.clone()) { Some(key_to_delete) => match self.cache.pop(&key_to_delete) { Some(evicted_value) => { + metrics::SHARD_CACHE_POP_HITS.with_label_values(&metrics_labels).inc(); self.total_size -= evicted_value.len() as u64; Some((key_to_delete, evicted_value)) } - None => None, + None => { + metrics::SHARD_CACHE_POP_MISSES.with_label_values(&metrics_labels).inc(); + None + } }, None => None, } } else { + metrics::SHARD_CACHE_GC_POP_MISSES.with_label_values(&metrics_labels).inc(); None } } @@ -178,34 +200,23 @@ impl TrieCache { self.0.lock().expect(POISONED_LOCK_ERR).clear() } - pub fn update_cache(&self, ops: Vec<(CryptoHash, Option<&Vec>)>, shard_uid: ShardUId) { - let shard_id_str = format!("{}", shard_uid.shard_id); - let labels: [&str; 1] = [&shard_id_str]; - + pub fn update_cache(&self, ops: Vec<(CryptoHash, Option<&Vec>)>) { let mut guard = self.0.lock().expect(POISONED_LOCK_ERR); + let metrics_labels: [&str; 2] = + [&format!("{}", guard.shard_id), &format!("{}", guard.is_view as u8)]; for (hash, opt_value_rc) in ops { if let Some(value_rc) = opt_value_rc { if let (Some(value), _rc) = decode_value_with_rc(&value_rc) { if value.len() < TRIE_LIMIT_CACHED_VALUE_SIZE { guard.put(hash, value.into()); } else { - metrics::SHARD_CACHE_TOO_LARGE.with_label_values(&labels).inc(); + metrics::SHARD_CACHE_TOO_LARGE.with_label_values(&metrics_labels).inc(); } } else { - match guard.pop(&hash) { - Some(_) => { - metrics::SHARD_CACHE_POPS.with_label_values(&labels).inc(); - } - _ => {} - }; + guard.pop(&hash); } } else { - match guard.pop(&hash) { - Some(_) => { - metrics::SHARD_CACHE_POPS.with_label_values(&labels).inc(); - } - _ => {} - }; + guard.pop(&hash); } } } From 5de3996f519cd52123399e538e3f15746315e6ed Mon Sep 17 00:00:00 2001 From: Longarithm Date: Sat, 20 Aug 2022 04:28:52 +0400 Subject: [PATCH 21/37] add default capacities --- core/store/src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/store/src/config.rs b/core/store/src/config.rs index 2551534612f..b51131377fa 100644 --- a/core/store/src/config.rs +++ b/core/store/src/config.rs @@ -102,7 +102,7 @@ impl Default for StoreConfig { // we use it since then. block_size: bytesize::ByteSize::kib(16), - trie_cache_capacities: Default::default(), + trie_cache_capacities: vec![(ShardUId { version: 1, shard_id: 3 }, 2_000_000)], } } } From 27ef6ace9b4b29e804ac22083959fc66a7284944 Mon Sep 17 00:00:00 2001 From: Longarithm Date: Sat, 20 Aug 2022 04:29:59 +0400 Subject: [PATCH 22/37] size limit --- core/store/src/trie/trie_storage.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index e74a4432781..f0ee80f6405 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -324,7 +324,7 @@ const TRIE_DEFAULT_SHARD_CACHE_SIZE: usize = 50000; const TRIE_DEFAULT_SHARD_CACHE_SIZE: usize = 1; #[cfg(not(feature = "no_cache"))] -const DEFAULT_SHARD_CACHE_TOTAL_SIZE_LIMIT: u64 = 50_000_000; // consider 4_500_000_000 +const DEFAULT_SHARD_CACHE_TOTAL_SIZE_LIMIT: u64 = 3_000_000_000; #[cfg(feature = "no_cache")] const DEFAULT_SHARD_CACHE_TOTAL_SIZE_LIMIT: u64 = 1; From aa4df83db2d4585f5f9a33d7fda990cfdb32bb6e Mon Sep 17 00:00:00 2001 From: Longarithm Date: Sat, 20 Aug 2022 04:33:02 +0400 Subject: [PATCH 23/37] continue --- core/store/src/trie/trie_storage.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index f0ee80f6405..9bec8824d33 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -112,6 +112,7 @@ impl SyncTrieCache { Some(value) => { metrics::SHARD_CACHE_POP_HITS.with_label_values(&metrics_labels).inc(); self.total_size -= value.len() as u64; + continue; } None => {} }, From 14028bf1fcfa1747dea49f3f105e5526dd7abb22 Mon Sep 17 00:00:00 2001 From: Longarithm Date: Mon, 22 Aug 2022 14:59:17 +0400 Subject: [PATCH 24/37] fix labels --- core/store/src/metrics.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/core/store/src/metrics.rs b/core/store/src/metrics.rs index 8297b6d3244..c021363a960 100644 --- a/core/store/src/metrics.rs +++ b/core/store/src/metrics.rs @@ -70,14 +70,18 @@ pub static CHUNK_CACHE_SIZE: Lazy = Lazy::new(|| { }); pub static SHARD_CACHE_POP_HITS: Lazy = Lazy::new(|| { - try_create_int_counter_vec("near_shard_cache_pop_hits", "Shard cache pop hits", &["shard_id"]) - .unwrap() + try_create_int_counter_vec( + "near_shard_cache_pop_hits", + "Shard cache pop hits", + &["shard_id", "is_view"], + ) + .unwrap() }); pub static SHARD_CACHE_POP_MISSES: Lazy = Lazy::new(|| { try_create_int_counter_vec( "near_shard_cache_pop_misses", "Shard cache pop misses", - &["shard_id"], + &["shard_id", "is_view"], ) .unwrap() }); @@ -85,7 +89,7 @@ pub static SHARD_CACHE_GC_POP_MISSES: Lazy = Lazy::new(|| { try_create_int_counter_vec( "near_shard_cache_gc_pop_misses", "Shard cache gc pop misses", - &["shard_id"], + &["shard_id", "is_view"], ) .unwrap() }); @@ -93,7 +97,7 @@ pub static SHARD_CACHE_DELETIONS_SIZE: Lazy = Lazy::new(|| { try_create_int_gauge_vec( "near_shard_cache_deletions_size", "Shard cache deletions size", - &["shard_id"], + &["shard_id", "is_view"], ) .unwrap() }); From 3c6b38ae117fd62b5c470132733d366fd4a6aa94 Mon Sep 17 00:00:00 2001 From: Longarithm Date: Mon, 22 Aug 2022 15:09:29 +0400 Subject: [PATCH 25/37] fix capacity --- core/store/src/config.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/store/src/config.rs b/core/store/src/config.rs index b51131377fa..f38687e067a 100644 --- a/core/store/src/config.rs +++ b/core/store/src/config.rs @@ -41,7 +41,7 @@ pub struct StoreConfig { pub block_size: bytesize::ByteSize, /// Trie cache capacities - /// Default value: ShardUId {version: 1, shard_id: 3} -> 2_000_000. TODO: clarify + /// Default value: ShardUId {version: 1, shard_id: 3} -> 45_000_000. TODO: clarify /// We're still experimenting with this parameter and it seems decreasing its value can improve /// the performance of the storage pub trie_cache_capacities: Vec<(ShardUId, usize)>, @@ -102,7 +102,7 @@ impl Default for StoreConfig { // we use it since then. block_size: bytesize::ByteSize::kib(16), - trie_cache_capacities: vec![(ShardUId { version: 1, shard_id: 3 }, 2_000_000)], + trie_cache_capacities: vec![(ShardUId { version: 1, shard_id: 3 }, 45_000_000)], } } } From ce4c8426d6c002ab0c1cd7de2defca3e84347827 Mon Sep 17 00:00:00 2001 From: firatNEAR Date: Mon, 22 Aug 2022 14:44:06 +0200 Subject: [PATCH 26/37] Add total_size metric --- core/store/src/metrics.rs | 9 +++++++++ core/store/src/trie/trie_storage.rs | 11 +++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/core/store/src/metrics.rs b/core/store/src/metrics.rs index c021363a960..4d148bba143 100644 --- a/core/store/src/metrics.rs +++ b/core/store/src/metrics.rs @@ -69,6 +69,15 @@ pub static CHUNK_CACHE_SIZE: Lazy = Lazy::new(|| { .unwrap() }); +pub static SHARD_CACHE_CURRENT_TOTAL_SIZE: Lazy = Lazy::new(|| { + try_create_int_gauge_vec( + "near_shard_cache_current_total_size", + "Shard cache current total size", + &["shard_id", "is_view"], + ) + .unwrap() +}); + pub static SHARD_CACHE_POP_HITS: Lazy = Lazy::new(|| { try_create_int_counter_vec( "near_shard_cache_pop_hits", diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index 9bec8824d33..28af5c14987 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -69,9 +69,9 @@ pub struct SyncTrieCache { /// Upper bound for the total size. total_size_limit: u64, /// Shard id of the nodes being cached. - pub shard_id: u32, + shard_id: u32, /// Whether cache is used for view calls execution. - pub is_view: bool, + is_view: bool, } impl SyncTrieCache { @@ -172,6 +172,10 @@ impl SyncTrieCache { pub fn len(&self) -> usize { self.cache.len() } + + pub fn current_total_size(&self) -> u64 { + self.total_size + } } /// Wrapper over LruCache to handle concurrent access. @@ -437,6 +441,9 @@ impl TrieStorage for TrieCachingStorage { // Try to get value from shard cache containing most recently touched nodes. let mut guard = self.shard_cache.0.lock().expect(POISONED_LOCK_ERR); metrics::SHARD_CACHE_SIZE.with_label_values(&metrics_labels).set(guard.len() as i64); + metrics::SHARD_CACHE_CURRENT_TOTAL_SIZE + .with_label_values(&metrics_labels) + .set(self.shard_cache.0.lock().expect(POISONED_LOCK_ERR).current_total_size() as i64); let val = match guard.get(hash) { Some(val) => { metrics::SHARD_CACHE_HITS.with_label_values(&metrics_labels).inc(); From 474af0a1bacc10fc77953a787aacda2f531999e6 Mon Sep 17 00:00:00 2001 From: firatNEAR Date: Mon, 22 Aug 2022 21:23:41 +0200 Subject: [PATCH 27/37] Fix mutex lock --- core/store/src/trie/trie_storage.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index 28af5c14987..a2d79fdc4e0 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -443,7 +443,7 @@ impl TrieStorage for TrieCachingStorage { metrics::SHARD_CACHE_SIZE.with_label_values(&metrics_labels).set(guard.len() as i64); metrics::SHARD_CACHE_CURRENT_TOTAL_SIZE .with_label_values(&metrics_labels) - .set(self.shard_cache.0.lock().expect(POISONED_LOCK_ERR).current_total_size() as i64); + .set(guard.current_total_size() as i64); let val = match guard.get(hash) { Some(val) => { metrics::SHARD_CACHE_HITS.with_label_values(&metrics_labels).inc(); From 2dd713e1c56010a949f9a771f72846aa51995299 Mon Sep 17 00:00:00 2001 From: Longarithm Date: Tue, 23 Aug 2022 20:02:40 +0400 Subject: [PATCH 28/37] move deletions_into call to commit --- chain/chain/src/chain.rs | 10 +++++----- chain/chain/src/store.rs | 11 ++--------- chain/chain/src/tests/gc.rs | 2 +- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/chain/chain/src/chain.rs b/chain/chain/src/chain.rs index 24b0069243c..c1e2791d2bd 100644 --- a/chain/chain/src/chain.rs +++ b/chain/chain/src/chain.rs @@ -4533,7 +4533,7 @@ impl<'a> ChainUpdate<'a> { &result.shard_uid, new_chunk_extra, ); - self.chain_store_update.save_trie_changes(result.trie_changes)?; + self.chain_store_update.save_trie_changes(result.trie_changes); } assert_eq!(sum_gas_used, total_gas_used); assert_eq!(sum_balance_burnt, total_balance_burnt); @@ -4580,7 +4580,7 @@ impl<'a> ChainUpdate<'a> { apply_result.total_balance_burnt, ), ); - self.chain_store_update.save_trie_changes(apply_result.trie_changes)?; + self.chain_store_update.save_trie_changes(apply_result.trie_changes); self.chain_store_update.save_outgoing_receipt( &block_hash, shard_id, @@ -4614,7 +4614,7 @@ impl<'a> ChainUpdate<'a> { *new_extra.state_root_mut() = apply_result.new_root; self.chain_store_update.save_chunk_extra(&block_hash, &shard_uid, new_extra); - self.chain_store_update.save_trie_changes(apply_result.trie_changes)?; + self.chain_store_update.save_trie_changes(apply_result.trie_changes); if let Some(apply_results_or_state_changes) = apply_split_result_or_state_changes { self.process_split_state( @@ -4966,7 +4966,7 @@ impl<'a> ChainUpdate<'a> { self.chain_store_update.save_chunk(chunk); - self.chain_store_update.save_trie_changes(apply_result.trie_changes)?; + self.chain_store_update.save_trie_changes(apply_result.trie_changes); let chunk_extra = ChunkExtra::new( &apply_result.new_root, outcome_root, @@ -5045,7 +5045,7 @@ impl<'a> ChainUpdate<'a> { Default::default(), )?; - self.chain_store_update.save_trie_changes(apply_result.trie_changes)?; + self.chain_store_update.save_trie_changes(apply_result.trie_changes); let mut new_chunk_extra = ChunkExtra::clone(&chunk_extra); *new_chunk_extra.state_root_mut() = apply_result.new_root; diff --git a/chain/chain/src/store.rs b/chain/chain/src/store.rs index 9ea30e8a0dd..c6d26b2e63f 100644 --- a/chain/chain/src/store.rs +++ b/chain/chain/src/store.rs @@ -1818,16 +1818,8 @@ impl<'a> ChainStoreUpdate<'a> { self.chain_store_cache_update.outcome_ids.insert((*block_hash, shard_id), outcome_ids); } - // fix me: catch error everywhere - pub fn save_trie_changes(&mut self, trie_changes: WrappedTrieChanges) -> Result<(), Error> { - // Pop deleted trie nodes from the shard cache. - // TODO: get rid of creating `StoreUpdate` - let mut store_update = self.store().store_update(); - trie_changes.deletions_into(&mut store_update); - store_update.update_cache()?; - + pub fn save_trie_changes(&mut self, trie_changes: WrappedTrieChanges) { self.trie_changes.push(trie_changes); - Ok(()) } pub fn add_state_changes_for_split_states( @@ -2785,6 +2777,7 @@ impl<'a> ChainStoreUpdate<'a> { } for mut wrapped_trie_changes in self.trie_changes.drain(..) { wrapped_trie_changes.insertions_into(&mut store_update); + wrapped_trie_changes.deletions_into(&mut store_update); wrapped_trie_changes.state_changes_into(&mut store_update); if self.chain_store.save_trie_changes { diff --git a/chain/chain/src/tests/gc.rs b/chain/chain/src/tests/gc.rs index 62b061a5a68..2c347916579 100644 --- a/chain/chain/src/tests/gc.rs +++ b/chain/chain/src/tests/gc.rs @@ -137,7 +137,7 @@ fn do_fork( Default::default(), *block.hash(), ); - store_update.save_trie_changes(wrapped_trie_changes).unwrap(); + store_update.save_trie_changes(wrapped_trie_changes); prev_state_roots[shard_id as usize] = new_root; trie_changes_shards.push(trie_changes_data); From 4f8ad9cb954850079e4bbc6d1c9b21eaff63f443 Mon Sep 17 00:00:00 2001 From: Longarithm Date: Tue, 23 Aug 2022 20:24:11 +0400 Subject: [PATCH 29/37] add log_assert --- core/store/src/trie/trie_storage.rs | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index a2d79fdc4e0..999b1332ce3 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -8,6 +8,7 @@ use crate::db::refcount::decode_value_with_rc; use crate::trie::POISONED_LOCK_ERR; use crate::{metrics, DBCol, StorageError, Store}; use lru::LruCache; +use near_o11y::log_assert; use near_primitives::shard_layout::ShardUId; use near_primitives::types::{TrieCacheMode, TrieNodesCount}; use std::cell::{Cell, RefCell}; @@ -59,7 +60,7 @@ impl BoundedQueue { /// to the queue. /// Needed to delay deletions when we have forks. In such case, many blocks can share same parent, and we want to keep /// old nodes in cache for a while to process all new roots. For example, it helps to read old state root. -pub struct SyncTrieCache { +pub struct TrieCacheInner { /// LRU cache keeping mapping from keys to values. cache: LruCache>, /// Queue of items which were popped, which postpones deletion of old nodes. @@ -74,7 +75,7 @@ pub struct SyncTrieCache { is_view: bool, } -impl SyncTrieCache { +impl TrieCacheInner { pub(crate) fn new( cache_capacity: usize, deletions_queue_capacity: usize, @@ -131,7 +132,7 @@ impl SyncTrieCache { self.total_size += value.len() as u64; match self.cache.push(key, value) { Some((_, evicted_value)) => { - // TODO: warn that it should never happen + log_assert!(false, "LRU cache with shard_id = {}, is_view = {} can't be full before inserting key {}", self.shard_id, self.is_view, key); self.total_size -= evicted_value.len() as u64; } None => {} @@ -180,7 +181,7 @@ impl SyncTrieCache { /// Wrapper over LruCache to handle concurrent access. #[derive(Clone)] -pub struct TrieCache(Arc>); +pub struct TrieCache(Arc>); impl TrieCache { pub fn new(shard_id: u32, is_view: bool) -> Self { @@ -188,7 +189,7 @@ impl TrieCache { } pub fn with_capacities(cap: usize, shard_id: u32, is_view: bool) -> Self { - Self(Arc::new(Mutex::new(SyncTrieCache::new( + Self(Arc::new(Mutex::new(TrieCacheInner::new( cap, DEFAULT_SHARD_CACHE_DELETIONS_QUEUE_CAPACITY, DEFAULT_SHARD_CACHE_TOTAL_SIZE_LIMIT, @@ -541,16 +542,16 @@ mod bounded_queue_tests { #[cfg(test)] mod trie_cache_tests { - use crate::trie::trie_storage::SyncTrieCache; + use crate::trie::trie_storage::TrieCacheInner; use near_primitives::hash::hash; - fn put_value(cache: &mut SyncTrieCache, value: &[u8]) { + fn put_value(cache: &mut TrieCacheInner, value: &[u8]) { cache.put(hash(value), value.into()); } #[test] fn test_size_limit() { - let mut cache = SyncTrieCache::new(100, 100, 5, 0, false); + let mut cache = TrieCacheInner::new(100, 100, 5, 0, false); // Add three values. Before each put, condition on total size should not be triggered. put_value(&mut cache, &[1, 1]); assert_eq!(cache.total_size, 2); @@ -568,7 +569,7 @@ mod trie_cache_tests { #[test] fn test_deletions_queue() { - let mut cache = SyncTrieCache::new(100, 2, 100, 0, false); + let mut cache = TrieCacheInner::new(100, 2, 100, 0, false); // Add two values to the cache. put_value(&mut cache, &[1]); put_value(&mut cache, &[1, 1]); @@ -584,7 +585,7 @@ mod trie_cache_tests { #[test] fn test_cache_capacity() { - let mut cache = SyncTrieCache::new(2, 100, 100, 0, false); + let mut cache = TrieCacheInner::new(2, 100, 100, 0, false); put_value(&mut cache, &[1]); put_value(&mut cache, &[2]); put_value(&mut cache, &[3]); From b55898935b48c78c74e64f46f92ece0d3b126eff Mon Sep 17 00:00:00 2001 From: Longarithm Date: Tue, 23 Aug 2022 20:58:52 +0400 Subject: [PATCH 30/37] size assertion --- core/store/src/trie/trie_storage.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index 999b1332ce3..26728aec111 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -83,6 +83,7 @@ impl TrieCacheInner { shard_id: u32, is_view: bool, ) -> Self { + assert!(cache_capacity > 0 && total_size_limit > 0); Self { cache: LruCache::new(cache_capacity), deletions: BoundedQueue::new(deletions_queue_capacity), From 5eeca036e0bab3b9009dbeb32a30c42dc63aa480 Mon Sep 17 00:00:00 2001 From: Longarithm Date: Tue, 23 Aug 2022 21:39:59 +0400 Subject: [PATCH 31/37] fix log_assert and metrics --- core/store/src/metrics.rs | 8 ++++++++ core/store/src/trie/shard_tries.rs | 2 ++ core/store/src/trie/trie_storage.rs | 13 +++++++------ 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/core/store/src/metrics.rs b/core/store/src/metrics.rs index 4d148bba143..1cb2a25ccb7 100644 --- a/core/store/src/metrics.rs +++ b/core/store/src/metrics.rs @@ -94,6 +94,14 @@ pub static SHARD_CACHE_POP_MISSES: Lazy = Lazy::new(|| { ) .unwrap() }); +pub static SHARD_CACHE_POP_LRU: Lazy = Lazy::new(|| { + try_create_int_counter_vec( + "near_shard_cache_pop_lru", + "Shard cache LRU pops", + &["shard_id", "is_view"], + ) + .unwrap() +}); pub static SHARD_CACHE_GC_POP_MISSES: Lazy = Lazy::new(|| { try_create_int_counter_vec( "near_shard_cache_gc_pop_misses", diff --git a/core/store/src/trie/shard_tries.rs b/core/store/src/trie/shard_tries.rs index 01cb8bd9e51..52a9e79b7fe 100644 --- a/core/store/src/trie/shard_tries.rs +++ b/core/store/src/trie/shard_tries.rs @@ -332,10 +332,12 @@ impl WrappedTrieChanges { &self.state_changes } + /// Save insertions of trie nodes into Store. pub fn insertions_into(&self, store_update: &mut StoreUpdate) { self.tries.apply_insertions(&self.trie_changes, self.shard_uid, store_update) } + /// Save deletions of trie nodes into Store. pub fn deletions_into(&self, store_update: &mut StoreUpdate) { self.tries.apply_deletions(&self.trie_changes, self.shard_uid, store_update) } diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index 26728aec111..ef9b0263571 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -116,14 +116,15 @@ impl TrieCacheInner { self.total_size -= value.len() as u64; continue; } - None => {} + None => { + metrics::SHARD_CACHE_POP_MISSES.with_label_values(&metrics_labels).inc(); + } }, - None => { - metrics::SHARD_CACHE_POP_MISSES.with_label_values(&metrics_labels).inc(); - } + None => {} } // Second, pop LRU value. + metrics::SHARD_CACHE_POP_LRU.with_label_values(&metrics_labels).inc(); let (_, value) = self.cache.pop_lru().expect("Cannot fail because total size capacity is > 0"); self.total_size -= value.len() as u64; @@ -132,8 +133,8 @@ impl TrieCacheInner { // Add value to the cache. self.total_size += value.len() as u64; match self.cache.push(key, value) { - Some((_, evicted_value)) => { - log_assert!(false, "LRU cache with shard_id = {}, is_view = {} can't be full before inserting key {}", self.shard_id, self.is_view, key); + Some((evicted_key, evicted_value)) => { + log_assert!(key != evicted_key, "LRU cache with shard_id = {}, is_view = {} can't be full before inserting key {}", self.shard_id, self.is_view, key); self.total_size -= evicted_value.len() as u64; } None => {} From 7cb42ec1a3e12b641a072a6130a8317162dabeaa Mon Sep 17 00:00:00 2001 From: Longarithm Date: Tue, 23 Aug 2022 21:47:10 +0400 Subject: [PATCH 32/37] comments --- core/store/src/trie/trie_storage.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index ef9b0263571..7cba5d2f285 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -331,12 +331,15 @@ const TRIE_DEFAULT_SHARD_CACHE_SIZE: usize = 50000; #[cfg(feature = "no_cache")] const TRIE_DEFAULT_SHARD_CACHE_SIZE: usize = 1; +/// Default total size of values which may simultaneously exist the cache. +/// It is chosen by the estimation of the largest contract storage size we are aware as of 23/08/2022. #[cfg(not(feature = "no_cache"))] const DEFAULT_SHARD_CACHE_TOTAL_SIZE_LIMIT: u64 = 3_000_000_000; #[cfg(feature = "no_cache")] const DEFAULT_SHARD_CACHE_TOTAL_SIZE_LIMIT: u64 = 1; /// Capacity for the deletions queue. +/// It is chosen to fit all hashes of deleted nodes for 3 completely full blocks. #[cfg(not(feature = "no_cache"))] const DEFAULT_SHARD_CACHE_DELETIONS_QUEUE_CAPACITY: usize = 100_000; #[cfg(feature = "no_cache")] From 12413051a0c1dcd2d46baed030d6310dc18c52b5 Mon Sep 17 00:00:00 2001 From: Longarithm Date: Tue, 23 Aug 2022 21:48:48 +0400 Subject: [PATCH 33/37] fix assertion --- core/store/src/trie/trie_storage.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/store/src/trie/trie_storage.rs b/core/store/src/trie/trie_storage.rs index 7cba5d2f285..f961cbff5e8 100644 --- a/core/store/src/trie/trie_storage.rs +++ b/core/store/src/trie/trie_storage.rs @@ -134,7 +134,7 @@ impl TrieCacheInner { self.total_size += value.len() as u64; match self.cache.push(key, value) { Some((evicted_key, evicted_value)) => { - log_assert!(key != evicted_key, "LRU cache with shard_id = {}, is_view = {} can't be full before inserting key {}", self.shard_id, self.is_view, key); + log_assert!(key == evicted_key, "LRU cache with shard_id = {}, is_view = {} can't be full before inserting key {}", self.shard_id, self.is_view, key); self.total_size -= evicted_value.len() as u64; } None => {} From 824dfeef5f1e66bebd3a4d02569a65d6c6730f90 Mon Sep 17 00:00:00 2001 From: Longarithm Date: Wed, 24 Aug 2022 01:00:45 +0400 Subject: [PATCH 34/37] put deletions to separate store update --- chain/chain/src/store.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/chain/chain/src/store.rs b/chain/chain/src/store.rs index c6d26b2e63f..a342f611267 100644 --- a/chain/chain/src/store.rs +++ b/chain/chain/src/store.rs @@ -2775,9 +2775,14 @@ impl<'a> ChainStoreUpdate<'a> { block_hash, )?; } + + // Convert trie changes to database ops for trie nodes. + // Create separate store update for deletions, because we want to update cache and don't want to remove nodes + // from the store. + let mut deletions_store_update = self.store().store_update(); for mut wrapped_trie_changes in self.trie_changes.drain(..) { wrapped_trie_changes.insertions_into(&mut store_update); - wrapped_trie_changes.deletions_into(&mut store_update); + wrapped_trie_changes.deletions_into(&mut deletions_store_update); wrapped_trie_changes.state_changes_into(&mut store_update); if self.chain_store.save_trie_changes { @@ -2786,6 +2791,8 @@ impl<'a> ChainStoreUpdate<'a> { .map_err(|err| Error::Other(err.to_string()))?; } } + deletions_store_update.update_cache()?; + for ((block_hash, shard_id), state_changes) in self.add_state_changes_for_split_states.drain() { From 3c31f7cfb24055f10a21661fdc97613235d349b7 Mon Sep 17 00:00:00 2001 From: Longarithm Date: Fri, 26 Aug 2022 18:14:46 +0400 Subject: [PATCH 35/37] update changelog --- CHANGELOG.md | 1 + core/store/src/config.rs | 2 +- core/store/src/metrics.rs | 12 ++++++++++-- core/store/src/trie/shard_tries.rs | 3 +++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1253f05ff6e..c9f8c9114fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ * `network.external_address` field in `config.json` file is deprecated. In fact it has never been used and only served to confuse everyone [#7300](https://github.com/near/nearcore/pull/7300) +* Improved shard cache for Trie nodes to use RAM more effectively [#7429](https://github.com/near/nearcore/pull/7429) ## 1.28.0 [2022-07-27] diff --git a/core/store/src/config.rs b/core/store/src/config.rs index 0efa82d5655..7781a4b4801 100644 --- a/core/store/src/config.rs +++ b/core/store/src/config.rs @@ -43,7 +43,7 @@ pub struct StoreConfig { pub block_size: bytesize::ByteSize, /// Trie cache capacities - /// Default value: ShardUId {version: 1, shard_id: 3} -> 45_000_000. TODO: clarify + /// Default value: ShardUId {version: 1, shard_id: 3} -> 45_000_000 /// We're still experimenting with this parameter and it seems decreasing its value can improve /// the performance of the storage pub trie_cache_capacities: Vec<(ShardUId, usize)>, diff --git a/core/store/src/metrics.rs b/core/store/src/metrics.rs index 1cb2a25ccb7..3477ebac715 100644 --- a/core/store/src/metrics.rs +++ b/core/store/src/metrics.rs @@ -121,7 +121,7 @@ pub static SHARD_CACHE_DELETIONS_SIZE: Lazy = Lazy::new(|| { pub static APPLIED_TRIE_DELETIONS: Lazy = Lazy::new(|| { try_create_int_counter_vec( "near_applied_trie_deletions", - "Applied deletions to trie", + "Trie deletions applied to store", &["shard_id"], ) .unwrap() @@ -129,7 +129,15 @@ pub static APPLIED_TRIE_DELETIONS: Lazy = Lazy::new(|| { pub static APPLIED_TRIE_INSERTIONS: Lazy = Lazy::new(|| { try_create_int_counter_vec( "near_applied_trie_insertions", - "Applied insertions to trie", + "Trie insertions applied to store", + &["shard_id"], + ) + .unwrap() +}); +pub static REVERTED_TRIE_INSERTIONS: Lazy = Lazy::new(|| { + try_create_int_counter_vec( + "near_reverted_trie_insertions", + "Trie insertions reverted due to GC of forks", &["shard_id"], ) .unwrap() diff --git a/core/store/src/trie/shard_tries.rs b/core/store/src/trie/shard_tries.rs index 52a9e79b7fe..6d16685d1e4 100644 --- a/core/store/src/trie/shard_tries.rs +++ b/core/store/src/trie/shard_tries.rs @@ -267,6 +267,9 @@ impl ShardTries { shard_uid: ShardUId, store_update: &mut StoreUpdate, ) { + metrics::REVERTED_TRIE_INSERTIONS + .with_label_values(&[&format!("{}", shard_uid.shard_id)]) + .inc_by(trie_changes.insertions.len() as u64); self.apply_deletions_inner(&trie_changes.insertions, shard_uid, store_update) } From 470a0635788338eeefbbecc21961b8f9bcc1204a Mon Sep 17 00:00:00 2001 From: Longarithm Date: Fri, 26 Aug 2022 18:41:09 +0400 Subject: [PATCH 36/37] mention ram in changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9f8c9114fb..bc006d2b273 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,9 @@ * `network.external_address` field in `config.json` file is deprecated. In fact it has never been used and only served to confuse everyone [#7300](https://github.com/near/nearcore/pull/7300) -* Improved shard cache for Trie nodes to use RAM more effectively [#7429](https://github.com/near/nearcore/pull/7429) +* Due to increasing state size, improved shard cache for Trie nodes to + put more nodes in memory. Requires 4 GB more RAM + [#7429](https://github.com/near/nearcore/pull/7429) ## 1.28.0 [2022-07-27] From 5666a84fe188951d5ad6eaa290aebac64e41f1da Mon Sep 17 00:00:00 2001 From: Longarithm Date: Fri, 26 Aug 2022 19:20:44 +0400 Subject: [PATCH 37/37] 4 -> 3 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc006d2b273..f11a5041084 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ deprecated. In fact it has never been used and only served to confuse everyone [#7300](https://github.com/near/nearcore/pull/7300) * Due to increasing state size, improved shard cache for Trie nodes to - put more nodes in memory. Requires 4 GB more RAM + put more nodes in memory. Requires 3 GB more RAM [#7429](https://github.com/near/nearcore/pull/7429) ## 1.28.0 [2022-07-27]