Change EntryInfo from enum to struct to reduce memory utilization

src/sync.rs

@@ -9,7 +9,7 @@ pub(crate) mod base_cache;
 mod builder;
 mod cache;
 mod deques;
-mod entry_info;
+pub(crate) mod entry_info;
 pub(crate) mod housekeeper;
 mod invalidator;
 mod segment;
@@ -68,14 +68,14 @@ impl<K> Clone for KeyHash<K> {
 
 pub(crate) struct KeyDate<K> {
     key: Arc<K>,
-    entry_info: EntryInfo,
+    entry_info: Arc<EntryInfo>,
 }
 
 impl<K> KeyDate<K> {
-    pub(crate) fn new(key: Arc<K>, entry_info: &EntryInfo) -> Self {
+    pub(crate) fn new(key: Arc<K>, entry_info: &Arc<EntryInfo>) -> Self {
         Self {
             key,
-            entry_info: entry_info.clone(),
+            entry_info: Arc::clone(entry_info),
         }
     }
 
@@ -91,15 +91,15 @@ impl<K> KeyDate<K> {
 pub(crate) struct KeyHashDate<K> {
     key: Arc<K>,
     hash: u64,
-    entry_info: EntryInfo,
+    entry_info: Arc<EntryInfo>,
 }
 
 impl<K> KeyHashDate<K> {
-    pub(crate) fn new(kh: KeyHash<K>, entry_info: &EntryInfo) -> Self {
+    pub(crate) fn new(kh: KeyHash<K>, entry_info: &Arc<EntryInfo>) -> Self {
         Self {
             key: kh.key,
             hash: kh.hash,
-            entry_info: entry_info.clone(),
+            entry_info: Arc::clone(entry_info),
         }
     }
 
@@ -173,7 +173,7 @@ type KeyDeqNodeAo<K> = NonNull<DeqNode<KeyHashDate<K>>>;
 // DeqNode for the write order queue.
 type KeyDeqNodeWo<K> = NonNull<DeqNode<KeyDate<K>>>;
 
-struct DeqNodes<K> {
+pub(crate) struct DeqNodes<K> {
     access_order_q_node: Option<KeyDeqNodeAo<K>>,
     write_order_q_node: Option<KeyDeqNodeWo<K>>,
 }
@@ -183,12 +183,12 @@ unsafe impl<K> Send for DeqNodes<K> {}
 
 pub(crate) struct ValueEntry<K, V> {
     pub(crate) value: V,
-    info: EntryInfo,
+    info: Arc<EntryInfo>,
     nodes: Mutex<DeqNodes<K>>,
 }
 
 impl<K, V> ValueEntry<K, V> {
-    fn new(value: V, entry_info: EntryInfo) -> Self {
+    fn new(value: V, entry_info: Arc<EntryInfo>) -> Self {
         Self {
             value,
             info: entry_info,
@@ -199,7 +199,7 @@ impl<K, V> ValueEntry<K, V> {
         }
     }
 
-    fn new_from(value: V, entry_info: EntryInfo, other: &Self) -> Self {
+    fn new_from(value: V, entry_info: Arc<EntryInfo>, other: &Self) -> Self {
         let nodes = {
             let other_nodes = other.nodes.lock();
             DeqNodes {
@@ -218,7 +218,7 @@ impl<K, V> ValueEntry<K, V> {
         }
     }
 
-    pub(crate) fn entry_info(&self) -> &EntryInfo {
+    pub(crate) fn entry_info(&self) -> &Arc<EntryInfo> {
         &self.info
     }
 
@@ -288,46 +288,6 @@ impl<K, V> AccessTime for Arc<ValueEntry<K, V>> {
     }
 }
 
-#[derive(Clone, Copy, Debug)]
-pub(crate) enum CacheFeatures {
-    Plain,
-    Weighted,
-}
-
-impl CacheFeatures {
-    pub(crate) fn new(is_weighter_defined: bool) -> Self {
-        if is_weighter_defined {
-            Self::Weighted
-        } else {
-            Self::Plain
-        }
-    }
-}
-
-pub(crate) struct ValueEntryBuilder(CacheFeatures);
-
-impl ValueEntryBuilder {
-    pub(crate) fn new(features: CacheFeatures) -> Self {
-        Self(features)
-    }
-
-    pub(crate) fn build<K, V>(&self, value: V, policy_weight: u32) -> ValueEntry<K, V> {
-        let info = EntryInfo::new(self.0, policy_weight);
-        ValueEntry::new(value, info)
-    }
-
-    pub(crate) fn build_from<K, V>(
-        &self,
-        value: V,
-        policy_weight: u32,
-        other: &ValueEntry<K, V>,
-    ) -> ValueEntry<K, V> {
-        let info = other.info.clone();
-        info.set_policy_weight(policy_weight);
-        ValueEntry::new_from(value, info, other)
-    }
-}
-
 pub(crate) enum ReadOp<K, V> {
     // u64 is the hash of the key.
     Hit(u64, Arc<ValueEntry<K, V>>, Instant),

src/sync/base_cache.rs

@@ -1,9 +1,10 @@
 use super::{
     deques::Deques,
+    entry_info::EntryInfo,
     housekeeper::{Housekeeper, InnerSync, SyncPace},
     invalidator::{GetOrRemoveEntry, InvalidationResult, Invalidator, KeyDateLite, PredicateFun},
-    AccessTime, CacheFeatures, KeyDate, KeyHash, KeyHashDate, KvEntry, PredicateId, ReadOp,
-    ValueEntry, ValueEntryBuilder, Weigher, WriteOp,
+    AccessTime, KeyDate, KeyHash, KeyHashDate, KvEntry, PredicateId, ReadOp, ValueEntry, Weigher,
+    WriteOp,
 };
 use crate::{
     common::{
@@ -321,7 +322,8 @@ where
 
     #[inline]
     fn new_value_entry(&self, value: V, policy_weight: u32) -> Arc<ValueEntry<K, V>> {
-        Arc::new(self.inner.value_entry_builder.build(value, policy_weight))
+        let info = Arc::new(EntryInfo::new(policy_weight));
+        Arc::new(ValueEntry::new(value, info))
     }
 
     #[inline]
@@ -331,11 +333,9 @@ where
         policy_weight: u32,
         other: &ValueEntry<K, V>,
     ) -> Arc<ValueEntry<K, V>> {
-        Arc::new(
-            self.inner
-                .value_entry_builder
-                .build_from(value, policy_weight, other),
-        )
+        let info = Arc::clone(other.entry_info());
+        info.set_policy_weight(policy_weight);
+        Arc::new(ValueEntry::new_from(value, info, other))
     }
 
     #[inline]
@@ -461,15 +461,12 @@ type CacheStore<K, V, S> = moka_cht::SegmentedHashMap<Arc<K>, Arc<ValueEntry<K,
 
 type CacheEntry<K, V> = (Arc<K>, Arc<ValueEntry<K, V>>);
 
-// type BoxedValueEntryBuilder<K, V> = Box<dyn ValueEntryBuilder<K, V> + Send + Sync + 'static>;
-
 pub(crate) struct Inner<K, V, S> {
     max_capacity: Option<u64>,
     entry_count: AtomicCell<u64>,
     weighted_size: AtomicCell<u64>,
     cache: CacheStore<K, V, S>,
     build_hasher: S,
-    value_entry_builder: ValueEntryBuilder,
     deques: Mutex<Deques<K>>,
     frequency_sketch: RwLock<FrequencySketch>,
     frequency_sketch_enabled: AtomicBool,
@@ -516,16 +513,12 @@ where
             build_hasher.clone(),
         );
 
-        let features = CacheFeatures::new(weigher.is_some());
-        let value_entry_builder = ValueEntryBuilder::new(features);
-
         Self {
             max_capacity: max_capacity.map(|n| n as u64),
             entry_count: Default::default(),
             weighted_size: Default::default(),
             cache,
             build_hasher,
-            value_entry_builder,
             deques: Mutex::new(Default::default()),
             frequency_sketch: RwLock::new(Default::default()),
             frequency_sketch_enabled: Default::default(),

src/sync/entry_info.rs

@@ -1,35 +1,17 @@
-use std::sync::{
-    atomic::{AtomicBool, AtomicU32, Ordering},
-    Arc,
-};
+use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
 
-use super::{AccessTime, CacheFeatures};
+use super::AccessTime;
 use crate::common::{atomic_time::AtomicInstant, time::Instant};
 
-// We use enum-based dynamic dispatch here, rather than using trait-object-based
-// dynamic dispatch. Our benchmark programs showed enum-based dispatch was slightly
-// (1% or 2%) faster than other in our use cases.
-
-pub(crate) enum EntryInfo {
-    Plain(Arc<Plain>),
-    Weighted(Arc<Weighted>),
-}
-
-#[derive(Default)]
-pub(crate) struct Plain {
-    is_admitted: AtomicBool,
-    last_accessed: AtomicInstant,
-    last_modified: AtomicInstant,
-}
-
-pub(crate) struct Weighted {
+pub(crate) struct EntryInfo {
     is_admitted: AtomicBool,
     last_accessed: AtomicInstant,
     last_modified: AtomicInstant,
     policy_weight: AtomicU32,
 }
 
-impl Weighted {
+impl EntryInfo {
+    #[inline]
     pub(crate) fn new(policy_weight: u32) -> Self {
         Self {
             is_admitted: Default::default(),
@@ -38,108 +20,51 @@ impl Weighted {
             policy_weight: AtomicU32::new(policy_weight),
         }
     }
-}
-
-impl Clone for EntryInfo {
-    fn clone(&self) -> Self {
-        match self {
-            Self::Plain(ei) => Self::Plain(Arc::clone(ei)),
-            Self::Weighted(ei) => Self::Weighted(Arc::clone(ei)),
-        }
-    }
-}
-
-impl EntryInfo {
-    #[inline]
-    pub(crate) fn new(features: CacheFeatures, policy_weight: u32) -> Self {
-        match features {
-            CacheFeatures::Plain => Self::Plain(Arc::new(Plain::default())),
-            CacheFeatures::Weighted => Self::Weighted(Arc::new(Weighted::new(policy_weight))),
-        }
-    }
 
     #[inline]
     pub(crate) fn is_admitted(&self) -> bool {
-        let v = match self {
-            Self::Plain(ei) => &ei.is_admitted,
-            Self::Weighted(ei) => &ei.is_admitted,
-        };
-        v.load(Ordering::Acquire)
+        self.is_admitted.load(Ordering::Acquire)
     }
 
     #[inline]
     pub(crate) fn set_is_admitted(&self, value: bool) {
-        let v = match self {
-            Self::Plain(ei) => &ei.is_admitted,
-            Self::Weighted(ei) => &ei.is_admitted,
-        };
-        v.store(value, Ordering::Release);
+        self.is_admitted.store(value, Ordering::Release);
     }
 
     #[inline]
     pub(crate) fn reset_timestamps(&self) {
-        match self {
-            Self::Plain(ei) => {
-                ei.last_accessed.reset();
-                ei.last_accessed.reset();
-            }
-            Self::Weighted(ei) => {
-                ei.last_accessed.reset();
-                ei.last_modified.reset();
-            }
-        }
+        self.last_accessed.reset();
+        self.last_modified.reset();
     }
 
     #[inline]
     pub(crate) fn policy_weight(&self) -> u32 {
-        match self {
-            Self::Plain(_) => 1,
-            Self::Weighted(ei) => ei.policy_weight.load(Ordering::Acquire),
-        }
+        self.policy_weight.load(Ordering::Acquire)
     }
 
     pub(crate) fn set_policy_weight(&self, size: u32) {
-        match self {
-            Self::Plain(_) => (),
-            Self::Weighted(ei) => ei.policy_weight.store(size, Ordering::Release),
-        }
+        self.policy_weight.store(size, Ordering::Release);
     }
 }
 
 impl AccessTime for EntryInfo {
     #[inline]
     fn last_accessed(&self) -> Option<Instant> {
-        let v = match self {
-            Self::Plain(ei) => &ei.last_accessed,
-            Self::Weighted(ei) => &ei.last_accessed,
-        };
-        v.instant()
+        self.last_accessed.instant()
     }
 
     #[inline]
     fn set_last_accessed(&self, timestamp: Instant) {
-        let v = match self {
-            Self::Plain(ei) => &ei.last_accessed,
-            Self::Weighted(ei) => &ei.last_accessed,
-        };
-        v.set_instant(timestamp);
+        self.last_accessed.set_instant(timestamp);
     }
 
     #[inline]
     fn last_modified(&self) -> Option<Instant> {
-        let v = match self {
-            Self::Plain(ei) => &ei.last_modified,
-            Self::Weighted(ei) => &ei.last_modified,
-        };
-        v.instant()
+        self.last_modified.instant()
     }
 
     #[inline]
     fn set_last_modified(&self, timestamp: Instant) {
-        let v = match self {
-            Self::Plain(ei) => &ei.last_modified,
-            Self::Weighted(ei) => &ei.last_modified,
-        };
-        v.set_instant(timestamp);
+        self.last_modified.set_instant(timestamp);
     }
 }