From 77d2a17dc77e7a9c9c1a1d639a7ebd7c066f338b Mon Sep 17 00:00:00 2001 From: Islam El-Ashi Date: Mon, 25 Sep 2023 10:18:09 +0300 Subject: [PATCH] feat: comprehensive fuzzing for BTreeMap (#143) Adds a fuzz test that can run indefinitely comparing the results of a stable BTreeMap to a standard BTreeMap. I ended up using `proptest` rather than the conventional `cargo fuzz` for two reasons: 1. `cargo fuzz` requires nightly rust. 2. With proptest, we can have a short version of the fuzz test that runs as part of CI (which is included in this commit). --- src/btreemap.rs | 2 +- src/btreemap/proptests.rs | 141 +++++++++++++++++++++++++++++++++++++- 2 files changed, 140 insertions(+), 3 deletions(-) diff --git a/src/btreemap.rs b/src/btreemap.rs index 827dc4a9..99f97523 100644 --- a/src/btreemap.rs +++ b/src/btreemap.rs @@ -1197,7 +1197,7 @@ mod test { use std::cell::RefCell; use std::rc::Rc; - fn make_memory() -> Rc>> { + pub(crate) fn make_memory() -> Rc>> { Rc::new(RefCell::new(Vec::new())) } diff --git a/src/btreemap/proptests.rs b/src/btreemap/proptests.rs index a023d177..1d7606a4 100644 --- a/src/btreemap/proptests.rs +++ b/src/btreemap/proptests.rs @@ -1,17 +1,92 @@ use crate::{ - btreemap::test::{b, btree_test}, + btreemap::{ + test::{b, btree_test, make_memory}, + BTreeMap, + }, storable::Blob, + Memory, }; use proptest::collection::btree_set as pset; use proptest::collection::vec as pvec; use proptest::prelude::*; -use std::collections::BTreeSet; +use std::collections::{BTreeMap as StdBTreeMap, BTreeSet}; use test_strategy::proptest; +#[derive(Debug, Clone)] +enum Operation { + Insert { key: Vec, value: Vec }, + Iter { from: usize, len: usize }, + Get(usize), + Remove(usize), +} + +// A custom strategy that gives unequal weights to the different operations. +// Note that `Insert` has a higher weight than `Remove` so that, on average, BTreeMaps +// are growing in size the more operations are executed. +fn operation_strategy() -> impl Strategy { + prop_oneof![ + 3 => (any::>(), any::>()) + .prop_map(|(key, value)| Operation::Insert { key, value }), + 1 => (any::(), any::()) + .prop_map(|(from, len)| Operation::Iter { from, len }), + 2 => (any::()).prop_map(Operation::Get), + 1 => (any::()).prop_map(Operation::Remove), + ] +} + fn arb_blob() -> impl Strategy> { pvec(0..u8::MAX, 0..10).prop_map(|v| Blob::<10>::try_from(v.as_slice()).unwrap()) } +// Runs a comprehensive test for the major stable BTreeMap operations. +// Results are validated against a standard BTreeMap. +#[proptest(cases = 10)] +fn comprehensive(#[strategy(pvec(operation_strategy(), 100..5_000))] ops: Vec) { + let mem = make_memory(); + let mut btree = BTreeMap::new(mem); + let mut std_btree = StdBTreeMap::new(); + + // Execute all the operations, validating that the stable btreemap behaves similarly to a std + // btreemap. + for op in ops.into_iter() { + execute_operation(&mut std_btree, &mut btree, op); + } +} + +// A comprehensive fuzz test that runs until it's explicitly terminated. To run: +// +// ``` +// cargo t comprehensive_fuzz -- --ignored --nocapture 2> comprehensive_fuzz.log +// ``` +// +// comprehensive_fuzz.log contains all the operations to help triage a failure. +#[test] +#[ignore] +fn comprehensive_fuzz() { + use proptest::strategy::ValueTree; + use proptest::test_runner::TestRunner; + let mut runner = TestRunner::default(); + + let mem = make_memory(); + let mut btree = BTreeMap::new(mem); + let mut std_btree = StdBTreeMap::new(); + + let mut i = 0; + + loop { + let operation = operation_strategy() + .new_tree(&mut runner) + .unwrap() + .current(); + execute_operation(&mut std_btree, &mut btree, operation); + i += 1; + if i % 1000 == 0 { + println!("=== Step {i} ==="); + println!("=== BTree Size: {}", btree.len()); + } + } +} + #[proptest(cases = 10)] fn insert(#[strategy(pset(arb_blob(), 1000..10_000))] keys: BTreeSet>) { btree_test(|mut btree| { @@ -61,3 +136,65 @@ fn map_upper_bound_iter(#[strategy(pvec(0u64..u64::MAX -1 , 10..100))] keys: Vec Ok(()) }); } + +// Given an operation, executes it on the given stable btreemap and standard btreemap, verifying +// that the result of the operation is equal in both btrees. +fn execute_operation( + std_btree: &mut StdBTreeMap, Vec>, + btree: &mut BTreeMap, Vec, M>, + op: Operation, +) { + match op { + Operation::Insert { key, value } => { + let std_res = std_btree.insert(key.clone(), value.clone()); + + eprintln!("Insert({}, {})", hex::encode(&key), hex::encode(&value)); + let res = btree.insert(key, value); + assert_eq!(std_res, res); + } + Operation::Iter { from, len } => { + assert_eq!(std_btree.len(), btree.len() as usize); + if std_btree.is_empty() { + return; + } + + let from = from % std_btree.len(); + let len = len % std_btree.len(); + + eprintln!("Iterate({}, {})", from, len); + let std_iter = std_btree.iter().skip(from).take(len); + let stable_iter = btree.iter().skip(from).take(len); + for ((k1, v1), (k2, v2)) in std_iter.zip(stable_iter) { + assert_eq!(k1, &k2); + assert_eq!(v1, &v2); + } + } + Operation::Get(idx) => { + assert_eq!(std_btree.len(), btree.len() as usize); + if std_btree.is_empty() { + return; + } + let idx = idx % std_btree.len(); + + if let Some((k, v)) = btree.iter().skip(idx).take(1).next() { + eprintln!("Get({})", hex::encode(&k)); + assert_eq!(std_btree.get(&k), Some(&v)); + assert_eq!(btree.get(&k), Some(v)); + } + } + Operation::Remove(idx) => { + assert_eq!(std_btree.len(), btree.len() as usize); + if std_btree.is_empty() { + return; + } + + let idx = idx % std_btree.len(); + + if let Some((k, v)) = btree.iter().skip(idx).take(1).next() { + eprintln!("Remove({})", hex::encode(&k)); + assert_eq!(std_btree.remove(&k), Some(v.clone())); + assert_eq!(btree.remove(&k), Some(v)); + } + } + }; +}