From a90a97579d9461863181acef9a0b7ea24f919a8e Mon Sep 17 00:00:00 2001 From: Valentin Huber Date: Thu, 23 Jan 2025 13:12:47 +0000 Subject: [PATCH 01/14] Introduce replaying fuzzer --- libafl/src/executors/mod.rs | 4 +- libafl/src/fuzzer/mod.rs | 8 +- libafl/src/fuzzer/replaying.rs | 839 +++++++++++++++++++++++++++++++++ 3 files changed, 846 insertions(+), 5 deletions(-) create mode 100644 libafl/src/fuzzer/replaying.rs diff --git a/libafl/src/executors/mod.rs b/libafl/src/executors/mod.rs index 993857df9e..3d14185b01 100644 --- a/libafl/src/executors/mod.rs +++ b/libafl/src/executors/mod.rs @@ -42,7 +42,7 @@ pub mod with_observers; pub mod hooks; /// How an execution finished. -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)] #[cfg_attr( any(not(feature = "serdeany_autoreg"), miri), expect(clippy::unsafe_derive_deserialize) @@ -68,7 +68,7 @@ pub enum ExitKind { } /// How one of the diffing executions finished. -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)] #[cfg_attr( any(not(feature = "serdeany_autoreg"), miri), expect(clippy::unsafe_derive_deserialize) diff --git a/libafl/src/fuzzer/mod.rs b/libafl/src/fuzzer/mod.rs index bbc1361442..b184a8a690 100644 --- a/libafl/src/fuzzer/mod.rs +++ b/libafl/src/fuzzer/mod.rs @@ -1,9 +1,11 @@ //! The `Fuzzer` is the main struct for a fuzz campaign. +pub mod replaying; + use alloc::{string::ToString, vec::Vec}; -use core::{fmt::Debug, time::Duration}; #[cfg(feature = "std")] -use std::hash::Hash; +use core::hash::Hash; +use core::{fmt::Debug, time::Duration}; #[cfg(feature = "std")] use fastbloom::BloomFilter; @@ -883,7 +885,7 @@ impl StdFuzzer { } } -#[cfg(feature = "std")] // hashing requires std +#[cfg(feature = "std")] impl StdFuzzer { /// Create a new [`StdFuzzer`], which, with a certain certainty, executes each input only once. /// diff --git a/libafl/src/fuzzer/replaying.rs b/libafl/src/fuzzer/replaying.rs new file mode 100644 index 0000000000..f7e05567a3 --- /dev/null +++ b/libafl/src/fuzzer/replaying.rs @@ -0,0 +1,839 @@ +//! Fuzzer instance that increases stability by executing the same input multiple times. + +use alloc::{borrow::Cow, string::ToString, vec::Vec}; +use core::{fmt::Debug, hash::Hash, marker::PhantomData}; + +use hashbrown::HashMap; +use libafl_bolts::{ + current_time, generic_hash_std, + tuples::{Handle, MatchName, MatchNameRef}, +}; +use serde::Serialize; + +#[cfg(feature = "std")] +use crate::fuzzer::BloomInputFilter; +#[cfg(feature = "introspection")] +use crate::monitors::PerfFeature; +use crate::{ + corpus::{Corpus, CorpusId, HasCurrentCorpusId, HasTestcase, Testcase}, + events::{ + CanSerializeObserver, Event, EventConfig, EventFirer, EventProcessor, ProgressReporter, + }, + executors::{Executor, ExitKind, HasObservers}, + feedbacks::Feedback, + fuzzer::{ + Evaluator, EvaluatorObservers, ExecuteInputResult, ExecutesInput, ExecutionProcessor, + Fuzzer, HasFeedback, HasObjective, HasScheduler, InputFilter, NopInputFilter, + STATS_TIMEOUT_DEFAULT, + }, + inputs::Input, + mark_feature_time, + monitors::{AggregatorOps, UserStats, UserStatsValue}, + observers::ObserversTuple, + schedulers::Scheduler, + stages::{HasCurrentStageId, StagesTuple}, + start_timer, + state::{ + HasCorpus, HasCurrentTestcase, HasExecutions, HasLastFoundTime, HasLastReportTime, + HasSolutions, MaybeHasClientPerfMonitor, Stoppable, + }, + Error, HasMetadata, +}; + +/// The maximum factor of times an input will be replayed, multiplied by `count`. +pub const MAX_REPLAY_FACTOR: usize = 5; + +/// A fuzzer instance for unstable targets that increases stability by executing the same input multiple times. +/// +/// The input will be executed at most `count` * [`MAX_REPLAY_FACTOR`] times. +#[derive(Debug)] +pub struct ReplayingFuzzer { + count: usize, + handles: Handle, + scheduler: CS, + feedback: F, + objective: OF, + input_filter: IF, +} + +impl HasScheduler<::Input, S> + for ReplayingFuzzer +where + S: HasCorpus, + CS: Scheduler<::Input, S>, +{ + type Scheduler = CS; + + fn scheduler(&self) -> &CS { + &self.scheduler + } + + fn scheduler_mut(&mut self) -> &mut CS { + &mut self.scheduler + } +} + +impl HasFeedback for ReplayingFuzzer { + type Feedback = F; + + fn feedback(&self) -> &Self::Feedback { + &self.feedback + } + + fn feedback_mut(&mut self) -> &mut Self::Feedback { + &mut self.feedback + } +} + +impl HasObjective for ReplayingFuzzer { + type Objective = OF; + + fn objective(&self) -> &OF { + &self.objective + } + + fn objective_mut(&mut self) -> &mut OF { + &mut self.objective + } +} + +impl ExecutionProcessor::Input, OT, S> + for ReplayingFuzzer +where + CS: Scheduler<::Input, S>, + EM: EventFirer<::Input, S> + CanSerializeObserver, + S: HasCorpus + MaybeHasClientPerfMonitor + HasCurrentTestcase + HasSolutions + HasLastFoundTime, + F: Feedback::Input, OT, S>, + OF: Feedback::Input, OT, S>, + OT: ObserversTuple<::Input, S> + Serialize, + ::Input: Input, + S::Solutions: Corpus::Input>, +{ + fn check_results( + &mut self, + state: &mut S, + manager: &mut EM, + input: &::Input, + observers: &OT, + exit_kind: &ExitKind, + ) -> Result { + let mut res = ExecuteInputResult::None; + + #[cfg(not(feature = "introspection"))] + let is_solution = self + .objective_mut() + .is_interesting(state, manager, input, observers, exit_kind)?; + + #[cfg(feature = "introspection")] + let is_solution = self + .objective_mut() + .is_interesting_introspection(state, manager, input, observers, exit_kind)?; + + if is_solution { + res = ExecuteInputResult::Solution; + } else { + #[cfg(not(feature = "introspection"))] + let corpus_worthy = self + .feedback_mut() + .is_interesting(state, manager, input, observers, exit_kind)?; + + #[cfg(feature = "introspection")] + let corpus_worthy = self + .feedback_mut() + .is_interesting_introspection(state, manager, input, observers, exit_kind)?; + + if corpus_worthy { + res = ExecuteInputResult::Corpus; + } + } + Ok(res) + } + + fn evaluate_execution( + &mut self, + state: &mut S, + manager: &mut EM, + input: ::Input, + observers: &OT, + exit_kind: &ExitKind, + send_events: bool, + ) -> Result<(ExecuteInputResult, Option), Error> { + let exec_res = self.check_results(state, manager, &input, observers, exit_kind)?; + let corpus_id = self.process_execution(state, manager, &input, &exec_res, observers)?; + if send_events { + self.serialize_and_dispatch(state, manager, input, &exec_res, observers, exit_kind)?; + } + if exec_res != ExecuteInputResult::None { + *state.last_found_time_mut() = current_time(); + } + Ok((exec_res, corpus_id)) + } + + fn serialize_and_dispatch( + &mut self, + state: &mut S, + manager: &mut EM, + input: ::Input, + exec_res: &ExecuteInputResult, + observers: &OT, + exit_kind: &ExitKind, + ) -> Result<(), Error> { + // Now send off the event + let observers_buf = match exec_res { + ExecuteInputResult::Corpus => { + if manager.should_send() { + // TODO, set None for fast targets + if manager.configuration() == EventConfig::AlwaysUnique { + None + } else { + manager.serialize_observers(observers)? + } + } else { + None + } + } + _ => None, + }; + + self.dispatch_event(state, manager, input, exec_res, observers_buf, exit_kind)?; + Ok(()) + } + + fn dispatch_event( + &mut self, + state: &mut S, + manager: &mut EM, + input: ::Input, + exec_res: &ExecuteInputResult, + observers_buf: Option>, + exit_kind: &ExitKind, + ) -> Result<(), Error> { + // Now send off the event + match exec_res { + ExecuteInputResult::Corpus => { + if manager.should_send() { + manager.fire( + state, + Event::NewTestcase { + input, + observers_buf, + exit_kind: *exit_kind, + corpus_size: state.corpus().count(), + client_config: manager.configuration(), + time: current_time(), + forward_id: None, + #[cfg(all(unix, feature = "std", feature = "multi_machine"))] + node_id: None, + }, + )?; + } + } + ExecuteInputResult::Solution => { + if manager.should_send() { + manager.fire( + state, + Event::Objective { + objective_size: state.solutions().count(), + time: current_time(), + }, + )?; + } + } + ExecuteInputResult::None => (), + } + Ok(()) + } + + /// Evaluate if a set of observation channels has an interesting state + fn process_execution( + &mut self, + state: &mut S, + manager: &mut EM, + input: &::Input, + exec_res: &ExecuteInputResult, + observers: &OT, + ) -> Result, Error> { + match exec_res { + ExecuteInputResult::None => { + self.feedback_mut().discard_metadata(state, input)?; + self.objective_mut().discard_metadata(state, input)?; + Ok(None) + } + ExecuteInputResult::Corpus => { + // Not a solution + self.objective_mut().discard_metadata(state, input)?; + + // Add the input to the main corpus + let mut testcase = Testcase::from(input.clone()); + #[cfg(feature = "track_hit_feedbacks")] + self.feedback_mut() + .append_hit_feedbacks(testcase.hit_feedbacks_mut())?; + self.feedback_mut() + .append_metadata(state, manager, observers, &mut testcase)?; + let id = state.corpus_mut().add(testcase)?; + self.scheduler_mut().on_add(state, id)?; + + Ok(Some(id)) + } + ExecuteInputResult::Solution => { + // Not interesting + self.feedback_mut().discard_metadata(state, input)?; + + // The input is a solution, add it to the respective corpus + let mut testcase = Testcase::from(input.clone()); + testcase.set_parent_id_optional(*state.corpus().current()); + if let Ok(mut tc) = state.current_testcase_mut() { + tc.found_objective(); + } + #[cfg(feature = "track_hit_feedbacks")] + self.objective_mut() + .append_hit_feedbacks(testcase.hit_objectives_mut())?; + self.objective_mut() + .append_metadata(state, manager, observers, &mut testcase)?; + state.solutions_mut().add(testcase)?; + + Ok(None) + } + } + } +} + +impl EvaluatorObservers::Input, S> + for ReplayingFuzzer +where + CS: Scheduler<::Input, S>, + E: HasObservers + Executor::Input, S, Self>, + E::Observers: MatchName + ObserversTuple<::Input, S> + Serialize, + EM: EventFirer<::Input, S> + CanSerializeObserver, + F: Feedback::Input, E::Observers, S>, + OF: Feedback::Input, E::Observers, S>, + S: HasCorpus + + HasSolutions + + MaybeHasClientPerfMonitor + + HasCurrentTestcase + + HasExecutions + + HasLastFoundTime, + ::Input: Input, + S::Solutions: Corpus::Input>, + O: Hash, +{ + /// Process one input, adding to the respective corpora if needed and firing the right events + #[inline] + fn evaluate_input_with_observers( + &mut self, + state: &mut S, + executor: &mut E, + manager: &mut EM, + input: ::Input, + send_events: bool, + ) -> Result<(ExecuteInputResult, Option), Error> { + let exit_kind = self.execute_input(state, executor, manager, &input)?; + let observers = executor.observers(); + + self.scheduler.on_evaluation(state, &input, &*observers)?; + + self.evaluate_execution(state, manager, input, &*observers, &exit_kind, send_events) + } +} + +impl Evaluator::Input, S> + for ReplayingFuzzer +where + CS: Scheduler<::Input, S>, + E: HasObservers + Executor::Input, S, Self>, + E::Observers: MatchName + ObserversTuple<::Input, S> + Serialize, + EM: EventFirer<::Input, S> + CanSerializeObserver, + F: Feedback::Input, E::Observers, S>, + OF: Feedback::Input, E::Observers, S>, + S: HasCorpus + + HasSolutions + + MaybeHasClientPerfMonitor + + HasCurrentTestcase + + HasLastFoundTime + + HasExecutions, + ::Input: Input, + S::Solutions: Corpus::Input>, + IF: InputFilter<::Input>, + O: Hash, +{ + fn evaluate_filtered( + &mut self, + state: &mut S, + executor: &mut E, + manager: &mut EM, + input: ::Input, + ) -> Result<(ExecuteInputResult, Option), Error> { + if self.input_filter.should_execute(&input) { + self.evaluate_input(state, executor, manager, input) + } else { + Ok((ExecuteInputResult::None, None)) + } + } + + /// Process one input, adding to the respective corpora if needed and firing the right events + #[inline] + fn evaluate_input_events( + &mut self, + state: &mut S, + executor: &mut E, + manager: &mut EM, + input: ::Input, + send_events: bool, + ) -> Result<(ExecuteInputResult, Option), Error> { + self.evaluate_input_with_observers(state, executor, manager, input, send_events) + } + + fn add_disabled_input( + &mut self, + state: &mut S, + input: ::Input, + ) -> Result { + let mut testcase = Testcase::from(input.clone()); + testcase.set_disabled(true); + // Add the disabled input to the main corpus + let id = state.corpus_mut().add_disabled(testcase)?; + Ok(id) + } + + /// Adds an input, even if it's not considered `interesting` by any of the executors + fn add_input( + &mut self, + state: &mut S, + executor: &mut E, + manager: &mut EM, + input: ::Input, + ) -> Result { + *state.last_found_time_mut() = current_time(); + + let exit_kind = self.execute_input(state, executor, manager, &input)?; + let observers = executor.observers(); + // Always consider this to be "interesting" + let mut testcase = Testcase::from(input.clone()); + + // Maybe a solution + #[cfg(not(feature = "introspection"))] + let is_solution: bool = + self.objective_mut() + .is_interesting(state, manager, &input, &*observers, &exit_kind)?; + + #[cfg(feature = "introspection")] + let is_solution = self.objective_mut().is_interesting_introspection( + state, + manager, + &input, + &*observers, + &exit_kind, + )?; + + if is_solution { + #[cfg(feature = "track_hit_feedbacks")] + self.objective_mut() + .append_hit_feedbacks(testcase.hit_objectives_mut())?; + self.objective_mut() + .append_metadata(state, manager, &*observers, &mut testcase)?; + let id = state.solutions_mut().add(testcase)?; + + manager.fire( + state, + Event::Objective { + objective_size: state.solutions().count(), + time: current_time(), + }, + )?; + return Ok(id); + } + + // Not a solution + self.objective_mut().discard_metadata(state, &input)?; + + // several is_interesting implementations collect some data about the run, later used in + // append_metadata; we *must* invoke is_interesting here to collect it + #[cfg(not(feature = "introspection"))] + let _corpus_worthy = + self.feedback_mut() + .is_interesting(state, manager, &input, &*observers, &exit_kind)?; + + #[cfg(feature = "introspection")] + let _corpus_worthy = self.feedback_mut().is_interesting_introspection( + state, + manager, + &input, + &*observers, + &exit_kind, + )?; + + #[cfg(feature = "track_hit_feedbacks")] + self.feedback_mut() + .append_hit_feedbacks(testcase.hit_feedbacks_mut())?; + // Add the input to the main corpus + self.feedback_mut() + .append_metadata(state, manager, &*observers, &mut testcase)?; + let id = state.corpus_mut().add(testcase)?; + self.scheduler_mut().on_add(state, id)?; + + let observers_buf = if manager.configuration() == EventConfig::AlwaysUnique { + None + } else { + manager.serialize_observers(&*observers)? + }; + manager.fire( + state, + Event::NewTestcase { + input, + observers_buf, + exit_kind, + corpus_size: state.corpus().count(), + client_config: manager.configuration(), + time: current_time(), + forward_id: None, + #[cfg(all(unix, feature = "std", feature = "multi_machine"))] + node_id: None, + }, + )?; + Ok(id) + } +} + +impl Fuzzer for ReplayingFuzzer +where + CS: Scheduler<::Input, S>, + EM: ProgressReporter + EventProcessor, + S: HasExecutions + + HasMetadata + + HasCorpus + + HasLastReportTime + + HasTestcase + + HasCurrentCorpusId + + HasCurrentStageId + + Stoppable + + MaybeHasClientPerfMonitor, + ST: StagesTuple, +{ + fn fuzz_one( + &mut self, + stages: &mut ST, + executor: &mut E, + state: &mut S, + manager: &mut EM, + ) -> Result { + // Init timer for scheduler + #[cfg(feature = "introspection")] + state.introspection_monitor_mut().start_timer(); + + // Get the next index from the scheduler + let id = if let Some(id) = state.current_corpus_id()? { + id // we are resuming + } else { + let id = self.scheduler.next(state)?; + state.set_corpus_id(id)?; // set up for resume + id + }; + + // Mark the elapsed time for the scheduler + #[cfg(feature = "introspection")] + state.introspection_monitor_mut().mark_scheduler_time(); + + // Mark the elapsed time for the scheduler + #[cfg(feature = "introspection")] + state.introspection_monitor_mut().reset_stage_index(); + + // Execute all stages + stages.perform_all(self, executor, state, manager)?; + + // Init timer for manager + #[cfg(feature = "introspection")] + state.introspection_monitor_mut().start_timer(); + + // Execute the manager + manager.process(self, state, executor)?; + + // Mark the elapsed time for the manager + #[cfg(feature = "introspection")] + state.introspection_monitor_mut().mark_manager_time(); + + { + if let Ok(mut testcase) = state.testcase_mut(id) { + let scheduled_count = testcase.scheduled_count(); + // increase scheduled count, this was fuzz_level in afl + testcase.set_scheduled_count(scheduled_count + 1); + } + } + + state.clear_corpus_id()?; + + if state.stop_requested() { + state.discard_stop_request(); + manager.on_shutdown()?; + return Err(Error::shutting_down()); + } + + Ok(id) + } + + fn fuzz_loop( + &mut self, + stages: &mut ST, + executor: &mut E, + state: &mut S, + manager: &mut EM, + ) -> Result<(), Error> { + let monitor_timeout = STATS_TIMEOUT_DEFAULT; + loop { + manager.maybe_report_progress(state, monitor_timeout)?; + + self.fuzz_one(stages, executor, state, manager)?; + } + } + + fn fuzz_loop_for( + &mut self, + stages: &mut ST, + executor: &mut E, + state: &mut S, + manager: &mut EM, + iters: u64, + ) -> Result { + if iters == 0 { + return Err(Error::illegal_argument( + "Cannot fuzz for 0 iterations!".to_string(), + )); + } + + let mut ret = None; + let monitor_timeout = STATS_TIMEOUT_DEFAULT; + + for _ in 0..iters { + manager.maybe_report_progress(state, monitor_timeout)?; + ret = Some(self.fuzz_one(stages, executor, state, manager)?); + } + + manager.report_progress(state)?; + + // If we would assume the fuzzer loop will always exit after this, we could do this here: + // manager.on_restart(state)?; + // But as the state may grow to a few megabytes, + // for now we won't, and the user has to do it (unless we find a way to do this on `Drop`). + + Ok(ret.unwrap()) + } +} + +impl ReplayingFuzzer { + /// Create a new [`StdFuzzer`] with standard behavior and the provided duplicate input execution filter. + pub fn with_input_filter( + count: usize, + handles: Handle, + scheduler: CS, + feedback: F, + objective: OF, + input_filter: IF, + ) -> Self { + Self { + count, + handles, + scheduler, + feedback, + objective, + input_filter, + } + } +} + +impl ReplayingFuzzer { + /// Create a new [`StdFuzzer`] with standard behavior and no duplicate input execution filtering. + pub fn new( + count: usize, + handles: Handle, + scheduler: CS, + feedback: F, + objective: OF, + ) -> Self { + Self::with_input_filter( + count, + handles, + scheduler, + feedback, + objective, + NopInputFilter, + ) + } +} + +#[cfg(feature = "std")] // hashing requires std +impl ReplayingFuzzer { + /// Create a new [`StdFuzzer`], which, with a certain certainty, executes each input only once. + /// + /// This is achieved by hashing each input and using a bloom filter to differentiate inputs. + /// + /// Use this implementation if hashing each input is very fast compared to executing potential duplicate inputs. + pub fn with_bloom_input_filter( + count: usize, + handles: Handle, + scheduler: CS, + feedback: F, + objective: OF, + items_count: usize, + fp_p: f64, + ) -> Self { + let input_filter = BloomInputFilter::new(items_count, fp_p); + Self::with_input_filter(count, handles, scheduler, feedback, objective, input_filter) + } +} + +impl ExecutesInput::Input, S> + for ReplayingFuzzer +where + CS: Scheduler<::Input, S>, + E: Executor::Input, S, Self> + HasObservers, + E::Observers: ObserversTuple<::Input, S> + MatchNameRef, + S: HasExecutions + HasCorpus + MaybeHasClientPerfMonitor, + O: Hash, + EM: EventFirer<::Input, S>, +{ + /// Runs the input and triggers observers and feedback + fn execute_input( + &mut self, + state: &mut S, + executor: &mut E, + event_mgr: &mut EM, + input: &::Input, + ) -> Result { + let mut results = HashMap::new(); + let (exit_kind, total_replayed) = loop { + start_timer!(state); + executor.observers_mut().pre_exec_all(state, input)?; + mark_feature_time!(state, PerfFeature::PreExecObservers); + + start_timer!(state); + let exit_kind = executor.run_target(self, state, event_mgr, input)?; + mark_feature_time!(state, PerfFeature::TargetExecution); + + start_timer!(state); + executor + .observers_mut() + .post_exec_all(state, input, &exit_kind)?; + + let observers = executor.observers(); + + mark_feature_time!(state, PerfFeature::PostExecObservers); + + let observer = observers.get(&self.handles).expect("observer not found"); + let hash = generic_hash_std(observer); + *results.entry((hash, exit_kind)).or_insert(0_usize) += 1; + + let total_replayed = results.values().sum::(); + + let ((max_hash, max_exit_kind), max_count) = + results.iter().max_by(|(_, a), (_, b)| a.cmp(b)).unwrap(); + + if *max_count < self.count { + continue; + } + + let consistent_enough = results + .values() + .filter(|e| **e != *max_count) + .all(|&count| count <= max_count - self.count); + + let latest_execution_is_dominant = hash == *max_hash && exit_kind == *max_exit_kind; + + if consistent_enough && latest_execution_is_dominant { + break (exit_kind, total_replayed); + } else if total_replayed >= MAX_REPLAY_FACTOR * self.count { + log::warn!( + "Replaying {} times did not lead to dominant result, using the latest observer value and most common exit_kind", + total_replayed + ); + break (*max_exit_kind, total_replayed); + } + }; + + event_mgr.fire( + state, + Event::UpdateUserStats { + name: Cow::Borrowed("consistency_replay_count"), + value: UserStats::new( + UserStatsValue::Number(total_replayed as u64), + AggregatorOps::Avg, + ), + phantom: PhantomData, + }, + )?; + + Ok(exit_kind) + } +} + +#[cfg(test)] +mod tests { + use alloc::rc::Rc; + use core::cell::RefCell; + + use libafl_bolts::{ + rands::StdRand, + tuples::{tuple_list, Handled}, + }; + + use crate::{ + corpus::InMemoryCorpus, + events::NopEventManager, + executors::{ExitKind, InProcessExecutor}, + fuzzer::ExecutesInput, + inputs::ValueInput, + observers::StdMapObserver, + replaying::ReplayingFuzzer, + schedulers::StdScheduler, + state::StdState, + }; + + #[test] + fn test_replaying() { + let map = Rc::new(RefCell::new(vec![0_usize])); + let return_value = Rc::new(RefCell::new(vec![0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])); + let mut map_borrow = map.borrow_mut(); + let observer = unsafe { + StdMapObserver::from_mut_ptr("observer", map_borrow.as_mut_ptr(), map_borrow.len()) + }; + drop(map_borrow); + let mut fuzzer = ReplayingFuzzer::new( + 2, + observer.handle(), + StdScheduler::new(), + tuple_list!(), + tuple_list!(), + ); + + let mut state = StdState::new( + StdRand::new(), + InMemoryCorpus::new(), + InMemoryCorpus::new(), + &mut tuple_list!(), + &mut tuple_list!(), + ) + .unwrap(); + let mut event_mgr = NopEventManager::new(); + let execution_count = Rc::new(RefCell::new(0)); + let mut harness = |_i: &ValueInput| { + let map_value = return_value.borrow_mut().remove(0); + map.borrow_mut()[0] = map_value; + *execution_count.borrow_mut() += 1; + + ExitKind::Ok + }; + let mut executor = InProcessExecutor::new( + &mut harness, + tuple_list!(observer), + &mut fuzzer, + &mut state, + &mut event_mgr, + ) + .unwrap(); + + let input: ValueInput = 42_usize.into(); + fuzzer + .execute_input(&mut state, &mut executor, &mut event_mgr, &input) + .unwrap(); + + assert_eq!(*execution_count.borrow(), 4); + } +} From ede449288d223108f991eed22e486cd2a6f66bb1 Mon Sep 17 00:00:00 2001 From: Valentin Huber Date: Thu, 23 Jan 2025 13:26:37 +0000 Subject: [PATCH 02/14] Make things more configurable, rename variables --- libafl/src/fuzzer/replaying.rs | 61 +++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/libafl/src/fuzzer/replaying.rs b/libafl/src/fuzzer/replaying.rs index f7e05567a3..dd12a613a3 100644 --- a/libafl/src/fuzzer/replaying.rs +++ b/libafl/src/fuzzer/replaying.rs @@ -40,16 +40,14 @@ use crate::{ Error, HasMetadata, }; -/// The maximum factor of times an input will be replayed, multiplied by `count`. -pub const MAX_REPLAY_FACTOR: usize = 5; - /// A fuzzer instance for unstable targets that increases stability by executing the same input multiple times. /// -/// The input will be executed at most `count` * [`MAX_REPLAY_FACTOR`] times. +/// The input will be executed as often as necessary until the most frequent result appears at least `min_count_diff` times more often than any other result and at most `max_trys` times. #[derive(Debug)] pub struct ReplayingFuzzer { - count: usize, - handles: Handle, + min_count_diff: usize, + max_trys: usize, + handle: Handle, scheduler: CS, feedback: F, objective: OF, @@ -621,16 +619,18 @@ where impl ReplayingFuzzer { /// Create a new [`StdFuzzer`] with standard behavior and the provided duplicate input execution filter. pub fn with_input_filter( - count: usize, - handles: Handle, + min_count_diff: usize, + max_trys: usize, + handle: Handle, scheduler: CS, feedback: F, objective: OF, input_filter: IF, ) -> Self { Self { - count, - handles, + min_count_diff, + max_trys, + handle, scheduler, feedback, objective, @@ -640,17 +640,19 @@ impl ReplayingFuzzer { } impl ReplayingFuzzer { - /// Create a new [`StdFuzzer`] with standard behavior and no duplicate input execution filtering. + /// Create a new [`ReplayingFuzzer`] with standard behavior and no duplicate input execution filtering. pub fn new( - count: usize, - handles: Handle, + min_count_diff: usize, + max_trys: usize, + handle: Handle, scheduler: CS, feedback: F, objective: OF, ) -> Self { Self::with_input_filter( - count, - handles, + min_count_diff, + max_trys, + handle, scheduler, feedback, objective, @@ -661,14 +663,16 @@ impl ReplayingFuzzer { #[cfg(feature = "std")] // hashing requires std impl ReplayingFuzzer { - /// Create a new [`StdFuzzer`], which, with a certain certainty, executes each input only once. + /// Create a new [`ReplayingFuzzer`], which, with a certain certainty, executes each input only once. /// /// This is achieved by hashing each input and using a bloom filter to differentiate inputs. /// /// Use this implementation if hashing each input is very fast compared to executing potential duplicate inputs. + #[expect(clippy::too_many_arguments)] pub fn with_bloom_input_filter( - count: usize, - handles: Handle, + min_count_diff: usize, + max_trys: usize, + handle: Handle, scheduler: CS, feedback: F, objective: OF, @@ -676,7 +680,15 @@ impl ReplayingFuzzer { fp_p: f64, ) -> Self { let input_filter = BloomInputFilter::new(items_count, fp_p); - Self::with_input_filter(count, handles, scheduler, feedback, objective, input_filter) + Self::with_input_filter( + min_count_diff, + max_trys, + handle, + scheduler, + feedback, + objective, + input_filter, + ) } } @@ -717,7 +729,7 @@ where mark_feature_time!(state, PerfFeature::PostExecObservers); - let observer = observers.get(&self.handles).expect("observer not found"); + let observer = observers.get(&self.handle).expect("observer not found"); let hash = generic_hash_std(observer); *results.entry((hash, exit_kind)).or_insert(0_usize) += 1; @@ -726,20 +738,20 @@ where let ((max_hash, max_exit_kind), max_count) = results.iter().max_by(|(_, a), (_, b)| a.cmp(b)).unwrap(); - if *max_count < self.count { + if *max_count < self.min_count_diff { continue; } let consistent_enough = results .values() .filter(|e| **e != *max_count) - .all(|&count| count <= max_count - self.count); + .all(|&count| count <= max_count - self.min_count_diff); let latest_execution_is_dominant = hash == *max_hash && exit_kind == *max_exit_kind; if consistent_enough && latest_execution_is_dominant { break (exit_kind, total_replayed); - } else if total_replayed >= MAX_REPLAY_FACTOR * self.count { + } else if total_replayed >= self.max_trys * self.min_count_diff { log::warn!( "Replaying {} times did not lead to dominant result, using the latest observer value and most common exit_kind", total_replayed @@ -753,7 +765,7 @@ where Event::UpdateUserStats { name: Cow::Borrowed("consistency_replay_count"), value: UserStats::new( - UserStatsValue::Number(total_replayed as u64), + UserStatsValue::Float(u32::try_from(total_replayed).unwrap().into()), AggregatorOps::Avg, ), phantom: PhantomData, @@ -797,6 +809,7 @@ mod tests { drop(map_borrow); let mut fuzzer = ReplayingFuzzer::new( 2, + 10, observer.handle(), StdScheduler::new(), tuple_list!(), From a095e7684a33567e5caab89b54996b2f03868784 Mon Sep 17 00:00:00 2001 From: Valentin Huber Date: Thu, 23 Jan 2025 15:07:07 +0000 Subject: [PATCH 03/14] Update to use new generics --- libafl/src/fuzzer/replaying.rs | 124 ++++++++++++++++----------------- 1 file changed, 59 insertions(+), 65 deletions(-) diff --git a/libafl/src/fuzzer/replaying.rs b/libafl/src/fuzzer/replaying.rs index dd12a613a3..6a812ff54e 100644 --- a/libafl/src/fuzzer/replaying.rs +++ b/libafl/src/fuzzer/replaying.rs @@ -54,11 +54,9 @@ pub struct ReplayingFuzzer { input_filter: IF, } -impl HasScheduler<::Input, S> - for ReplayingFuzzer +impl HasScheduler for ReplayingFuzzer where - S: HasCorpus, - CS: Scheduler<::Input, S>, + CS: Scheduler, { type Scheduler = CS; @@ -95,23 +93,26 @@ impl HasObjective for ReplayingFuzzer { } } -impl ExecutionProcessor::Input, OT, S> +impl ExecutionProcessor for ReplayingFuzzer where - CS: Scheduler<::Input, S>, - EM: EventFirer<::Input, S> + CanSerializeObserver, - S: HasCorpus + MaybeHasClientPerfMonitor + HasCurrentTestcase + HasSolutions + HasLastFoundTime, - F: Feedback::Input, OT, S>, - OF: Feedback::Input, OT, S>, - OT: ObserversTuple<::Input, S> + Serialize, - ::Input: Input, - S::Solutions: Corpus::Input>, + CS: Scheduler, + EM: EventFirer + CanSerializeObserver, + F: Feedback, + I: Input, + OF: Feedback, + OT: ObserversTuple + Serialize, + S: HasCorpus + + MaybeHasClientPerfMonitor + + HasCurrentTestcase + + HasSolutions + + HasLastFoundTime, { fn check_results( &mut self, state: &mut S, manager: &mut EM, - input: &::Input, + input: &I, observers: &OT, exit_kind: &ExitKind, ) -> Result { @@ -151,7 +152,7 @@ where &mut self, state: &mut S, manager: &mut EM, - input: ::Input, + input: I, observers: &OT, exit_kind: &ExitKind, send_events: bool, @@ -171,7 +172,7 @@ where &mut self, state: &mut S, manager: &mut EM, - input: ::Input, + input: I, exec_res: &ExecuteInputResult, observers: &OT, exit_kind: &ExitKind, @@ -201,7 +202,7 @@ where &mut self, state: &mut S, manager: &mut EM, - input: ::Input, + input: I, exec_res: &ExecuteInputResult, observers_buf: Option>, exit_kind: &ExitKind, @@ -247,7 +248,7 @@ where &mut self, state: &mut S, manager: &mut EM, - input: &::Input, + input: &I, exec_res: &ExecuteInputResult, observers: &OT, ) -> Result, Error> { @@ -296,23 +297,22 @@ where } } -impl EvaluatorObservers::Input, S> +impl EvaluatorObservers for ReplayingFuzzer where - CS: Scheduler<::Input, S>, - E: HasObservers + Executor::Input, S, Self>, - E::Observers: MatchName + ObserversTuple<::Input, S> + Serialize, - EM: EventFirer<::Input, S> + CanSerializeObserver, - F: Feedback::Input, E::Observers, S>, - OF: Feedback::Input, E::Observers, S>, - S: HasCorpus - + HasSolutions + CS: Scheduler, + E: HasObservers + Executor, + E::Observers: MatchName + ObserversTuple + Serialize, + EM: EventFirer + CanSerializeObserver, + F: Feedback, + OF: Feedback, + S: HasCorpus + + HasSolutions + MaybeHasClientPerfMonitor - + HasCurrentTestcase + + HasCurrentTestcase + HasExecutions + HasLastFoundTime, - ::Input: Input, - S::Solutions: Corpus::Input>, + I: Input, O: Hash, { /// Process one input, adding to the respective corpora if needed and firing the right events @@ -322,7 +322,7 @@ where state: &mut S, executor: &mut E, manager: &mut EM, - input: ::Input, + input: I, send_events: bool, ) -> Result<(ExecuteInputResult, Option), Error> { let exit_kind = self.execute_input(state, executor, manager, &input)?; @@ -334,24 +334,22 @@ where } } -impl Evaluator::Input, S> - for ReplayingFuzzer +impl Evaluator for ReplayingFuzzer where - CS: Scheduler<::Input, S>, - E: HasObservers + Executor::Input, S, Self>, - E::Observers: MatchName + ObserversTuple<::Input, S> + Serialize, - EM: EventFirer<::Input, S> + CanSerializeObserver, - F: Feedback::Input, E::Observers, S>, - OF: Feedback::Input, E::Observers, S>, - S: HasCorpus - + HasSolutions + CS: Scheduler, + E: HasObservers + Executor, + E::Observers: MatchName + ObserversTuple + Serialize, + EM: EventFirer + CanSerializeObserver, + F: Feedback, + OF: Feedback, + S: HasCorpus + + HasSolutions + MaybeHasClientPerfMonitor - + HasCurrentTestcase + + HasCurrentTestcase + HasLastFoundTime + HasExecutions, - ::Input: Input, - S::Solutions: Corpus::Input>, - IF: InputFilter<::Input>, + I: Input, + IF: InputFilter, O: Hash, { fn evaluate_filtered( @@ -359,7 +357,7 @@ where state: &mut S, executor: &mut E, manager: &mut EM, - input: ::Input, + input: I, ) -> Result<(ExecuteInputResult, Option), Error> { if self.input_filter.should_execute(&input) { self.evaluate_input(state, executor, manager, input) @@ -375,17 +373,13 @@ where state: &mut S, executor: &mut E, manager: &mut EM, - input: ::Input, + input: I, send_events: bool, ) -> Result<(ExecuteInputResult, Option), Error> { self.evaluate_input_with_observers(state, executor, manager, input, send_events) } - fn add_disabled_input( - &mut self, - state: &mut S, - input: ::Input, - ) -> Result { + fn add_disabled_input(&mut self, state: &mut S, input: I) -> Result { let mut testcase = Testcase::from(input.clone()); testcase.set_disabled(true); // Add the disabled input to the main corpus @@ -399,7 +393,7 @@ where state: &mut S, executor: &mut E, manager: &mut EM, - input: ::Input, + input: I, ) -> Result { *state.last_found_time_mut() = current_time(); @@ -492,15 +486,16 @@ where } } -impl Fuzzer for ReplayingFuzzer +impl Fuzzer + for ReplayingFuzzer where - CS: Scheduler<::Input, S>, + CS: Scheduler, EM: ProgressReporter + EventProcessor, S: HasExecutions + HasMetadata - + HasCorpus + + HasCorpus + HasLastReportTime - + HasTestcase + + HasTestcase + HasCurrentCorpusId + HasCurrentStageId + Stoppable @@ -692,15 +687,14 @@ impl ReplayingFuzzer { } } -impl ExecutesInput::Input, S> - for ReplayingFuzzer +impl ExecutesInput for ReplayingFuzzer where - CS: Scheduler<::Input, S>, - E: Executor::Input, S, Self> + HasObservers, - E::Observers: ObserversTuple<::Input, S> + MatchNameRef, - S: HasExecutions + HasCorpus + MaybeHasClientPerfMonitor, + CS: Scheduler, + E: Executor + HasObservers, + E::Observers: ObserversTuple, + S: HasExecutions + HasCorpus + MaybeHasClientPerfMonitor, O: Hash, - EM: EventFirer<::Input, S>, + EM: EventFirer, { /// Runs the input and triggers observers and feedback fn execute_input( @@ -708,7 +702,7 @@ where state: &mut S, executor: &mut E, event_mgr: &mut EM, - input: &::Input, + input: &I, ) -> Result { let mut results = HashMap::new(); let (exit_kind, total_replayed) = loop { From 1c6011dc7b203a3a149119828ae718e7726e2d82 Mon Sep 17 00:00:00 2001 From: Valentin Huber Date: Thu, 23 Jan 2025 15:14:24 +0000 Subject: [PATCH 04/14] Fix docs --- libafl/src/fuzzer/replaying.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libafl/src/fuzzer/replaying.rs b/libafl/src/fuzzer/replaying.rs index 6a812ff54e..34952d0b4c 100644 --- a/libafl/src/fuzzer/replaying.rs +++ b/libafl/src/fuzzer/replaying.rs @@ -612,7 +612,7 @@ where } impl ReplayingFuzzer { - /// Create a new [`StdFuzzer`] with standard behavior and the provided duplicate input execution filter. + /// Create a new [`ReplayingFuzzer`] with standard behavior and the provided duplicate input execution filter. pub fn with_input_filter( min_count_diff: usize, max_trys: usize, From b4fa95474923b7f98f745dde4ad7387d3fd07e9e Mon Sep 17 00:00:00 2001 From: Valentin Huber Date: Thu, 23 Jan 2025 15:30:29 +0000 Subject: [PATCH 05/14] Reorder things, fix docs v2 --- libafl/src/fuzzer/mod.rs | 2 - libafl/src/fuzzer/replaying.rs | 142 +++++++++++++++++---------------- 2 files changed, 73 insertions(+), 71 deletions(-) diff --git a/libafl/src/fuzzer/mod.rs b/libafl/src/fuzzer/mod.rs index 8bd7d70299..380b202922 100644 --- a/libafl/src/fuzzer/mod.rs +++ b/libafl/src/fuzzer/mod.rs @@ -479,7 +479,6 @@ where Event::Objective { #[cfg(feature = "share_objectives")] input, - objective_size: state.solutions().count(), time: current_time(), }, @@ -674,7 +673,6 @@ where Event::Objective { #[cfg(feature = "share_objectives")] input, - objective_size: state.solutions().count(), time: current_time(), }, diff --git a/libafl/src/fuzzer/replaying.rs b/libafl/src/fuzzer/replaying.rs index 34952d0b4c..51088ec680 100644 --- a/libafl/src/fuzzer/replaying.rs +++ b/libafl/src/fuzzer/replaying.rs @@ -148,24 +148,57 @@ where Ok(res) } - fn evaluate_execution( + /// Evaluate if a set of observation channels has an interesting state + fn process_execution( &mut self, state: &mut S, manager: &mut EM, - input: I, + input: &I, + exec_res: &ExecuteInputResult, observers: &OT, - exit_kind: &ExitKind, - send_events: bool, - ) -> Result<(ExecuteInputResult, Option), Error> { - let exec_res = self.check_results(state, manager, &input, observers, exit_kind)?; - let corpus_id = self.process_execution(state, manager, &input, &exec_res, observers)?; - if send_events { - self.serialize_and_dispatch(state, manager, input, &exec_res, observers, exit_kind)?; - } - if exec_res != ExecuteInputResult::None { - *state.last_found_time_mut() = current_time(); + ) -> Result, Error> { + match exec_res { + ExecuteInputResult::None => { + self.feedback_mut().discard_metadata(state, input)?; + self.objective_mut().discard_metadata(state, input)?; + Ok(None) + } + ExecuteInputResult::Corpus => { + // Not a solution + self.objective_mut().discard_metadata(state, input)?; + + // Add the input to the main corpus + let mut testcase = Testcase::from(input.clone()); + #[cfg(feature = "track_hit_feedbacks")] + self.feedback_mut() + .append_hit_feedbacks(testcase.hit_feedbacks_mut())?; + self.feedback_mut() + .append_metadata(state, manager, observers, &mut testcase)?; + let id = state.corpus_mut().add(testcase)?; + self.scheduler_mut().on_add(state, id)?; + + Ok(Some(id)) + } + ExecuteInputResult::Solution => { + // Not interesting + self.feedback_mut().discard_metadata(state, input)?; + + // The input is a solution, add it to the respective corpus + let mut testcase = Testcase::from(input.clone()); + testcase.set_parent_id_optional(*state.corpus().current()); + if let Ok(mut tc) = state.current_testcase_mut() { + tc.found_objective(); + } + #[cfg(feature = "track_hit_feedbacks")] + self.objective_mut() + .append_hit_feedbacks(testcase.hit_objectives_mut())?; + self.objective_mut() + .append_metadata(state, manager, observers, &mut testcase)?; + state.solutions_mut().add(testcase)?; + + Ok(None) + } } - Ok((exec_res, corpus_id)) } fn serialize_and_dispatch( @@ -181,7 +214,7 @@ where let observers_buf = match exec_res { ExecuteInputResult::Corpus => { if manager.should_send() { - // TODO, set None for fast targets + // TODO set None for fast targets if manager.configuration() == EventConfig::AlwaysUnique { None } else { @@ -232,6 +265,8 @@ where manager.fire( state, Event::Objective { + #[cfg(feature = "share_objectives")] + input, objective_size: state.solutions().count(), time: current_time(), }, @@ -243,57 +278,24 @@ where Ok(()) } - /// Evaluate if a set of observation channels has an interesting state - fn process_execution( + fn evaluate_execution( &mut self, state: &mut S, manager: &mut EM, - input: &I, - exec_res: &ExecuteInputResult, + input: I, observers: &OT, - ) -> Result, Error> { - match exec_res { - ExecuteInputResult::None => { - self.feedback_mut().discard_metadata(state, input)?; - self.objective_mut().discard_metadata(state, input)?; - Ok(None) - } - ExecuteInputResult::Corpus => { - // Not a solution - self.objective_mut().discard_metadata(state, input)?; - - // Add the input to the main corpus - let mut testcase = Testcase::from(input.clone()); - #[cfg(feature = "track_hit_feedbacks")] - self.feedback_mut() - .append_hit_feedbacks(testcase.hit_feedbacks_mut())?; - self.feedback_mut() - .append_metadata(state, manager, observers, &mut testcase)?; - let id = state.corpus_mut().add(testcase)?; - self.scheduler_mut().on_add(state, id)?; - - Ok(Some(id)) - } - ExecuteInputResult::Solution => { - // Not interesting - self.feedback_mut().discard_metadata(state, input)?; - - // The input is a solution, add it to the respective corpus - let mut testcase = Testcase::from(input.clone()); - testcase.set_parent_id_optional(*state.corpus().current()); - if let Ok(mut tc) = state.current_testcase_mut() { - tc.found_objective(); - } - #[cfg(feature = "track_hit_feedbacks")] - self.objective_mut() - .append_hit_feedbacks(testcase.hit_objectives_mut())?; - self.objective_mut() - .append_metadata(state, manager, observers, &mut testcase)?; - state.solutions_mut().add(testcase)?; - - Ok(None) - } + exit_kind: &ExitKind, + send_events: bool, + ) -> Result<(ExecuteInputResult, Option), Error> { + let exec_res = self.check_results(state, manager, &input, observers, exit_kind)?; + let corpus_id = self.process_execution(state, manager, &input, &exec_res, observers)?; + if send_events { + self.serialize_and_dispatch(state, manager, input, &exec_res, observers, exit_kind)?; } + if exec_res != ExecuteInputResult::None { + *state.last_found_time_mut() = current_time(); + } + Ok((exec_res, corpus_id)) } } @@ -379,14 +381,6 @@ where self.evaluate_input_with_observers(state, executor, manager, input, send_events) } - fn add_disabled_input(&mut self, state: &mut S, input: I) -> Result { - let mut testcase = Testcase::from(input.clone()); - testcase.set_disabled(true); - // Add the disabled input to the main corpus - let id = state.corpus_mut().add_disabled(testcase)?; - Ok(id) - } - /// Adds an input, even if it's not considered `interesting` by any of the executors fn add_input( &mut self, @@ -428,6 +422,8 @@ where manager.fire( state, Event::Objective { + #[cfg(feature = "share_objectives")] + input, objective_size: state.solutions().count(), time: current_time(), }, @@ -484,6 +480,14 @@ where )?; Ok(id) } + + fn add_disabled_input(&mut self, state: &mut S, input: I) -> Result { + let mut testcase = Testcase::from(input.clone()); + testcase.set_disabled(true); + // Add the disabled input to the main corpus + let id = state.corpus_mut().add_disabled(testcase)?; + Ok(id) + } } impl Fuzzer @@ -602,7 +606,7 @@ where manager.report_progress(state)?; - // If we would assume the fuzzer loop will always exit after this, we could do this here: + // If we assumed the fuzzer loop will always exit after this, we could do this here: // manager.on_restart(state)?; // But as the state may grow to a few megabytes, // for now we won't, and the user has to do it (unless we find a way to do this on `Drop`). From 30847d6a6a30a3477420afa243868ea90d5bba2e Mon Sep 17 00:00:00 2001 From: Valentin Huber Date: Thu, 23 Jan 2025 15:57:27 +0000 Subject: [PATCH 06/14] Improve logging, fix bug --- libafl/src/fuzzer/replaying.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libafl/src/fuzzer/replaying.rs b/libafl/src/fuzzer/replaying.rs index 51088ec680..9558a87670 100644 --- a/libafl/src/fuzzer/replaying.rs +++ b/libafl/src/fuzzer/replaying.rs @@ -749,9 +749,9 @@ where if consistent_enough && latest_execution_is_dominant { break (exit_kind, total_replayed); - } else if total_replayed >= self.max_trys * self.min_count_diff { + } else if total_replayed >= self.max_trys { log::warn!( - "Replaying {} times did not lead to dominant result, using the latest observer value and most common exit_kind", + "Replaying {} times did not lead to dominant result, using the latest observer value and most common exit_kind. Details: {results:?}", total_replayed ); break (*max_exit_kind, total_replayed); From 40dfa782a3c6d6bfc46114e69b9a2ffe06fb8eab Mon Sep 17 00:00:00 2001 From: Valentin Huber Date: Thu, 23 Jan 2025 18:56:05 +0000 Subject: [PATCH 07/14] Add factor to replaying fuzzer --- libafl/src/fuzzer/replaying.rs | 41 ++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/libafl/src/fuzzer/replaying.rs b/libafl/src/fuzzer/replaying.rs index 9558a87670..51c0f1c26f 100644 --- a/libafl/src/fuzzer/replaying.rs +++ b/libafl/src/fuzzer/replaying.rs @@ -42,10 +42,14 @@ use crate::{ /// A fuzzer instance for unstable targets that increases stability by executing the same input multiple times. /// -/// The input will be executed as often as necessary until the most frequent result appears at least `min_count_diff` times more often than any other result and at most `max_trys` times. +/// The input will be executed as often as necessary until the most frequent result appears +/// - at least `min_count_diff` times more often than any other result +/// - at least `min_factor_diff` times more often than any other result +/// - at most `max_trys` times #[derive(Debug)] pub struct ReplayingFuzzer { min_count_diff: usize, + min_factor_diff: f64, max_trys: usize, handle: Handle, scheduler: CS, @@ -617,8 +621,10 @@ where impl ReplayingFuzzer { /// Create a new [`ReplayingFuzzer`] with standard behavior and the provided duplicate input execution filter. + #[expect(clippy::too_many_arguments)] pub fn with_input_filter( min_count_diff: usize, + min_factor_diff: f64, max_trys: usize, handle: Handle, scheduler: CS, @@ -628,6 +634,7 @@ impl ReplayingFuzzer { ) -> Self { Self { min_count_diff, + min_factor_diff, max_trys, handle, scheduler, @@ -642,6 +649,7 @@ impl ReplayingFuzzer { /// Create a new [`ReplayingFuzzer`] with standard behavior and no duplicate input execution filtering. pub fn new( min_count_diff: usize, + min_factor_diff: f64, max_trys: usize, handle: Handle, scheduler: CS, @@ -650,6 +658,7 @@ impl ReplayingFuzzer { ) -> Self { Self::with_input_filter( min_count_diff, + min_factor_diff, max_trys, handle, scheduler, @@ -670,6 +679,7 @@ impl ReplayingFuzzer { #[expect(clippy::too_many_arguments)] pub fn with_bloom_input_filter( min_count_diff: usize, + min_factor_diff: f64, max_trys: usize, handle: Handle, scheduler: CS, @@ -681,6 +691,7 @@ impl ReplayingFuzzer { let input_filter = BloomInputFilter::new(items_count, fp_p); Self::with_input_filter( min_count_diff, + min_factor_diff, max_trys, handle, scheduler, @@ -709,6 +720,7 @@ where input: &I, ) -> Result { let mut results = HashMap::new(); + let mut inconsistent = 0; let (exit_kind, total_replayed) = loop { start_timer!(state); executor.observers_mut().pre_exec_all(state, input)?; @@ -736,14 +748,16 @@ where let ((max_hash, max_exit_kind), max_count) = results.iter().max_by(|(_, a), (_, b)| a.cmp(b)).unwrap(); - if *max_count < self.min_count_diff { - continue; - } - let consistent_enough = results .values() .filter(|e| **e != *max_count) - .all(|&count| count <= max_count - self.min_count_diff); + .all(|&count| { + let min_value_count = count + self.min_count_diff; + let min_value_factor = + f64::from(u32::try_from(*max_count).unwrap()) * self.min_factor_diff; + min_value_count <= *max_count + && min_value_factor <= f64::from(u32::try_from(*max_count).unwrap()) + }); let latest_execution_is_dominant = hash == *max_hash && exit_kind == *max_exit_kind; @@ -754,6 +768,7 @@ where "Replaying {} times did not lead to dominant result, using the latest observer value and most common exit_kind. Details: {results:?}", total_replayed ); + inconsistent = 1; break (*max_exit_kind, total_replayed); } }; @@ -761,7 +776,7 @@ where event_mgr.fire( state, Event::UpdateUserStats { - name: Cow::Borrowed("consistency_replay_count"), + name: Cow::Borrowed("consistency-caused-replay-per-input"), value: UserStats::new( UserStatsValue::Float(u32::try_from(total_replayed).unwrap().into()), AggregatorOps::Avg, @@ -769,6 +784,17 @@ where phantom: PhantomData, }, )?; + event_mgr.fire( + state, + Event::UpdateUserStats { + name: Cow::Borrowed("uncaptured-inconsistent-rate"), + value: UserStats::new( + UserStatsValue::Float(u32::try_from(inconsistent).unwrap().into()), + AggregatorOps::Avg, + ), + phantom: PhantomData, + }, + )?; Ok(exit_kind) } @@ -807,6 +833,7 @@ mod tests { drop(map_borrow); let mut fuzzer = ReplayingFuzzer::new( 2, + 1.0, 10, observer.handle(), StdScheduler::new(), From 3a346a58eeed7fd796fe7c26d1641acb3fe16a1c Mon Sep 17 00:00:00 2001 From: Valentin Huber Date: Thu, 23 Jan 2025 19:06:13 +0000 Subject: [PATCH 08/14] Fix types --- libafl/src/fuzzer/replaying.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/libafl/src/fuzzer/replaying.rs b/libafl/src/fuzzer/replaying.rs index 51c0f1c26f..3180db13a6 100644 --- a/libafl/src/fuzzer/replaying.rs +++ b/libafl/src/fuzzer/replaying.rs @@ -741,9 +741,9 @@ where let observer = observers.get(&self.handle).expect("observer not found"); let hash = generic_hash_std(observer); - *results.entry((hash, exit_kind)).or_insert(0_usize) += 1; + *results.entry((hash, exit_kind)).or_insert(0_u32) += 1; - let total_replayed = results.values().sum::(); + let total_replayed = results.values().sum::() as usize; let ((max_hash, max_exit_kind), max_count) = results.iter().max_by(|(_, a), (_, b)| a.cmp(b)).unwrap(); @@ -752,11 +752,9 @@ where .values() .filter(|e| **e != *max_count) .all(|&count| { - let min_value_count = count + self.min_count_diff; - let min_value_factor = - f64::from(u32::try_from(*max_count).unwrap()) * self.min_factor_diff; - min_value_count <= *max_count - && min_value_factor <= f64::from(u32::try_from(*max_count).unwrap()) + let min_value_count = count + self.min_count_diff as u32; + let min_value_factor = f64::from(count) * self.min_factor_diff; + min_value_count <= *max_count && min_value_factor <= f64::from(*max_count) }); let latest_execution_is_dominant = hash == *max_hash && exit_kind == *max_exit_kind; From 6db686bd7f3fcdb22b6810a13bce0ec7c057e040 Mon Sep 17 00:00:00 2001 From: Valentin Huber Date: Thu, 23 Jan 2025 19:18:51 +0000 Subject: [PATCH 09/14] Simplify types --- libafl/src/fuzzer/replaying.rs | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/libafl/src/fuzzer/replaying.rs b/libafl/src/fuzzer/replaying.rs index 3180db13a6..c980d420ab 100644 --- a/libafl/src/fuzzer/replaying.rs +++ b/libafl/src/fuzzer/replaying.rs @@ -43,14 +43,15 @@ use crate::{ /// A fuzzer instance for unstable targets that increases stability by executing the same input multiple times. /// /// The input will be executed as often as necessary until the most frequent result appears +/// - at least `min_count_diff` times /// - at least `min_count_diff` times more often than any other result /// - at least `min_factor_diff` times more often than any other result /// - at most `max_trys` times #[derive(Debug)] pub struct ReplayingFuzzer { - min_count_diff: usize, + min_count_diff: u32, min_factor_diff: f64, - max_trys: usize, + max_trys: u32, handle: Handle, scheduler: CS, feedback: F, @@ -623,9 +624,9 @@ impl ReplayingFuzzer { /// Create a new [`ReplayingFuzzer`] with standard behavior and the provided duplicate input execution filter. #[expect(clippy::too_many_arguments)] pub fn with_input_filter( - min_count_diff: usize, + min_count_diff: u32, min_factor_diff: f64, - max_trys: usize, + max_trys: u32, handle: Handle, scheduler: CS, feedback: F, @@ -648,9 +649,9 @@ impl ReplayingFuzzer { impl ReplayingFuzzer { /// Create a new [`ReplayingFuzzer`] with standard behavior and no duplicate input execution filtering. pub fn new( - min_count_diff: usize, + min_count_diff: u32, min_factor_diff: f64, - max_trys: usize, + max_trys: u32, handle: Handle, scheduler: CS, feedback: F, @@ -678,9 +679,9 @@ impl ReplayingFuzzer { /// Use this implementation if hashing each input is very fast compared to executing potential duplicate inputs. #[expect(clippy::too_many_arguments)] pub fn with_bloom_input_filter( - min_count_diff: usize, + min_count_diff: u32, min_factor_diff: f64, - max_trys: usize, + max_trys: u32, handle: Handle, scheduler: CS, feedback: F, @@ -743,16 +744,20 @@ where let hash = generic_hash_std(observer); *results.entry((hash, exit_kind)).or_insert(0_u32) += 1; - let total_replayed = results.values().sum::() as usize; + let total_replayed = results.values().sum::(); let ((max_hash, max_exit_kind), max_count) = results.iter().max_by(|(_, a), (_, b)| a.cmp(b)).unwrap(); + if *max_count < self.min_count_diff { + continue; // require at least min_count_diff replays + } + let consistent_enough = results .values() .filter(|e| **e != *max_count) .all(|&count| { - let min_value_count = count + self.min_count_diff as u32; + let min_value_count = count + self.min_count_diff; let min_value_factor = f64::from(count) * self.min_factor_diff; min_value_count <= *max_count && min_value_factor <= f64::from(*max_count) }); @@ -776,7 +781,7 @@ where Event::UpdateUserStats { name: Cow::Borrowed("consistency-caused-replay-per-input"), value: UserStats::new( - UserStatsValue::Float(u32::try_from(total_replayed).unwrap().into()), + UserStatsValue::Float(total_replayed.into()), AggregatorOps::Avg, ), phantom: PhantomData, From b0b511c4653a16609d5ddeec47b116b485490dbc Mon Sep 17 00:00:00 2001 From: Valentin Huber Date: Thu, 23 Jan 2025 19:23:43 +0000 Subject: [PATCH 10/14] Improve docs --- libafl/src/fuzzer/replaying.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libafl/src/fuzzer/replaying.rs b/libafl/src/fuzzer/replaying.rs index c980d420ab..b6c669c259 100644 --- a/libafl/src/fuzzer/replaying.rs +++ b/libafl/src/fuzzer/replaying.rs @@ -47,6 +47,8 @@ use crate::{ /// - at least `min_count_diff` times more often than any other result /// - at least `min_factor_diff` times more often than any other result /// - at most `max_trys` times +/// +/// If `max_trys` is hit, the last observer values are left in place and the most frequent [`ExitKind`] is returned. #[derive(Debug)] pub struct ReplayingFuzzer { min_count_diff: u32, From 1b97aba1bdc0555f2f119d847ec6c5a42edcb39f Mon Sep 17 00:00:00 2001 From: Valentin Huber Date: Thu, 23 Jan 2025 19:46:06 +0000 Subject: [PATCH 11/14] Add functionality to ignore inconsistent inputs --- libafl/src/executors/mod.rs | 5 +++++ libafl/src/fuzzer/mod.rs | 4 ++++ libafl/src/fuzzer/replaying.rs | 27 ++++++++++++++++++++++++--- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/libafl/src/executors/mod.rs b/libafl/src/executors/mod.rs index 3d14185b01..2d91bf5df6 100644 --- a/libafl/src/executors/mod.rs +++ b/libafl/src/executors/mod.rs @@ -56,6 +56,8 @@ pub enum ExitKind { Oom, /// The run timed out Timeout, + /// The run reports inconsistent results, this means the input is not added to the corpus nor the solutions + Inconsistent, /// Special case for [`DiffExecutor`] when both exitkinds don't match Diff { /// The exitkind of the primary executor @@ -82,6 +84,8 @@ pub enum DiffExitKind { Oom, /// The run timed out Timeout, + /// The run reports inconsistent results + Inconsistent, /// One of the executors itelf repots a differential, we can't go into further details. Diff, // The run resulted in a custom `ExitKind`. @@ -97,6 +101,7 @@ impl From for DiffExitKind { ExitKind::Crash => DiffExitKind::Crash, ExitKind::Oom => DiffExitKind::Oom, ExitKind::Timeout => DiffExitKind::Timeout, + ExitKind::Inconsistent => DiffExitKind::Inconsistent, ExitKind::Diff { .. } => DiffExitKind::Diff, } } diff --git a/libafl/src/fuzzer/mod.rs b/libafl/src/fuzzer/mod.rs index 380b202922..e5c4c8f944 100644 --- a/libafl/src/fuzzer/mod.rs +++ b/libafl/src/fuzzer/mod.rs @@ -330,6 +330,10 @@ where ) -> Result { let mut res = ExecuteInputResult::None; + if *exit_kind == ExitKind::Inconsistent { + return Ok(ExecuteInputResult::None); + } + #[cfg(not(feature = "introspection"))] let is_solution = self .objective_mut() diff --git a/libafl/src/fuzzer/replaying.rs b/libafl/src/fuzzer/replaying.rs index b6c669c259..4827531c11 100644 --- a/libafl/src/fuzzer/replaying.rs +++ b/libafl/src/fuzzer/replaying.rs @@ -49,11 +49,13 @@ use crate::{ /// - at most `max_trys` times /// /// If `max_trys` is hit, the last observer values are left in place and the most frequent [`ExitKind`] is returned. +/// If `ignore_inconsistent_inputs` is set, [`ExitKind::Inconsistent`] is reported and the input is added to neighter the corpus nor the solutions. #[derive(Debug)] pub struct ReplayingFuzzer { min_count_diff: u32, min_factor_diff: f64, max_trys: u32, + ignore_inconsistent_inputs: bool, handle: Handle, scheduler: CS, feedback: F, @@ -125,6 +127,10 @@ where ) -> Result { let mut res = ExecuteInputResult::None; + if *exit_kind == ExitKind::Inconsistent { + return Ok(ExecuteInputResult::None); + } + #[cfg(not(feature = "introspection"))] let is_solution = self .objective_mut() @@ -629,6 +635,7 @@ impl ReplayingFuzzer { min_count_diff: u32, min_factor_diff: f64, max_trys: u32, + ignore_inconsistent_inputs: bool, handle: Handle, scheduler: CS, feedback: F, @@ -639,6 +646,7 @@ impl ReplayingFuzzer { min_count_diff, min_factor_diff, max_trys, + ignore_inconsistent_inputs, handle, scheduler, feedback, @@ -650,10 +658,12 @@ impl ReplayingFuzzer { impl ReplayingFuzzer { /// Create a new [`ReplayingFuzzer`] with standard behavior and no duplicate input execution filtering. + #[expect(clippy::too_many_arguments)] pub fn new( min_count_diff: u32, min_factor_diff: f64, max_trys: u32, + ignore_inconsistent_inputs: bool, handle: Handle, scheduler: CS, feedback: F, @@ -663,6 +673,7 @@ impl ReplayingFuzzer { min_count_diff, min_factor_diff, max_trys, + ignore_inconsistent_inputs, handle, scheduler, feedback, @@ -684,6 +695,7 @@ impl ReplayingFuzzer { min_count_diff: u32, min_factor_diff: f64, max_trys: u32, + ignore_inconsistent_inputs: bool, handle: Handle, scheduler: CS, feedback: F, @@ -696,6 +708,7 @@ impl ReplayingFuzzer { min_count_diff, min_factor_diff, max_trys, + ignore_inconsistent_inputs, handle, scheduler, feedback, @@ -774,7 +787,12 @@ where total_replayed ); inconsistent = 1; - break (*max_exit_kind, total_replayed); + let returned_exit_kind = if self.ignore_inconsistent_inputs { + ExitKind::Inconsistent + } else { + *max_exit_kind + }; + break (returned_exit_kind, total_replayed); } }; @@ -816,7 +834,7 @@ mod tests { }; use crate::{ - corpus::InMemoryCorpus, + corpus::{Corpus as _, InMemoryCorpus}, events::NopEventManager, executors::{ExitKind, InProcessExecutor}, fuzzer::ExecutesInput, @@ -824,7 +842,7 @@ mod tests { observers::StdMapObserver, replaying::ReplayingFuzzer, schedulers::StdScheduler, - state::StdState, + state::{HasCorpus, HasSolutions, StdState}, }; #[test] @@ -840,6 +858,7 @@ mod tests { 2, 1.0, 10, + true, observer.handle(), StdScheduler::new(), tuple_list!(), @@ -878,5 +897,7 @@ mod tests { .unwrap(); assert_eq!(*execution_count.borrow(), 4); + assert!(state.corpus().is_empty()); + assert!(state.solutions().is_empty()); } } From fd913b2cd383811fdbb9b8d0e8bf971bc217b62a Mon Sep 17 00:00:00 2001 From: Valentin Huber Date: Fri, 24 Jan 2025 13:43:30 +0000 Subject: [PATCH 12/14] Move to builder pattern for fuzzer, merge replaying fuzzer --- libafl/src/fuzzer/filter.rs | 48 ++ libafl/src/fuzzer/mod.rs | 152 ++---- libafl/src/fuzzer/replaying.rs | 822 +++++---------------------------- libafl/src/lib.rs | 10 +- 4 files changed, 198 insertions(+), 834 deletions(-) create mode 100644 libafl/src/fuzzer/filter.rs diff --git a/libafl/src/fuzzer/filter.rs b/libafl/src/fuzzer/filter.rs new file mode 100644 index 0000000000..0ada7eea4d --- /dev/null +++ b/libafl/src/fuzzer/filter.rs @@ -0,0 +1,48 @@ +//! Input filter implementations +#[cfg(feature = "std")] +use core::hash::Hash; +#[cfg(feature = "std")] +use fastbloom::BloomFilter; + +/// Filtering input execution in the fuzzer +pub trait InputFilter { + /// Check if the input should be executed + fn should_execute(&mut self, input: &I) -> bool; +} + +/// A pseudo-filter that will execute each input. +#[derive(Debug)] +pub struct NopInputFilter; +impl InputFilter for NopInputFilter { + #[inline] + #[must_use] + fn should_execute(&mut self, _input: &I) -> bool { + true + } +} + +/// A filter that probabilistically prevents duplicate execution of the same input based on a bloom filter. +#[cfg(feature = "std")] +#[derive(Debug)] +pub struct BloomInputFilter { + bloom: BloomFilter, +} + +#[cfg(feature = "std")] +impl BloomInputFilter { + #[must_use] + /// Create a new [`BloomInputFilter`] + pub fn new(items_count: usize, fp_p: f64) -> Self { + let bloom = BloomFilter::with_false_pos(fp_p).expected_items(items_count); + Self { bloom } + } +} + +#[cfg(feature = "std")] +impl InputFilter for BloomInputFilter { + #[inline] + #[must_use] + fn should_execute(&mut self, input: &I) -> bool { + !self.bloom.insert(input) + } +} diff --git a/libafl/src/fuzzer/mod.rs b/libafl/src/fuzzer/mod.rs index e5c4c8f944..d61867e078 100644 --- a/libafl/src/fuzzer/mod.rs +++ b/libafl/src/fuzzer/mod.rs @@ -1,14 +1,14 @@ //! The `Fuzzer` is the main struct for a fuzz campaign. +pub mod filter; pub mod replaying; use alloc::{string::ToString, vec::Vec}; -#[cfg(feature = "std")] -use core::hash::Hash; use core::{fmt::Debug, time::Duration}; +use filter::InputFilter; +use replaying::WrappedExecutesInput; +use typed_builder::TypedBuilder; -#[cfg(feature = "std")] -use fastbloom::BloomFilter; use libafl_bolts::{current_time, tuples::MatchName}; use serde::Serialize; @@ -22,11 +22,9 @@ use crate::{ executors::{Executor, ExitKind, HasObservers}, feedbacks::Feedback, inputs::Input, - mark_feature_time, observers::ObserversTuple, schedulers::Scheduler, stages::{HasCurrentStageId, StagesTuple}, - start_timer, state::{ HasCorpus, HasCurrentTestcase, HasExecutions, HasLastFoundTime, HasLastReportTime, HasSolutions, MaybeHasClientPerfMonitor, Stoppable, @@ -259,15 +257,21 @@ pub enum ExecuteInputResult { } /// Your default fuzzer instance, for everyday use. -#[derive(Debug)] -pub struct StdFuzzer { +/// Create a new [`StdFuzzer`], which, with a certain certainty, executes each input only once. +/// +/// This is achieved by hashing each input and using a bloom filter to differentiate inputs. +/// +/// Use this implementation if hashing each input is very fast compared to executing potential duplicate inputs. +#[derive(Debug, TypedBuilder)] +pub struct StdFuzzer { scheduler: CS, feedback: F, objective: OF, input_filter: IF, + replaying_config: R, } -impl HasScheduler for StdFuzzer +impl HasScheduler for StdFuzzer where CS: Scheduler, { @@ -282,7 +286,7 @@ where } } -impl HasFeedback for StdFuzzer { +impl HasFeedback for StdFuzzer { type Feedback = F; fn feedback(&self) -> &Self::Feedback { @@ -294,7 +298,7 @@ impl HasFeedback for StdFuzzer { } } -impl HasObjective for StdFuzzer { +impl HasObjective for StdFuzzer { type Objective = OF; fn objective(&self) -> &OF { @@ -306,7 +310,8 @@ impl HasObjective for StdFuzzer { } } -impl ExecutionProcessor for StdFuzzer +impl ExecutionProcessor + for StdFuzzer where CS: Scheduler, EM: EventFirer + CanSerializeObserver, @@ -426,7 +431,7 @@ where observers: &OT, exit_kind: &ExitKind, ) -> Result<(), Error> { - // Now send off the event + // Now send OF, f the event let observers_buf = match exec_res { ExecuteInputResult::Corpus => { if manager.should_send() { @@ -456,7 +461,7 @@ where observers_buf: Option>, exit_kind: &ExitKind, ) -> Result<(), Error> { - // Now send off the event + // Now send OF, f the event match exec_res { ExecuteInputResult::Corpus => { if manager.should_send() { @@ -515,7 +520,7 @@ where } } -impl EvaluatorObservers for StdFuzzer +impl EvaluatorObservers for StdFuzzer where CS: Scheduler, E: HasObservers + Executor, @@ -530,6 +535,7 @@ where + HasExecutions + HasLastFoundTime, I: Input, + R: WrappedExecutesInput + Clone, { /// Process one input, adding to the respective corpora if needed and firing the right events #[inline] @@ -550,47 +556,7 @@ where } } -trait InputFilter { - fn should_execute(&mut self, input: &I) -> bool; -} - -/// A pseudo-filter that will execute each input. -#[derive(Debug)] -pub struct NopInputFilter; -impl InputFilter for NopInputFilter { - #[inline] - #[must_use] - fn should_execute(&mut self, _input: &I) -> bool { - true - } -} - -/// A filter that probabilistically prevents duplicate execution of the same input based on a bloom filter. -#[cfg(feature = "std")] -#[derive(Debug)] -pub struct BloomInputFilter { - bloom: BloomFilter, -} - -#[cfg(feature = "std")] -impl BloomInputFilter { - #[must_use] - fn new(items_count: usize, fp_p: f64) -> Self { - let bloom = BloomFilter::with_false_pos(fp_p).expected_items(items_count); - Self { bloom } - } -} - -#[cfg(feature = "std")] -impl InputFilter for BloomInputFilter { - #[inline] - #[must_use] - fn should_execute(&mut self, input: &I) -> bool { - !self.bloom.insert(input) - } -} - -impl Evaluator for StdFuzzer +impl Evaluator for StdFuzzer where CS: Scheduler, E: HasObservers + Executor, @@ -606,6 +572,7 @@ where + HasExecutions, I: Input, IF: InputFilter, + R: WrappedExecutesInput + Clone, { fn evaluate_filtered( &mut self, @@ -743,7 +710,7 @@ where } } -impl Fuzzer for StdFuzzer +impl Fuzzer for StdFuzzer where CS: Scheduler, EM: ProgressReporter + EventProcessor, @@ -866,45 +833,6 @@ where Ok(ret.unwrap()) } } - -impl StdFuzzer { - /// Create a new [`StdFuzzer`] with standard behavior and the provided duplicate input execution filter. - pub fn with_input_filter(scheduler: CS, feedback: F, objective: OF, input_filter: IF) -> Self { - Self { - scheduler, - feedback, - objective, - input_filter, - } - } -} - -impl StdFuzzer { - /// Create a new [`StdFuzzer`] with standard behavior and no duplicate input execution filtering. - pub fn new(scheduler: CS, feedback: F, objective: OF) -> Self { - Self::with_input_filter(scheduler, feedback, objective, NopInputFilter) - } -} - -#[cfg(feature = "std")] -impl StdFuzzer { - /// Create a new [`StdFuzzer`], which, with a certain certainty, executes each input only once. - /// - /// This is achieved by hashing each input and using a bloom filter to differentiate inputs. - /// - /// Use this implementation if hashing each input is very fast compared to executing potential duplicate inputs. - pub fn with_bloom_input_filter( - scheduler: CS, - feedback: F, - objective: OF, - items_count: usize, - fp_p: f64, - ) -> Self { - let input_filter = BloomInputFilter::new(items_count, fp_p); - Self::with_input_filter(scheduler, feedback, objective, input_filter) - } -} - /// Structs with this trait will execute an input pub trait ExecutesInput { /// Runs the input and triggers observers and feedback @@ -917,14 +845,14 @@ pub trait ExecutesInput { ) -> Result; } -impl ExecutesInput for StdFuzzer +impl ExecutesInput for StdFuzzer where CS: Scheduler, E: Executor + HasObservers, E::Observers: ObserversTuple, S: HasExecutions + HasCorpus + MaybeHasClientPerfMonitor, + R: WrappedExecutesInput + Clone, { - /// Runs the input and triggers observers and feedback fn execute_input( &mut self, state: &mut S, @@ -932,21 +860,9 @@ where event_mgr: &mut EM, input: &I, ) -> Result { - start_timer!(state); - executor.observers_mut().pre_exec_all(state, input)?; - mark_feature_time!(state, PerfFeature::PreExecObservers); - - start_timer!(state); - let exit_kind = executor.run_target(self, state, event_mgr, input)?; - mark_feature_time!(state, PerfFeature::TargetExecution); - - start_timer!(state); - executor - .observers_mut() - .post_exec_all(state, input, &exit_kind)?; - mark_feature_time!(state, PerfFeature::PostExecObservers); - - Ok(exit_kind) + self.replaying_config + .clone() + .wrapped_execute_input(self, state, executor, event_mgr, input) } } @@ -1017,6 +933,8 @@ mod tests { corpus::InMemoryCorpus, events::NopEventManager, executors::{ExitKind, InProcessExecutor}, + filter::BloomInputFilter, + fuzzer::replaying::NoReplayingConfig, inputs::BytesInput, schedulers::StdScheduler, state::StdState, @@ -1026,7 +944,13 @@ mod tests { fn filtered_execution() { let execution_count = RefCell::new(0); let scheduler = StdScheduler::new(); - let mut fuzzer = StdFuzzer::with_bloom_input_filter(scheduler, (), (), 100, 1e-4); + let mut fuzzer = StdFuzzer::builder() + .scheduler(scheduler) + .feedback(()) + .objective(()) + .input_filter(BloomInputFilter::new(100, 1e-4)) + .replaying_config(NoReplayingConfig) + .build(); let mut state = StdState::new( StdRand::new(), InMemoryCorpus::new(), diff --git a/libafl/src/fuzzer/replaying.rs b/libafl/src/fuzzer/replaying.rs index 4827531c11..1e431eac99 100644 --- a/libafl/src/fuzzer/replaying.rs +++ b/libafl/src/fuzzer/replaying.rs @@ -1,735 +1,127 @@ -//! Fuzzer instance that increases stability by executing the same input multiple times. +//! Replaying input to ensure evaluation consistency -use alloc::{borrow::Cow, string::ToString, vec::Vec}; -use core::{fmt::Debug, hash::Hash, marker::PhantomData}; - -use hashbrown::HashMap; -use libafl_bolts::{ - current_time, generic_hash_std, - tuples::{Handle, MatchName, MatchNameRef}, -}; -use serde::Serialize; - -#[cfg(feature = "std")] -use crate::fuzzer::BloomInputFilter; -#[cfg(feature = "introspection")] -use crate::monitors::PerfFeature; use crate::{ - corpus::{Corpus, CorpusId, HasCurrentCorpusId, HasTestcase, Testcase}, - events::{ - CanSerializeObserver, Event, EventConfig, EventFirer, EventProcessor, ProgressReporter, - }, + events::{Event, EventFirer}, executors::{Executor, ExitKind, HasObservers}, - feedbacks::Feedback, - fuzzer::{ - Evaluator, EvaluatorObservers, ExecuteInputResult, ExecutesInput, ExecutionProcessor, - Fuzzer, HasFeedback, HasObjective, HasScheduler, InputFilter, NopInputFilter, - STATS_TIMEOUT_DEFAULT, - }, - inputs::Input, + fuzzer::ExecutesInput, mark_feature_time, monitors::{AggregatorOps, UserStats, UserStatsValue}, observers::ObserversTuple, - schedulers::Scheduler, - stages::{HasCurrentStageId, StagesTuple}, start_timer, - state::{ - HasCorpus, HasCurrentTestcase, HasExecutions, HasLastFoundTime, HasLastReportTime, - HasSolutions, MaybeHasClientPerfMonitor, Stoppable, - }, - Error, HasMetadata, +}; +use alloc::borrow::Cow; +use core::{hash::Hash, marker::PhantomData}; +use hashbrown::HashMap; +use libafl_bolts::{ + generic_hash_std, + tuples::{Handle, MatchNameRef}, + Error, }; -/// A fuzzer instance for unstable targets that increases stability by executing the same input multiple times. +/// Helper trait until [`ExecutesInput`] no longer requires the fuzzer to be passed. /// -/// The input will be executed as often as necessary until the most frequent result appears -/// - at least `min_count_diff` times -/// - at least `min_count_diff` times more often than any other result -/// - at least `min_factor_diff` times more often than any other result -/// - at most `max_trys` times -/// -/// If `max_trys` is hit, the last observer values are left in place and the most frequent [`ExitKind`] is returned. -/// If `ignore_inconsistent_inputs` is set, [`ExitKind::Inconsistent`] is reported and the input is added to neighter the corpus nor the solutions. -#[derive(Debug)] -pub struct ReplayingFuzzer { - min_count_diff: u32, - min_factor_diff: f64, - max_trys: u32, - ignore_inconsistent_inputs: bool, - handle: Handle, - scheduler: CS, - feedback: F, - objective: OF, - input_filter: IF, -} - -impl HasScheduler for ReplayingFuzzer -where - CS: Scheduler, -{ - type Scheduler = CS; - - fn scheduler(&self) -> &CS { - &self.scheduler - } - - fn scheduler_mut(&mut self) -> &mut CS { - &mut self.scheduler - } -} - -impl HasFeedback for ReplayingFuzzer { - type Feedback = F; - - fn feedback(&self) -> &Self::Feedback { - &self.feedback - } - - fn feedback_mut(&mut self) -> &mut Self::Feedback { - &mut self.feedback - } -} - -impl HasObjective for ReplayingFuzzer { - type Objective = OF; - - fn objective(&self) -> &OF { - &self.objective - } - - fn objective_mut(&mut self) -> &mut OF { - &mut self.objective - } -} - -impl ExecutionProcessor - for ReplayingFuzzer -where - CS: Scheduler, - EM: EventFirer + CanSerializeObserver, - F: Feedback, - I: Input, - OF: Feedback, - OT: ObserversTuple + Serialize, - S: HasCorpus - + MaybeHasClientPerfMonitor - + HasCurrentTestcase - + HasSolutions - + HasLastFoundTime, -{ - fn check_results( - &mut self, - state: &mut S, - manager: &mut EM, - input: &I, - observers: &OT, - exit_kind: &ExitKind, - ) -> Result { - let mut res = ExecuteInputResult::None; - - if *exit_kind == ExitKind::Inconsistent { - return Ok(ExecuteInputResult::None); - } - - #[cfg(not(feature = "introspection"))] - let is_solution = self - .objective_mut() - .is_interesting(state, manager, input, observers, exit_kind)?; - - #[cfg(feature = "introspection")] - let is_solution = self - .objective_mut() - .is_interesting_introspection(state, manager, input, observers, exit_kind)?; - - if is_solution { - res = ExecuteInputResult::Solution; - } else { - #[cfg(not(feature = "introspection"))] - let corpus_worthy = self - .feedback_mut() - .is_interesting(state, manager, input, observers, exit_kind)?; - - #[cfg(feature = "introspection")] - let corpus_worthy = self - .feedback_mut() - .is_interesting_introspection(state, manager, input, observers, exit_kind)?; - - if corpus_worthy { - res = ExecuteInputResult::Corpus; - } - } - Ok(res) - } - - /// Evaluate if a set of observation channels has an interesting state - fn process_execution( - &mut self, +/// See +pub trait WrappedExecutesInput { + /// Evaluate the input + fn wrapped_execute_input( + &self, + fuzzer: &mut F, state: &mut S, - manager: &mut EM, + executor: &mut E, + event_mgr: &mut EM, input: &I, - exec_res: &ExecuteInputResult, - observers: &OT, - ) -> Result, Error> { - match exec_res { - ExecuteInputResult::None => { - self.feedback_mut().discard_metadata(state, input)?; - self.objective_mut().discard_metadata(state, input)?; - Ok(None) - } - ExecuteInputResult::Corpus => { - // Not a solution - self.objective_mut().discard_metadata(state, input)?; - - // Add the input to the main corpus - let mut testcase = Testcase::from(input.clone()); - #[cfg(feature = "track_hit_feedbacks")] - self.feedback_mut() - .append_hit_feedbacks(testcase.hit_feedbacks_mut())?; - self.feedback_mut() - .append_metadata(state, manager, observers, &mut testcase)?; - let id = state.corpus_mut().add(testcase)?; - self.scheduler_mut().on_add(state, id)?; - - Ok(Some(id)) - } - ExecuteInputResult::Solution => { - // Not interesting - self.feedback_mut().discard_metadata(state, input)?; - - // The input is a solution, add it to the respective corpus - let mut testcase = Testcase::from(input.clone()); - testcase.set_parent_id_optional(*state.corpus().current()); - if let Ok(mut tc) = state.current_testcase_mut() { - tc.found_objective(); - } - #[cfg(feature = "track_hit_feedbacks")] - self.objective_mut() - .append_hit_feedbacks(testcase.hit_objectives_mut())?; - self.objective_mut() - .append_metadata(state, manager, observers, &mut testcase)?; - state.solutions_mut().add(testcase)?; - - Ok(None) - } - } - } - - fn serialize_and_dispatch( - &mut self, - state: &mut S, - manager: &mut EM, - input: I, - exec_res: &ExecuteInputResult, - observers: &OT, - exit_kind: &ExitKind, - ) -> Result<(), Error> { - // Now send off the event - let observers_buf = match exec_res { - ExecuteInputResult::Corpus => { - if manager.should_send() { - // TODO set None for fast targets - if manager.configuration() == EventConfig::AlwaysUnique { - None - } else { - manager.serialize_observers(observers)? - } - } else { - None - } - } - _ => None, - }; - - self.dispatch_event(state, manager, input, exec_res, observers_buf, exit_kind)?; - Ok(()) - } - - fn dispatch_event( - &mut self, - state: &mut S, - manager: &mut EM, - input: I, - exec_res: &ExecuteInputResult, - observers_buf: Option>, - exit_kind: &ExitKind, - ) -> Result<(), Error> { - // Now send off the event - match exec_res { - ExecuteInputResult::Corpus => { - if manager.should_send() { - manager.fire( - state, - Event::NewTestcase { - input, - observers_buf, - exit_kind: *exit_kind, - corpus_size: state.corpus().count(), - client_config: manager.configuration(), - time: current_time(), - forward_id: None, - #[cfg(all(unix, feature = "std", feature = "multi_machine"))] - node_id: None, - }, - )?; - } - } - ExecuteInputResult::Solution => { - if manager.should_send() { - manager.fire( - state, - Event::Objective { - #[cfg(feature = "share_objectives")] - input, - objective_size: state.solutions().count(), - time: current_time(), - }, - )?; - } - } - ExecuteInputResult::None => (), - } - Ok(()) - } - - fn evaluate_execution( - &mut self, - state: &mut S, - manager: &mut EM, - input: I, - observers: &OT, - exit_kind: &ExitKind, - send_events: bool, - ) -> Result<(ExecuteInputResult, Option), Error> { - let exec_res = self.check_results(state, manager, &input, observers, exit_kind)?; - let corpus_id = self.process_execution(state, manager, &input, &exec_res, observers)?; - if send_events { - self.serialize_and_dispatch(state, manager, input, &exec_res, observers, exit_kind)?; - } - if exec_res != ExecuteInputResult::None { - *state.last_found_time_mut() = current_time(); - } - Ok((exec_res, corpus_id)) - } + ) -> Result; } -impl EvaluatorObservers - for ReplayingFuzzer -where - CS: Scheduler, - E: HasObservers + Executor, - E::Observers: MatchName + ObserversTuple + Serialize, - EM: EventFirer + CanSerializeObserver, - F: Feedback, - OF: Feedback, - S: HasCorpus - + HasSolutions - + MaybeHasClientPerfMonitor - + HasCurrentTestcase - + HasExecutions - + HasLastFoundTime, - I: Input, - O: Hash, -{ - /// Process one input, adding to the respective corpora if needed and firing the right events - #[inline] - fn evaluate_input_with_observers( - &mut self, - state: &mut S, - executor: &mut E, - manager: &mut EM, - input: I, - send_events: bool, - ) -> Result<(ExecuteInputResult, Option), Error> { - let exit_kind = self.execute_input(state, executor, manager, &input)?; - let observers = executor.observers(); - - self.scheduler.on_evaluation(state, &input, &*observers)?; - - self.evaluate_execution(state, manager, input, &*observers, &exit_kind, send_events) - } -} +/// Evaluate the input only once +#[derive(Debug, Clone)] +pub struct NoReplayingConfig; -impl Evaluator for ReplayingFuzzer +impl WrappedExecutesInput for NoReplayingConfig where - CS: Scheduler, - E: HasObservers + Executor, - E::Observers: MatchName + ObserversTuple + Serialize, - EM: EventFirer + CanSerializeObserver, - F: Feedback, - OF: Feedback, - S: HasCorpus - + HasSolutions - + MaybeHasClientPerfMonitor - + HasCurrentTestcase - + HasLastFoundTime - + HasExecutions, - I: Input, - IF: InputFilter, - O: Hash, + F: ExecutesInput, + E: Executor + HasObservers, + E::Observers: ObserversTuple, { - fn evaluate_filtered( - &mut self, - state: &mut S, - executor: &mut E, - manager: &mut EM, - input: I, - ) -> Result<(ExecuteInputResult, Option), Error> { - if self.input_filter.should_execute(&input) { - self.evaluate_input(state, executor, manager, input) - } else { - Ok((ExecuteInputResult::None, None)) - } - } - - /// Process one input, adding to the respective corpora if needed and firing the right events #[inline] - fn evaluate_input_events( - &mut self, + fn wrapped_execute_input( + &self, + fuzzer: &mut F, state: &mut S, executor: &mut E, - manager: &mut EM, - input: I, - send_events: bool, - ) -> Result<(ExecuteInputResult, Option), Error> { - self.evaluate_input_with_observers(state, executor, manager, input, send_events) - } - - /// Adds an input, even if it's not considered `interesting` by any of the executors - fn add_input( - &mut self, - state: &mut S, - executor: &mut E, - manager: &mut EM, - input: I, - ) -> Result { - *state.last_found_time_mut() = current_time(); - - let exit_kind = self.execute_input(state, executor, manager, &input)?; - let observers = executor.observers(); - // Always consider this to be "interesting" - let mut testcase = Testcase::from(input.clone()); - - // Maybe a solution - #[cfg(not(feature = "introspection"))] - let is_solution: bool = - self.objective_mut() - .is_interesting(state, manager, &input, &*observers, &exit_kind)?; - - #[cfg(feature = "introspection")] - let is_solution = self.objective_mut().is_interesting_introspection( - state, - manager, - &input, - &*observers, - &exit_kind, - )?; - - if is_solution { - #[cfg(feature = "track_hit_feedbacks")] - self.objective_mut() - .append_hit_feedbacks(testcase.hit_objectives_mut())?; - self.objective_mut() - .append_metadata(state, manager, &*observers, &mut testcase)?; - let id = state.solutions_mut().add(testcase)?; - - manager.fire( - state, - Event::Objective { - #[cfg(feature = "share_objectives")] - input, - objective_size: state.solutions().count(), - time: current_time(), - }, - )?; - return Ok(id); - } - - // Not a solution - self.objective_mut().discard_metadata(state, &input)?; + event_mgr: &mut EM, + input: &I, + ) -> Result { + start_timer!(state); + executor.observers_mut().pre_exec_all(state, input)?; + mark_feature_time!(state, PerfFeature::PreExecObservers); - // several is_interesting implementations collect some data about the run, later used in - // append_metadata; we *must* invoke is_interesting here to collect it - #[cfg(not(feature = "introspection"))] - let _corpus_worthy = - self.feedback_mut() - .is_interesting(state, manager, &input, &*observers, &exit_kind)?; + start_timer!(state); + let exit_kind = executor.run_target(fuzzer, state, event_mgr, input)?; + mark_feature_time!(state, PerfFeature::TargetExecution); - #[cfg(feature = "introspection")] - let _corpus_worthy = self.feedback_mut().is_interesting_introspection( - state, - manager, - &input, - &*observers, - &exit_kind, - )?; - - #[cfg(feature = "track_hit_feedbacks")] - self.feedback_mut() - .append_hit_feedbacks(testcase.hit_feedbacks_mut())?; - // Add the input to the main corpus - self.feedback_mut() - .append_metadata(state, manager, &*observers, &mut testcase)?; - let id = state.corpus_mut().add(testcase)?; - self.scheduler_mut().on_add(state, id)?; + start_timer!(state); + executor + .observers_mut() + .post_exec_all(state, input, &exit_kind)?; - let observers_buf = if manager.configuration() == EventConfig::AlwaysUnique { - None - } else { - manager.serialize_observers(&*observers)? - }; - manager.fire( - state, - Event::NewTestcase { - input, - observers_buf, - exit_kind, - corpus_size: state.corpus().count(), - client_config: manager.configuration(), - time: current_time(), - forward_id: None, - #[cfg(all(unix, feature = "std", feature = "multi_machine"))] - node_id: None, - }, - )?; - Ok(id) - } - - fn add_disabled_input(&mut self, state: &mut S, input: I) -> Result { - let mut testcase = Testcase::from(input.clone()); - testcase.set_disabled(true); - // Add the disabled input to the main corpus - let id = state.corpus_mut().add_disabled(testcase)?; - Ok(id) + mark_feature_time!(state, PerfFeature::PostExecObservers); + Ok(exit_kind) } } -impl Fuzzer - for ReplayingFuzzer -where - CS: Scheduler, - EM: ProgressReporter + EventProcessor, - S: HasExecutions - + HasMetadata - + HasCorpus - + HasLastReportTime - + HasTestcase - + HasCurrentCorpusId - + HasCurrentStageId - + Stoppable - + MaybeHasClientPerfMonitor, - ST: StagesTuple, -{ - fn fuzz_one( - &mut self, - stages: &mut ST, - executor: &mut E, - state: &mut S, - manager: &mut EM, - ) -> Result { - // Init timer for scheduler - #[cfg(feature = "introspection")] - state.introspection_monitor_mut().start_timer(); - - // Get the next index from the scheduler - let id = if let Some(id) = state.current_corpus_id()? { - id // we are resuming - } else { - let id = self.scheduler.next(state)?; - state.set_corpus_id(id)?; // set up for resume - id - }; - - // Mark the elapsed time for the scheduler - #[cfg(feature = "introspection")] - state.introspection_monitor_mut().mark_scheduler_time(); - - // Mark the elapsed time for the scheduler - #[cfg(feature = "introspection")] - state.introspection_monitor_mut().reset_stage_index(); - - // Execute all stages - stages.perform_all(self, executor, state, manager)?; - - // Init timer for manager - #[cfg(feature = "introspection")] - state.introspection_monitor_mut().start_timer(); - - // Execute the manager - manager.process(self, state, executor)?; - - // Mark the elapsed time for the manager - #[cfg(feature = "introspection")] - state.introspection_monitor_mut().mark_manager_time(); - - { - if let Ok(mut testcase) = state.testcase_mut(id) { - let scheduled_count = testcase.scheduled_count(); - // increase scheduled count, this was fuzz_level in afl - testcase.set_scheduled_count(scheduled_count + 1); - } - } - - state.clear_corpus_id()?; - - if state.stop_requested() { - state.discard_stop_request(); - manager.on_shutdown()?; - return Err(Error::shutting_down()); - } - - Ok(id) - } - - fn fuzz_loop( - &mut self, - stages: &mut ST, - executor: &mut E, - state: &mut S, - manager: &mut EM, - ) -> Result<(), Error> { - let monitor_timeout = STATS_TIMEOUT_DEFAULT; - loop { - manager.maybe_report_progress(state, monitor_timeout)?; - - self.fuzz_one(stages, executor, state, manager)?; - } - } - - fn fuzz_loop_for( - &mut self, - stages: &mut ST, - executor: &mut E, - state: &mut S, - manager: &mut EM, - iters: u64, - ) -> Result { - if iters == 0 { - return Err(Error::illegal_argument( - "Cannot fuzz for 0 iterations!".to_string(), - )); - } - - let mut ret = None; - let monitor_timeout = STATS_TIMEOUT_DEFAULT; - - for _ in 0..iters { - manager.maybe_report_progress(state, monitor_timeout)?; - ret = Some(self.fuzz_one(stages, executor, state, manager)?); - } - - manager.report_progress(state)?; - - // If we assumed the fuzzer loop will always exit after this, we could do this here: - // manager.on_restart(state)?; - // But as the state may grow to a few megabytes, - // for now we won't, and the user has to do it (unless we find a way to do this on `Drop`). - - Ok(ret.unwrap()) - } +/// Runs the evaluation multiple times to ensure consistency +/// +/// The input will be evaluated as often as necessary until the most frequent result appears +/// - at least `min_count_diff` times +/// - at least `min_count_diff` times more often than any other result +/// - at least `min_factor_diff` times more often than any other result +/// - at most `max_trys` times +/// +/// If `max_trys` is hit, the last observer values are left in place and the most frequent [`ExitKind`] is returned. +/// If `ignore_inconsistent_inputs` is set, [`ExitKind::Inconsistent`] is reported and the input is added to neighter the corpus nor the solutions. +#[derive(Debug, Clone)] +pub struct ReplayingConfig { + min_count_diff: u32, + min_factor_diff: f64, + max_trys: u32, + ignore_inconsistent_inputs: bool, + observer: Handle, } -impl ReplayingFuzzer { - /// Create a new [`ReplayingFuzzer`] with standard behavior and the provided duplicate input execution filter. - #[expect(clippy::too_many_arguments)] - pub fn with_input_filter( +impl ReplayingConfig { + /// Create a new [`ReplayingConfig`] + #[must_use] + #[inline] + pub fn new( min_count_diff: u32, min_factor_diff: f64, max_trys: u32, ignore_inconsistent_inputs: bool, - handle: Handle, - scheduler: CS, - feedback: F, - objective: OF, - input_filter: IF, + observer: Handle, ) -> Self { Self { min_count_diff, min_factor_diff, max_trys, ignore_inconsistent_inputs, - handle, - scheduler, - feedback, - objective, - input_filter, + observer, } } } -impl ReplayingFuzzer { - /// Create a new [`ReplayingFuzzer`] with standard behavior and no duplicate input execution filtering. - #[expect(clippy::too_many_arguments)] - pub fn new( - min_count_diff: u32, - min_factor_diff: f64, - max_trys: u32, - ignore_inconsistent_inputs: bool, - handle: Handle, - scheduler: CS, - feedback: F, - objective: OF, - ) -> Self { - Self::with_input_filter( - min_count_diff, - min_factor_diff, - max_trys, - ignore_inconsistent_inputs, - handle, - scheduler, - feedback, - objective, - NopInputFilter, - ) - } -} - -#[cfg(feature = "std")] // hashing requires std -impl ReplayingFuzzer { - /// Create a new [`ReplayingFuzzer`], which, with a certain certainty, executes each input only once. - /// - /// This is achieved by hashing each input and using a bloom filter to differentiate inputs. - /// - /// Use this implementation if hashing each input is very fast compared to executing potential duplicate inputs. - #[expect(clippy::too_many_arguments)] - pub fn with_bloom_input_filter( - min_count_diff: u32, - min_factor_diff: f64, - max_trys: u32, - ignore_inconsistent_inputs: bool, - handle: Handle, - scheduler: CS, - feedback: F, - objective: OF, - items_count: usize, - fp_p: f64, - ) -> Self { - let input_filter = BloomInputFilter::new(items_count, fp_p); - Self::with_input_filter( - min_count_diff, - min_factor_diff, - max_trys, - ignore_inconsistent_inputs, - handle, - scheduler, - feedback, - objective, - input_filter, - ) - } -} - -impl ExecutesInput for ReplayingFuzzer +impl WrappedExecutesInput for ReplayingConfig where - CS: Scheduler, - E: Executor + HasObservers, + F: ExecutesInput, + E: Executor + HasObservers, E::Observers: ObserversTuple, - S: HasExecutions + HasCorpus + MaybeHasClientPerfMonitor, O: Hash, EM: EventFirer, { - /// Runs the input and triggers observers and feedback - fn execute_input( - &mut self, + #[inline] + fn wrapped_execute_input( + &self, + fuzzer: &mut F, state: &mut S, executor: &mut E, event_mgr: &mut EM, @@ -738,24 +130,13 @@ where let mut results = HashMap::new(); let mut inconsistent = 0; let (exit_kind, total_replayed) = loop { - start_timer!(state); - executor.observers_mut().pre_exec_all(state, input)?; - mark_feature_time!(state, PerfFeature::PreExecObservers); - - start_timer!(state); - let exit_kind = executor.run_target(self, state, event_mgr, input)?; - mark_feature_time!(state, PerfFeature::TargetExecution); - - start_timer!(state); - executor - .observers_mut() - .post_exec_all(state, input, &exit_kind)?; - + let exit_kind = NoReplayingConfig + .wrapped_execute_input(fuzzer, state, executor, event_mgr, input)?; let observers = executor.observers(); - mark_feature_time!(state, PerfFeature::PostExecObservers); - - let observer = observers.get(&self.handle).expect("observer not found"); + let observer = observers + .get(&self.observer) + .expect("Observer to track consistency of not found"); let hash = generic_hash_std(observer); *results.entry((hash, exit_kind)).or_insert(0_u32) += 1; @@ -783,9 +164,9 @@ where break (exit_kind, total_replayed); } else if total_replayed >= self.max_trys { log::warn!( - "Replaying {} times did not lead to dominant result, using the latest observer value and most common exit_kind. Details: {results:?}", - total_replayed - ); + "Replaying {} times did not lead to dominant result, using the latest observer value and most common exit_kind. Details: {results:?}", + total_replayed + ); inconsistent = 1; let returned_exit_kind = if self.ignore_inconsistent_inputs { ExitKind::Inconsistent @@ -807,6 +188,7 @@ where phantom: PhantomData, }, )?; + event_mgr.fire( state, Event::UpdateUserStats { @@ -818,7 +200,6 @@ where phantom: PhantomData, }, )?; - Ok(exit_kind) } } @@ -837,12 +218,13 @@ mod tests { corpus::{Corpus as _, InMemoryCorpus}, events::NopEventManager, executors::{ExitKind, InProcessExecutor}, - fuzzer::ExecutesInput, + filter::NopInputFilter, + fuzzer::{replaying::ReplayingConfig, ExecutesInput}, inputs::ValueInput, observers::StdMapObserver, - replaying::ReplayingFuzzer, schedulers::StdScheduler, state::{HasCorpus, HasSolutions, StdState}, + StdFuzzer, }; #[test] @@ -854,25 +236,27 @@ mod tests { StdMapObserver::from_mut_ptr("observer", map_borrow.as_mut_ptr(), map_borrow.len()) }; drop(map_borrow); - let mut fuzzer = ReplayingFuzzer::new( - 2, - 1.0, - 10, - true, - observer.handle(), - StdScheduler::new(), - tuple_list!(), - tuple_list!(), - ); + + let mut feedback = (); + let mut objective = (); let mut state = StdState::new( StdRand::new(), InMemoryCorpus::new(), InMemoryCorpus::new(), - &mut tuple_list!(), - &mut tuple_list!(), + &mut feedback, + &mut objective, ) .unwrap(); + + let mut fuzzer = StdFuzzer::builder() + .input_filter(NopInputFilter) + .replaying_config(ReplayingConfig::new(2, 1.0, 10, true, observer.handle())) + .scheduler(StdScheduler::new()) + .feedback(feedback) + .objective(objective) + .build(); + let mut event_mgr = NopEventManager::new(); let execution_count = Rc::new(RefCell::new(0)); let mut harness = |_i: &ValueInput| { diff --git a/libafl/src/lib.rs b/libafl/src/lib.rs index 346feb244b..3527c99629 100644 --- a/libafl/src/lib.rs +++ b/libafl/src/lib.rs @@ -118,10 +118,12 @@ mod tests { events::NopEventManager, executors::{ExitKind, InProcessExecutor}, feedbacks::ConstFeedback, + filter::NopInputFilter, fuzzer::Fuzzer, inputs::BytesInput, monitors::SimpleMonitor, mutators::{mutations::BitFlipMutator, StdScheduledMutator}, + replaying::NoReplayingConfig, schedulers::RandScheduler, stages::StdMutationalStage, state::{HasCorpus, StdState}, @@ -164,7 +166,13 @@ mod tests { let objective = ConstFeedback::new(false); let scheduler = RandScheduler::new(); - let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); + let mut fuzzer = StdFuzzer::builder() + .input_filter(NopInputFilter) + .replaying_config(NoReplayingConfig) + .scheduler(scheduler) + .feedback(feedback) + .objective(objective) + .build(); let mut harness = |_buf: &BytesInput| ExitKind::Ok; let mut executor = InProcessExecutor::new( From 1fb7bb64ce4a6241a775951b6cc8a9f2be36df73 Mon Sep 17 00:00:00 2001 From: Valentin Huber Date: Fri, 24 Jan 2025 21:26:51 +0100 Subject: [PATCH 13/14] Improve reporting --- libafl/src/fuzzer/replaying.rs | 59 ++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/libafl/src/fuzzer/replaying.rs b/libafl/src/fuzzer/replaying.rs index 1e431eac99..47b7a284b0 100644 --- a/libafl/src/fuzzer/replaying.rs +++ b/libafl/src/fuzzer/replaying.rs @@ -9,7 +9,7 @@ use crate::{ observers::ObserversTuple, start_timer, }; -use alloc::borrow::Cow; +use alloc::{borrow::Cow, string::String}; use core::{hash::Hash, marker::PhantomData}; use hashbrown::HashMap; use libafl_bolts::{ @@ -84,7 +84,7 @@ where pub struct ReplayingConfig { min_count_diff: u32, min_factor_diff: f64, - max_trys: u32, + max_trys: u64, ignore_inconsistent_inputs: bool, observer: Handle, } @@ -96,7 +96,7 @@ impl ReplayingConfig { pub fn new( min_count_diff: u32, min_factor_diff: f64, - max_trys: u32, + max_trys: u64, ignore_inconsistent_inputs: bool, observer: Handle, ) -> Self { @@ -129,51 +129,47 @@ where ) -> Result { let mut results = HashMap::new(); let mut inconsistent = 0; - let (exit_kind, total_replayed) = loop { + let (exit_kind, total_replayed, max_count) = loop { let exit_kind = NoReplayingConfig .wrapped_execute_input(fuzzer, state, executor, event_mgr, input)?; let observers = executor.observers(); - let observer = observers - .get(&self.observer) - .expect("Observer to track consistency of not found"); + let observer = observers.get(&self.observer).expect("observer not found"); let hash = generic_hash_std(observer); - *results.entry((hash, exit_kind)).or_insert(0_u32) += 1; + *results.entry((hash, exit_kind)).or_insert(0) += 1; let total_replayed = results.values().sum::(); - let ((max_hash, max_exit_kind), max_count) = + let ((max_hash, max_exit_kind), &max_count) = results.iter().max_by(|(_, a), (_, b)| a.cmp(b)).unwrap(); - if *max_count < self.min_count_diff { + if max_count < self.min_count_diff { continue; // require at least min_count_diff replays } - let consistent_enough = results - .values() - .filter(|e| **e != *max_count) - .all(|&count| { - let min_value_count = count + self.min_count_diff; - let min_value_factor = f64::from(count) * self.min_factor_diff; - min_value_count <= *max_count && min_value_factor <= f64::from(*max_count) - }); + let consistent_enough = results.values().filter(|e| **e != max_count).all(|&count| { + let min_value_count = count + self.min_count_diff; + let min_value_factor = f64::from(count) * self.min_factor_diff; + min_value_count <= max_count && min_value_factor <= f64::from(max_count) + }); let latest_execution_is_dominant = hash == *max_hash && exit_kind == *max_exit_kind; if consistent_enough && latest_execution_is_dominant { - break (exit_kind, total_replayed); - } else if total_replayed >= self.max_trys { + break (exit_kind, total_replayed, max_count); + } else if u64::from(total_replayed) >= self.max_trys { log::warn!( - "Replaying {} times did not lead to dominant result, using the latest observer value and most common exit_kind. Details: {results:?}", - total_replayed - ); + "Input still not consistent after {} tries, using the latest observer value and most common exit_kind. Results: {}", + total_replayed, + results.iter().map(|((hash, exit_kind), count)| format!("{count} times with hash {hash} and ExitKind::{exit_kind:?}")).fold(String::new(), |acc,e| format!("{acc}, {e}")) + ); inconsistent = 1; let returned_exit_kind = if self.ignore_inconsistent_inputs { ExitKind::Inconsistent } else { *max_exit_kind }; - break (returned_exit_kind, total_replayed); + break (returned_exit_kind, total_replayed, max_count); } }; @@ -182,7 +178,19 @@ where Event::UpdateUserStats { name: Cow::Borrowed("consistency-caused-replay-per-input"), value: UserStats::new( - UserStatsValue::Float(total_replayed.into()), + UserStatsValue::Ratio(total_replayed.into(), 1), + AggregatorOps::Avg, + ), + phantom: PhantomData, + }, + )?; + + event_mgr.fire( + state, + Event::UpdateUserStats { + name: Cow::Borrowed("consistency-caused-replay-per-input-success"), + value: UserStats::new( + UserStatsValue::Ratio(max_count.into(), 1), AggregatorOps::Avg, ), phantom: PhantomData, @@ -200,6 +208,7 @@ where phantom: PhantomData, }, )?; + Ok(exit_kind) } } From d2208ee6a8e4b41d328196fbfb6bd951ef9bca1c Mon Sep 17 00:00:00 2001 From: Valentin Huber Date: Fri, 24 Jan 2025 21:35:49 +0100 Subject: [PATCH 14/14] Fix reporting v2 --- libafl/src/fuzzer/replaying.rs | 43 +++++++++++++++++----------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/libafl/src/fuzzer/replaying.rs b/libafl/src/fuzzer/replaying.rs index 47b7a284b0..a96c83d3a9 100644 --- a/libafl/src/fuzzer/replaying.rs +++ b/libafl/src/fuzzer/replaying.rs @@ -128,8 +128,8 @@ where input: &I, ) -> Result { let mut results = HashMap::new(); - let mut inconsistent = 0; - let (exit_kind, total_replayed, max_count) = loop { + let mut probably_consistent = true; + let (exit_kind, total_replayed) = loop { let exit_kind = NoReplayingConfig .wrapped_execute_input(fuzzer, state, executor, event_mgr, input)?; let observers = executor.observers(); @@ -156,53 +156,54 @@ where let latest_execution_is_dominant = hash == *max_hash && exit_kind == *max_exit_kind; if consistent_enough && latest_execution_is_dominant { - break (exit_kind, total_replayed, max_count); + break (exit_kind, total_replayed); } else if u64::from(total_replayed) >= self.max_trys { log::warn!( "Input still not consistent after {} tries, using the latest observer value and most common exit_kind. Results: {}", total_replayed, results.iter().map(|((hash, exit_kind), count)| format!("{count} times with hash {hash} and ExitKind::{exit_kind:?}")).fold(String::new(), |acc,e| format!("{acc}, {e}")) ); - inconsistent = 1; + probably_consistent = false; let returned_exit_kind = if self.ignore_inconsistent_inputs { ExitKind::Inconsistent } else { *max_exit_kind }; - break (returned_exit_kind, total_replayed, max_count); + break (returned_exit_kind, total_replayed); } }; + let execution_count = UserStats::new( + UserStatsValue::Ratio(total_replayed.into(), 1), + AggregatorOps::Avg, + ); + event_mgr.fire( state, Event::UpdateUserStats { name: Cow::Borrowed("consistency-caused-replay-per-input"), - value: UserStats::new( - UserStatsValue::Ratio(total_replayed.into(), 1), - AggregatorOps::Avg, - ), + value: execution_count.clone(), phantom: PhantomData, }, )?; - event_mgr.fire( - state, - Event::UpdateUserStats { - name: Cow::Borrowed("consistency-caused-replay-per-input-success"), - value: UserStats::new( - UserStatsValue::Ratio(max_count.into(), 1), - AggregatorOps::Avg, - ), - phantom: PhantomData, - }, - )?; + if probably_consistent { + event_mgr.fire( + state, + Event::UpdateUserStats { + name: Cow::Borrowed("consistency-caused-replay-per-input-success"), + value: execution_count, + phantom: PhantomData, + }, + )?; + } event_mgr.fire( state, Event::UpdateUserStats { name: Cow::Borrowed("uncaptured-inconsistent-rate"), value: UserStats::new( - UserStatsValue::Float(u32::try_from(inconsistent).unwrap().into()), + UserStatsValue::Ratio(u64::from(probably_consistent), 1), AggregatorOps::Avg, ), phantom: PhantomData,