invariant shrink #6683: check if test failed instead revert (#7257)

* closes #6683: when deciding min seq to shrink to check if test failure instead revert

* Fix lint

* Changes after review + ensure test shrinked sequence is 3 or less

crates/evm/evm/src/executors/invariant/error.rs

@@ -214,12 +214,18 @@ impl InvariantFuzzError {
                 .call_raw_committing(*sender, *addr, bytes.clone(), U256::ZERO)
                 .expect("bad call to evm");
 
-            // Checks the invariant. If we exit before the last call, all the better.
+            // Checks the invariant. If we revert or fail before the last call, all the better.
             if let Some(func) = &self.func {
-                let error_call_result = executor
+                let mut call_result = executor
                     .call_raw(CALLER, self.addr, func.clone(), U256::ZERO)
                     .expect("bad call to evm");
-                if error_call_result.reverted {
+                let is_success = executor.is_raw_call_success(
+                    self.addr,
+                    call_result.state_changeset.take().unwrap(),
+                    &call_result,
+                    false,
+                );
+                if !is_success {
                     let mut locked = curr_seq.write();
                     if new_sequence[..=seq_idx].len() < locked.len() {
                         // update the curr_sequence if the new sequence is lower than

crates/forge/tests/it/invariant.rs

@@ -2,7 +2,7 @@
 
 use crate::config::*;
 use alloy_primitives::U256;
-use forge::fuzz::CounterExample;
+use forge::{fuzz::CounterExample, result::TestStatus, TestOptions};
 use foundry_test_utils::Filter;
 use std::collections::BTreeMap;
 
@@ -118,6 +118,26 @@ async fn test_invariant() {
                     None,
                 )],
             ),
+            (
+                "fuzz/invariant/common/InvariantShrinkWithAssert.t.sol:InvariantShrinkWithAssert",
+                vec![(
+                    "invariant_with_assert()",
+                    false,
+                    Some("<empty revert data>".into()),
+                    None,
+                    None,
+                )],
+            ),
+            (
+                "fuzz/invariant/common/InvariantShrinkWithAssert.t.sol:InvariantShrinkWithRequire",
+                vec![(
+                    "invariant_with_require()",
+                    false,
+                    Some("revert: wrong counter".into()),
+                    None,
+                    None,
+                )],
+            ),
         ]),
     );
 }
@@ -259,3 +279,50 @@ async fn test_invariant_shrink() {
         }
     };
 }
+
+#[tokio::test(flavor = "multi_thread")]
+#[cfg_attr(windows, ignore = "for some reason there's different rng")]
+async fn test_invariant_assert_shrink() {
+    let mut opts = test_opts();
+    opts.fuzz.seed = Some(U256::from(119u32));
+
+    // ensure assert and require shrinks to same sequence of 3 or less
+    test_shrink(opts.clone(), "InvariantShrinkWithAssert").await;
+    test_shrink(opts.clone(), "InvariantShrinkWithRequire").await;
+}
+
+async fn test_shrink(opts: TestOptions, contract_pattern: &str) {
+    let mut runner = runner().await;
+    runner.test_options = opts.clone();
+    let results = runner
+        .test_collect(
+            &Filter::new(
+                ".*",
+                contract_pattern,
+                ".*fuzz/invariant/common/InvariantShrinkWithAssert.t.sol",
+            ),
+            opts,
+        )
+        .await;
+    let results = results.values().last().expect("`InvariantShrinkWithAssert` should be testable.");
+
+    let result = results
+        .test_results
+        .values()
+        .last()
+        .expect("`InvariantShrinkWithAssert` should be testable.");
+
+    assert_eq!(result.status, TestStatus::Failure);
+
+    let counter = result
+        .counterexample
+        .as_ref()
+        .expect("`InvariantShrinkWithAssert` should have failed with a counterexample.");
+
+    match counter {
+        CounterExample::Single(_) => panic!("CounterExample should be a sequence."),
+        CounterExample::Sequence(sequence) => {
+            assert!(sequence.len() <= 3);
+        }
+    };
+}

testdata/fuzz/invariant/common/InvariantShrinkWithAssert.t.sol

@@ -0,0 +1,115 @@
+// SPDX-License-Identifier: UNLICENSED
+pragma solidity ^0.8.13;
+
+import "ds-test/test.sol";
+import "../../../cheats/Vm.sol";
+
+struct FuzzSelector {
+    address addr;
+    bytes4[] selectors;
+}
+
+contract Counter {
+    uint256 public number;
+
+    function setNumber(uint256 newNumber) public {
+        number = newNumber;
+    }
+
+    function increment() public {
+        number++;
+    }
+
+    function decrement() public {
+        number--;
+    }
+
+    function double() public {
+        number *= 2;
+    }
+
+    function half() public {
+        number /= 2;
+    }
+
+    function triple() public {
+        number *= 3;
+    }
+
+    function third() public {
+        number /= 3;
+    }
+
+    function quadruple() public {
+        number *= 4;
+    }
+
+    function quarter() public {
+        number /= 4;
+    }
+}
+
+contract Handler is DSTest {
+    Counter public counter;
+
+    constructor(Counter _counter) {
+        counter = _counter;
+        counter.setNumber(0);
+    }
+
+    function increment() public {
+        counter.increment();
+    }
+
+    function setNumber(uint256 x) public {
+        counter.setNumber(x);
+    }
+}
+
+contract InvariantShrinkWithAssert is DSTest {
+    Vm constant vm = Vm(HEVM_ADDRESS);
+    Counter public counter;
+    Handler handler;
+
+    function setUp() public {
+        counter = new Counter();
+        handler = new Handler(counter);
+    }
+
+    function targetSelectors() public returns (FuzzSelector[] memory) {
+        FuzzSelector[] memory targets = new FuzzSelector[](1);
+        bytes4[] memory selectors = new bytes4[](2);
+        selectors[0] = handler.increment.selector;
+        selectors[1] = handler.setNumber.selector;
+        targets[0] = FuzzSelector(address(handler), selectors);
+        return targets;
+    }
+
+    function invariant_with_assert() public {
+        assertTrue(counter.number() != 3);
+    }
+}
+
+contract InvariantShrinkWithRequire is DSTest {
+    Vm constant vm = Vm(HEVM_ADDRESS);
+    Counter public counter;
+    Handler handler;
+
+    function setUp() public {
+        counter = new Counter();
+        handler = new Handler(counter);
+    }
+
+    function targetSelectors() public returns (FuzzSelector[] memory) {
+        FuzzSelector[] memory targets = new FuzzSelector[](1);
+        bytes4[] memory selectors = new bytes4[](2);
+        selectors[0] = handler.increment.selector;
+        selectors[1] = handler.setNumber.selector;
+        targets[0] = FuzzSelector(address(handler), selectors);
+        return targets;
+    }
+
+    function invariant_with_require() public {
+        require(counter.number() != 3, "wrong counter");
+    }
+}