Skip to content

Commit

Permalink
feat(forge): remove testFail* (#9574)
Browse files Browse the repository at this point in the history 
* debt(`forge`): deprecate `testFail*`

* fix tests using testFail*

* fix

* refactor: expect revert failures tests to cli

* rm `testFail` from ExpectRevert.t.sol
Moved to ExpectRevertFailures.t.sol in cli tests

* mv ExpectCall.t.sol failure tests to ExpectCallFailures.t.sol in forge cli tests

* mv ExpectEmit.t.sol `testFail` to ExpectEmitFailures.t.sol as cli test

* mv MemSafety failure tests

* fmt

* mv DSStyleTest failling assertion to cli test

* failure_assertions

* failing setup test

* multiple aftertInvariants

* multiple setups

* emit diff anonymous

* Err out on `testFail

* fix: test_core

* fix

* fix: test_logs

* fix: test_fuzz

* fix: repro_7481

* fix: testShouldFailRevertNotOnImmediateNextCall, mv to failure_assertions

* fix + forge fmt

* clippy

* chore: update and document external tests (#9644)

* rm `should_fail` from runner and TestFunctionKind

* update, document and add additional external tests

* remove newly added morpho for now

* fix

---------

Co-authored-by: Yash Atreya <44857776+yash-atreya@users.noreply.github.com>

* ignore solmate tests

* nit

* rm should_fail from run_unit_test & run_fuzz_test

* fix fmt

* fmt

* forge fmt

* fix

* rm

* fix tests

* fix: repro_7238

* fix tests

* forge fmt

* fix: repro tests

* bump snekmate ext test

* revert snekmate bump

* bump snekmate

* Revert "bump snekmate"

This reverts commit a9e7568.

* ignore ERC4626VaultTest in snekmate

* remove solmate ext_test

---------

Co-authored-by: zerosnacks <95942363+zerosnacks@users.noreply.github.com>
Co-authored-by: zerosnacks <zerosnacks@protonmail.com>
Co-authored-by: grandizzy <38490174+grandizzy@users.noreply.github.com>
  • Loading branch information
@yash-atreya @zerosnacks @grandizzy
4 people authored Jan 31, 2025
1 parent fe92e7e commit abf269e
Show file tree
Hide file tree
Showing 33 changed files with 1,967 additions and 1,100 deletions.
6 changes: 2 additions & 4 deletions crates/evm/evm/src/executors/fuzz/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,6 @@ impl FuzzedExecutor {
fuzz_fixtures: &FuzzFixtures,
deployed_libs: &[Address],
address: Address,
should_fail: bool,
rd: &RevertDecoder,
progress: Option<&ProgressBar>,
) -> FuzzTestResult {
Expand All @@ -109,7 +108,7 @@ impl FuzzedExecutor {
return Err(TestCaseError::fail(TEST_TIMEOUT));
}

let fuzz_res = self.single_fuzz(address, should_fail, calldata)?;
let fuzz_res = self.single_fuzz(address, calldata)?;

// If running with progress then increment current run.
if let Some(progress) = progress {
Expand Down Expand Up @@ -238,7 +237,6 @@ impl FuzzedExecutor {
pub fn single_fuzz(
&self,
address: Address,
should_fail: bool,
calldata: alloy_primitives::Bytes,
) -> Result<FuzzOutcome, TestCaseError> {
let mut call = self
Expand All @@ -256,7 +254,7 @@ impl FuzzedExecutor {
(cheats.breakpoints.clone(), cheats.deprecated.clone())
});

let success = self.executor.is_raw_call_mut_success(address, &mut call, should_fail);
let success = self.executor.is_raw_call_mut_success(address, &mut call, false);
if success {
Ok(FuzzOutcome::Case(CaseOutcome {
case: FuzzCase { calldata, gas: call.gas_used, stipend: call.stipend },
Expand Down
48 changes: 25 additions & 23 deletions crates/forge/src/runner.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -366,6 +366,26 @@ impl<'a> ContractRunner<'a> {
let identified_contracts = has_invariants.then(|| {
load_contracts(setup.traces.iter().map(|(_, t)| &t.arena), &self.mcr.known_contracts)
});

let test_fail_instances = functions
.iter()
.filter_map(|func| {
TestFunctionKind::classify(&func.name, !func.inputs.is_empty())
.is_any_test_fail()
.then_some(func.name.clone())
})
.collect::<Vec<_>>();

if !test_fail_instances.is_empty() {
let instances = format!(
"Found {} instances: {}",
test_fail_instances.len(),
test_fail_instances.join(", ")
);
let fail = TestResult::fail("`testFail*` has been removed. Consider changing to test_Revert[If|When]_Condition and expecting a revert".to_string());
return SuiteResult::new(start.elapsed(), [(instances, fail)].into(), warnings)
}

let test_results = functions
.par_iter()
.map(|&func| {
Expand Down Expand Up @@ -402,23 +422,6 @@ impl<'a> ContractRunner<'a> {
.collect::<BTreeMap<_, _>>();

let duration = start.elapsed();
let test_fail_deprecations = self
.contract
.abi
.functions()
.filter_map(|func| {
TestFunctionKind::classify(&func.name, !func.inputs.is_empty())
.is_any_test_fail()
.then_some(func.name.clone())
})
.collect::<Vec<_>>()
.join(", ");

if !test_fail_deprecations.is_empty() {
warnings.push(format!(
"`testFail*` has been deprecated and will be removed in the next release. Consider changing to test_Revert[If|When]_Condition and expecting a revert. Found deprecated testFail* function(s): {test_fail_deprecations}.",
));
}
SuiteResult::new(duration, test_results, warnings)
}
}
Expand Down Expand Up @@ -490,8 +493,8 @@ impl<'a> FunctionRunner<'a> {
}

match kind {
TestFunctionKind::UnitTest { should_fail } => self.run_unit_test(func, should_fail),
TestFunctionKind::FuzzTest { should_fail } => self.run_fuzz_test(func, should_fail),
TestFunctionKind::UnitTest { .. } => self.run_unit_test(func),
TestFunctionKind::FuzzTest { .. } => self.run_fuzz_test(func),
TestFunctionKind::InvariantTest => {
self.run_invariant_test(func, call_after_invariant, identified_contracts.unwrap())
}
Expand All @@ -507,7 +510,7 @@ impl<'a> FunctionRunner<'a> {
/// (therefore the unit test call will be made on modified state).
/// State modifications of before test txes and unit test function call are discarded after
/// test ends, similar to `eth_call`.
fn run_unit_test(mut self, func: &Function, should_fail: bool) -> TestResult {
fn run_unit_test(mut self, func: &Function) -> TestResult {
// Prepare unit test execution.
if self.prepare_test(func).is_err() {
return self.result;
Expand Down Expand Up @@ -535,7 +538,7 @@ impl<'a> FunctionRunner<'a> {
};

let success =
self.executor.is_raw_call_mut_success(self.address, &mut raw_call_result, should_fail);
self.executor.is_raw_call_mut_success(self.address, &mut raw_call_result, false);
self.result.single_result(success, reason, raw_call_result);
self.result
}
Expand Down Expand Up @@ -734,7 +737,7 @@ impl<'a> FunctionRunner<'a> {
/// (therefore the fuzz test will use the modified state).
/// State modifications of before test txes and fuzz test are discarded after test ends,
/// similar to `eth_call`.
fn run_fuzz_test(mut self, func: &Function, should_fail: bool) -> TestResult {
fn run_fuzz_test(mut self, func: &Function) -> TestResult {
// Prepare fuzz test execution.
if self.prepare_test(func).is_err() {
return self.result;
Expand All @@ -754,7 +757,6 @@ impl<'a> FunctionRunner<'a> {
&self.setup.fuzz_fixtures,
&self.setup.deployed_libs,
self.address,
should_fail,
&self.cr.mcr.revert_decoder,
progress.as_ref(),
);
Expand Down
90 changes: 34 additions & 56 deletions crates/forge/tests/cli/ext_integration.rs
View file
Original file line number Diff line number Diff line change
@@ -1,50 +1,44 @@
use foundry_test_utils::util::ExtTester;

// Actively maintained tests

// <https://github.com/foundry-rs/forge-std>
#[test]
fn forge_std() {
ExtTester::new("foundry-rs", "forge-std", "2b59872eee0b8088ddcade39fe8c041e17bb79c0")
ExtTester::new("foundry-rs", "forge-std", "08d6af5d6c8a9a60e308b689cd19751876d321e0")
// Skip fork tests.
.args(["--nmc", "Fork"])
.run();
}

#[test]
fn solmate() {
let mut tester =
ExtTester::new("transmissions11", "solmate", "c892309933b25c03d32b1b0d674df7ae292ba925");

if cfg!(feature = "isolate-by-default") {
tester = tester.args(["--nmc", "ReentrancyGuardTest"]);
}

tester.run();
}

// <https://github.com/PaulRBerg/prb-math>
#[test]
#[cfg_attr(windows, ignore = "Windows cannot find installed programs")]
fn prb_math() {
ExtTester::new("PaulRBerg", "prb-math", "5b6279a0cf7c1b1b6a5cc96082811f7ef620cf60")
ExtTester::new("PaulRBerg", "prb-math", "b03f814a03558ed5b62f89a57bcc8d720a393f67")
.install_command(&["bun", "install", "--prefer-offline"])
// Try npm if bun fails / is not installed.
.install_command(&["npm", "install", "--prefer-offline"])
.run();
}

// <https://github.com/PaulRBerg/prb-proxy>
#[test]
#[cfg_attr(windows, ignore = "Windows cannot find installed programs")]
fn prb_proxy() {
ExtTester::new("PaulRBerg", "prb-proxy", "fa13cf09fbf544a2d575b45884b8e94a79a02c06")
ExtTester::new("PaulRBerg", "prb-proxy", "e45f5325d4b6003227a6c4bdaefac9453f89de2e")
.install_command(&["bun", "install", "--prefer-offline"])
// Try npm if bun fails / is not installed.
.install_command(&["npm", "install", "--prefer-offline"])
.run();
}

// <https://github.com/sablier-labs/v2-core>
#[test]
#[cfg_attr(windows, ignore = "Windows cannot find installed programs")]
fn sablier_v2() {
fn sablier_v2_core() {
let mut tester =
ExtTester::new("sablier-labs", "v2-core", "84758a40077bf3ccb1c8f7bb8d00278e672fbfef")
ExtTester::new("sablier-labs", "v2-core", "43cf7c9d968e61a5a03e9237a71a27165b125414")
// Skip fork tests.
.args(["--nmc", "Fork"])
// Increase the gas limit: https://github.com/sablier-labs/v2-core/issues/956
Expand All @@ -64,79 +58,63 @@ fn sablier_v2() {
tester.run();
}

// <https://github.com/Vectorized/solady>
#[test]
fn solady() {
ExtTester::new("Vectorized", "solady", "54ea1543a229b88b44ccb6ec5ea570135811a7d9").run();
ExtTester::new("Vectorized", "solady", "de9aee59648862bb98affd578248d1e75c7073ad").run();
}

// <https://github.com/pcaversaccio/snekmate>
#[test]
#[cfg_attr(windows, ignore = "weird git fail")]
#[cfg_attr(windows, ignore = "Windows cannot find installed programs")]
#[cfg(not(feature = "isolate-by-default"))]
fn geb() {
ExtTester::new("reflexer-labs", "geb", "1a59f16a377386c49f520006ed0f7fd9d128cb09")
.env("FOUNDRY_LEGACY_ASSERTIONS", "true")
.args(["--chain-id", "99", "--sender", "0x00a329c0648769A73afAc7F9381E08FB43dBEA72"])
fn snekmate() {
ExtTester::new("pcaversaccio", "snekmate", "df226f4a45e86c8f8c3ff1f9fa3443d260002050")
.args(["--nmc", "ERC4626VaultTest"])
.install_command(&["pnpm", "install", "--prefer-offline"])
// Try npm if pnpm fails / is not installed.
.install_command(&["npm", "install", "--prefer-offline"])
.run();
}

// <https://github.com/mds1/multicall>
#[test]
fn stringutils() {
ExtTester::new("Arachnid", "solidity-stringutils", "4b2fcc43fa0426e19ce88b1f1ec16f5903a2e461")
.run();
fn mds1_multicall3() {
ExtTester::new("mds1", "multicall", "f534fbc9f98386a217eaaf9b29d3d4f6f920d5ec").run();
}

// Legacy tests

// <https://github.com/Arachnid/solidity-stringutils>
#[test]
fn lootloose() {
ExtTester::new("gakonst", "lootloose", "7b639efe97836155a6a6fc626bf1018d4f8b2495")
.install_command(&["make", "install"])
.args(["--evm-version", "paris"])
fn solidity_stringutils() {
ExtTester::new("Arachnid", "solidity-stringutils", "4b2fcc43fa0426e19ce88b1f1ec16f5903a2e461")
.run();
}

// <https://github.com/m1guelpf/lil-web3>
#[test]
fn lil_web3() {
ExtTester::new("m1guelpf", "lil-web3", "7346bd28c2586da3b07102d5290175a276949b15").run();
}

#[test]
#[cfg_attr(windows, ignore = "Windows cannot find installed programs")]
#[cfg(not(feature = "isolate-by-default"))]
fn snekmate() {
ExtTester::new("pcaversaccio", "snekmate", "df226f4a45e86c8f8c3ff1f9fa3443d260002050")
.install_command(&["pnpm", "install", "--prefer-offline"])
// Try npm if pnpm fails / is not installed.
.install_command(&["npm", "install", "--prefer-offline"])
.run();
}

// <https://github.com/makerdao/multicall>
#[test]
fn makerdao_multicall() {
ExtTester::new("makerdao", "multicall", "103a8a28e4e372d582d6539b30031bda4cd48e21").run();
}

#[test]
fn mds1_multicall() {
ExtTester::new("mds1", "multicall", "263ef67f29ab9e450142b42dde617ad69adbf211").run();
}

// Forking tests

#[test]
fn drai() {
ExtTester::new("mds1", "drai", "f31ce4fb15bbb06c94eefea2a3a43384c75b95cf")
.args(["--chain-id", "99", "--sender", "0x00a329c0648769A73afAc7F9381E08FB43dBEA72"])
.env("FOUNDRY_LEGACY_ASSERTIONS", "true")
.fork_block(13633752)
.run();
}
// Legacy forking tests

// <https://github.com/hexonaut/guni-lev>
#[test]
fn gunilev() {
ExtTester::new("hexonaut", "guni-lev", "15ee8b4c2d28e553c5cd5ba9a2a274af97563bc4")
.fork_block(13633752)
.run();
}

// <https://github.com/mds1/convex-shutdown-simulation>
#[test]
fn convex_shutdown_simulation() {
ExtTester::new(
Expand Down
Loading

0 comments on commit abf269e

Please sign in to comment.