From 3b88c9b1b8d90921ef0ca933cd784954c1ef0b2d Mon Sep 17 00:00:00 2001
From: "Celina G. Val" <celinval@amazon.com>
Date: Mon, 23 Jan 2023 13:35:35 -0800
Subject: [PATCH] Add `--no-fail-fast` to compiletest (#2145)

Compiletest behavior has changed to always failing fast after #2045. This PR introduces a --no-fail-fast flag to compiletest which will execute the entire suite regardless of failure.

The regression script uses --no-fail-fast so all failures in a suite are printed as part of the CI.
---
 scripts/kani-regression.sh      | 3 ++-
 tools/compiletest/src/common.rs | 4 ++++
 tools/compiletest/src/main.rs   | 6 +++++-
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/scripts/kani-regression.sh b/scripts/kani-regression.sh
index a57d651ff3be4..bfc61a2f6aacb 100755
--- a/scripts/kani-regression.sh
+++ b/scripts/kani-regression.sh
@@ -74,7 +74,8 @@ for testp in "${TESTS[@]}"; do
   suite=${testl[0]}
   mode=${testl[1]}
   echo "Check compiletest suite=$suite mode=$mode"
-  cargo run -p compiletest --quiet -- --suite $suite --mode $mode --quiet
+  cargo run -p compiletest --quiet -- --suite $suite --mode $mode \
+      --quiet --no-fail-fast
 done
 
 # Check codegen for the standard library
diff --git a/tools/compiletest/src/common.rs b/tools/compiletest/src/common.rs
index 890e06260121b..49e9e1e3f60f1 100644
--- a/tools/compiletest/src/common.rs
+++ b/tools/compiletest/src/common.rs
@@ -134,6 +134,10 @@ pub struct Config {
     /// Timeout duration for each test.
     pub timeout: Option<Duration>,
 
+    /// Whether we will abort execution when a failure occurs.
+    /// When set to false, this will execute the entire test suite regardless of any failure.
+    pub fail_fast: bool,
+
     /// Whether we will run the tests or not.
     pub dry_run: bool,
 }
diff --git a/tools/compiletest/src/main.rs b/tools/compiletest/src/main.rs
index eb268436c23f0..0be16c5c3ba56 100644
--- a/tools/compiletest/src/main.rs
+++ b/tools/compiletest/src/main.rs
@@ -86,6 +86,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
         .optflag("h", "help", "show this message")
         .optopt("", "edition", "default Rust edition", "EDITION")
         .optopt("", "timeout", "the timeout for each test in seconds", "TIMEOUT")
+        .optflag("", "no-fail-fast", "run all tests regardless of failure")
         .optflag("", "dry-run", "don't actually run the tests")
     ;
 
@@ -157,6 +158,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
         color,
         edition: matches.opt_str("edition"),
         force_rerun: matches.opt_present("force-rerun"),
+        fail_fast: !matches.opt_present("no-fail-fast"),
         dry_run: matches.opt_present("dry-run"),
         timeout,
     }
@@ -176,6 +178,8 @@ pub fn log_config(config: &Config) {
     logv(c, format!("verbose: {}", config.verbose));
     logv(c, format!("quiet: {}", config.quiet));
     logv(c, format!("timeout: {:?}", config.timeout));
+    logv(c, format!("fail-fast: {:?}", config.fail_fast));
+    logv(c, format!("dry-run: {:?}", config.dry_run));
     logv(
         c,
         format!(
@@ -285,7 +289,7 @@ pub fn test_opts(config: &Config) -> test::TestOpts {
         list: false,
         options: test::Options::new(),
         time_options: None,
-        fail_fast: true,
+        fail_fast: config.fail_fast,
         force_run_in_process: false,
     }
 }