chore: separate integration test cases into directories based on expe…

…cted result (#2198)

* chore: split expected failure and success cases in integration tests

* chore: backwards compatibility for CI

* chore: separate `compile_success` and `execution_success` tests

* chore: update `.gitignore`

.gitignore

@@ -20,5 +20,5 @@ result
 *.vk
 **/Verifier.toml
 **/target
-!crates/nargo_cli/tests/test_data/*/target
-!crates/nargo_cli/tests/test_data/*/target/witness.tr
+!crates/nargo_cli/tests/execution_success/*/target
+!crates/nargo_cli/tests/execution_success/*/target/witness.tr

crates/nargo_cli/build.rs

@@ -1,5 +1,4 @@
 use rustc_version::{version, Version};
-use std::collections::BTreeMap;
 use std::fs::File;
 use std::io::Write;
 use std::path::{Path, PathBuf};
@@ -32,43 +31,25 @@ fn main() {
     let destination = Path::new(&out_dir).join("execute.rs");
     let mut test_file = File::create(destination).unwrap();
 
-    generate_tests(&mut test_file);
-}
-
-fn load_conf(conf_path: &Path) -> BTreeMap<String, Vec<String>> {
-    let config_str = std::fs::read_to_string(conf_path).unwrap();
-
-    let mut conf_data = match toml::from_str(&config_str) {
-        Ok(t) => t,
-        Err(_) => {
-            BTreeMap::from([("exclude".to_string(), Vec::new()), ("fail".to_string(), Vec::new())])
-        }
-    };
-    if conf_data.get("exclude").is_none() {
-        conf_data.insert("exclude".to_string(), Vec::new());
-    }
-    if conf_data.get("fail").is_none() {
-        conf_data.insert("fail".to_string(), Vec::new());
-    }
-    conf_data
-}
-
-fn generate_tests(test_file: &mut File) {
     // Try to find the directory that Cargo sets when it is running; otherwise fallback to assuming the CWD
     // is the root of the repository and append the crate path
     let manifest_dir = match std::env::var("CARGO_MANIFEST_DIR") {
         Ok(dir) => PathBuf::from(dir),
         Err(_) => std::env::current_dir().unwrap().join("crates").join("nargo_cli"),
     };
-    let test_sub_dir = "test_data";
-    let test_data_dir = manifest_dir.join("tests").join(test_sub_dir);
-    let config_path = test_data_dir.join("config.toml");
+    let test_dir = manifest_dir.join("tests");
+
+    generate_execution_success_tests(&mut test_file, &test_dir);
+    generate_compile_success_tests(&mut test_file, &test_dir);
+    generate_compile_failure_tests(&mut test_file, &test_dir);
+}
 
-    // Load config.toml file from `test_data` directory
-    let config_data: BTreeMap<String, Vec<String>> = load_conf(&config_path);
+fn generate_execution_success_tests(test_file: &mut File, test_data_dir: &Path) {
+    let test_sub_dir = "execution_success";
+    let test_data_dir = test_data_dir.join(test_sub_dir);
 
     let test_case_dirs =
-        fs::read_dir(&test_data_dir).unwrap().flatten().filter(|c| c.path().is_dir());
+        fs::read_dir(test_data_dir).unwrap().flatten().filter(|c| c.path().is_dir());
 
     for test_dir in test_case_dirs {
         let test_name =
@@ -80,28 +61,93 @@ fn generate_tests(test_file: &mut File) {
         };
         let test_dir = &test_dir.path();
 
-        let exclude_macro =
-            if config_data["exclude"].contains(&test_name) { "#[ignore]" } else { "" };
+        write!(
+            test_file,
+            r#"
+#[test]
+fn execution_success_{test_name}() {{
+    let test_program_dir = PathBuf::from("{test_dir}");
 
-        let should_fail = config_data["fail"].contains(&test_name);
+    let mut cmd = Command::cargo_bin("nargo").unwrap();
+    cmd.arg("--program-dir").arg(test_program_dir);
+    cmd.arg("execute");
+
+    cmd.assert().success();
+}}
+            "#,
+            test_dir = test_dir.display(),
+        )
+        .expect("Could not write templated test file.");
+    }
+}
+
+fn generate_compile_success_tests(test_file: &mut File, test_data_dir: &Path) {
+    let test_sub_dir = "compile_success";
+    let test_data_dir = test_data_dir.join(test_sub_dir);
+
+    let test_case_dirs =
+        fs::read_dir(test_data_dir).unwrap().flatten().filter(|c| c.path().is_dir());
+
+    for test_dir in test_case_dirs {
+        let test_name =
+            test_dir.file_name().into_string().expect("Directory can't be converted to string");
+        if test_name.contains('-') {
+            panic!(
+                "Invalid test directory: {test_name}. Cannot include `-`, please convert to `_`"
+            );
+        };
+        let test_dir = &test_dir.path();
+
+        write!(
+            test_file,
+            r#"
+#[test]
+fn compile_success_{test_name}() {{
+    let test_program_dir = PathBuf::from("{test_dir}");
+
+    let mut cmd = Command::cargo_bin("nargo").unwrap();
+    cmd.arg("--program-dir").arg(test_program_dir);
+    cmd.arg("info");
+
+    // `compile_success` tests should be able to compile down to an empty circuit.
+    cmd.assert().stdout(predicate::str::contains("Total ACIR opcodes generated for language PLONKCSat {{ width: 3 }}: 0"));
+}}
+            "#,
+            test_dir = test_dir.display(),
+        )
+        .expect("Could not write templated test file.");
+    }
+}
+
+fn generate_compile_failure_tests(test_file: &mut File, test_data_dir: &Path) {
+    let test_sub_dir = "compile_failure";
+    let test_data_dir = test_data_dir.join(test_sub_dir);
+
+    let test_case_dirs =
+        fs::read_dir(test_data_dir).unwrap().flatten().filter(|c| c.path().is_dir());
+
+    for test_dir in test_case_dirs {
+        let test_name =
+            test_dir.file_name().into_string().expect("Directory can't be converted to string");
+        if test_name.contains('-') {
+            panic!(
+                "Invalid test directory: {test_name}. Cannot include `-`, please convert to `_`"
+            );
+        };
+        let test_dir = &test_dir.path();
 
         write!(
             test_file,
             r#"
-{exclude_macro}
 #[test]
-fn execute_{test_sub_dir}_{test_name}() {{
+fn compile_failure_{test_name}() {{
     let test_program_dir = PathBuf::from("{test_dir}");
 
     let mut cmd = Command::cargo_bin("nargo").unwrap();
     cmd.arg("--program-dir").arg(test_program_dir);
     cmd.arg("execute");
 
-    if {should_fail} {{
-        cmd.assert().failure();
-    }} else {{
-        cmd.assert().success();
-    }}
+    cmd.assert().failure();
 }}
             "#,
             test_dir = test_dir.display(),

crates/nargo_cli/tests/execute.rs

@@ -3,6 +3,7 @@
 mod tests {
     // Some of these imports are consumed by the injected tests
     use assert_cmd::prelude::*;
+    use predicates::prelude::*;
     use tempdir::TempDir;
 
     use std::collections::BTreeMap;

crates/nargo_cli/tests/execution_success/array_dynamic/target/array_dynamic.json

@@ -0,0 +1 @@
+{"backend":"acvm-backend-barretenberg","abi":{"parameters":[{"name":"x","type":{"kind":"array","length":5,"type":{"kind":"integer","sign":"unsigned","width":32}},"visibility":"private"},{"name":"z","type":{"kind":"integer","sign":"unsigned","width":32},"visibility":"private"},{"name":"t","type":{"kind":"integer","sign":"unsigned","width":32},"visibility":"private"},{"name":"index","type":{"kind":"array","length":5,"type":{"kind":"field"}},"visibility":"private"},{"name":"index2","type":{"kind":"array","length":5,"type":{"kind":"field"}},"visibility":"private"},{"name":"offset","type":{"kind":"field"},"visibility":"private"},{"name":"sublen","type":{"kind":"field"},"visibility":"private"}],"param_witnesses":{"index":[8,9,10,11,12],"index2":[13,14,15,16,17],"offset":[18],"sublen":[19],"t":[7],"x":[1,2,3,4,5],"z":[6]},"return_type":null,"return_witnesses":[]},"bytecode":"H4sIAAAAAAAA/+1d6XITRxBuS7LBHAZibOxwbUziAAG8q8OWCAkYCDnIfd8BgWRsCAFCpUIVyQ/yFnm4vArZxrNidr0yRc3XU9vFTtXUHpZ7vj7m69lxa/0PEf1L620k7hVzDKzrSua6mrmuZa5HzfWokTuakV81vzNq3RvLyNhiyRixZGyN+3jct8V9e9x3mJ9VrM/sjPtE3HfFfXfc95gxq7SxjZjjOXMM3VprC05WmAPXVXYkKHswRsWSudccp6x74+aYxBa3McsfiZ84RpZpo69GrPOK+Ux1k8+MDJEzbt1Lfn/CwkI4m4RjBI+1cMKSiQYcJRORHRjQ04k5Za4px3igsVOTqBEuNpu9pXovakRXw3qn226FzVZ3sR21o1a7db3ebjR67WZ7qdPtLIWdqNnoRf1Wp943eux1l9UwssIpnI7hMEKqgP04Viz9JQnIO7lNm+M+697zkNuc9XvDyG2Onk1ueXJKchveBuQ2bRmTr9mRQWZMMLmJTUhXWX/gdAw3s58roU8Ddd4H9CvAfgMC87RajRD699dbNweuOkKfMcdZ615J6BiZXgh9htKEzo4MMmOiCd2eRK7kNkM4cpsl3OSuUX5zlZ+0RB6a4Gap+BhfFsDIrSKI0zU29wNk9Z6EZqdepRxiEfDTfpys0MZ7wDqvZXzHLeErAVKOKDNO1o6ipC3lpAMCcg8SLvil9D6I91GKUIpu06ShiW8WiPMQoYiv39OalA8pwHiYSEVSPgyMzQAWm51rvpJyQDJJ+RXrvEzKjjIDY1C03DkqdlJmvefwPhJNymibJq3ISfkI4ZLyZr5xxfkqzn6h8OIhkorPI1T8xYMiP73QOy+vCWDkVhHE6coh84TiOn87L/Mks8h73TovF3mOMueNQdFyj1KxF3ms91G8j0QXeWibDsPpKvsYUGetifSYAozHgRh9JRUkZhvvG9Z5mVQcZR43BkXLPUHFTiqs9wm8j0STCtKmvgpgJJ5SCOc37wUwJ83xlHXveQpgHtNGX2ULYB7Tswtg8uSUBTDD26AAhh34Hz0tgGFHBpkx0QUwgK24QQHMScKRyCnCrkp8EBICsynaWcyBq46QFhKZ1r2yIg8j0wshLVC6Io8dGWTGRBOSPYlcCWmBcIQUEm5yl/vCchgj0pc4EJj7OX/DAsWV98RRN8eGda9MHBiZXhJHndKJgx0ZZMZEJw57ErkmjjrhEkeDZCY3er+2ifOF2j98Im0ghbElgFEinhaBOmuNp0UFGJdIRzy1cTjrWuOprQBjh3TE02kczobWeDqtAOObpCOezuBwNrXG0xkFGN8CYuS3FvFzTPL2Il6bcT5lDuS45bFqlL+JgNJHyk4aNnsapIMb3gbi1Fr8qyGekH4iwXg6S9hcI8hPYr46qwDjOdIRT8vgeCKF8bSsAON5komnCjiekH94u0DY3Ib2Cet6nvBz8k+wryW47YKA3g9IJsarYJwXgbYE+jp6ANDRfEmk6aueF2jLVD3vO9Z5Wc/rKPOiMSha7iXCTUopvS/hfSRaz4u26TCcrrLfhemcLplD43wPhrPTl8T5PgxnXdTvH8BwdkVxXobhDNu8SZi8Op0bxz7HFfuM7XHZ/DzvAQqEQezBZKsCjB8KYJSIuY9IByd+TDo48RPSwYmfkg5O/IywnJh0bhz7HFfsM7YHj2Vzpt1AGMT4ZlwBxs8FMErE3BekgxO/JB2c+BXp4MSvSQcnfkNYTuQNiYTzOPY5rthnbA8eq0bpimPK6BM6tmF2Ct1atE0Bxm8FMErE3HekgxO/Jx2c+APp4MQfSQcn/kRYTuT/LJRwIsc+xxX7jO3BY9XMZ7INhEGMb7YrwPizAEaJmPuFdHDiFdLBiVdJByd2SQcnXiMsJ/K+W8KJHPtX4s4+Y3vwWDXzmWwDYRDjmx0KMF4XwMgN/cU44L5nar/AdS48VGI/4PNAKo+62u8vJfYDzpPoIdB+f3uynyvOHtB+wJiJkPbzVWgCtGWq0MRehJWFJo4ye8agaLkrVOxCE9Z7Be8j0QXtCmFJwMfrNiaFbAvym/fXbdwwx1XrXvniOIxML6/bYAfaL45jRwaZMdGrOnsSub5u4wbhCGmV/BNSATAnpNHIgauOkNbM8aZ1r3z/D0amF0Jao/T7f9iRQWZM9GPSKuEIaQ2I6ybJTO4K2HcvAXW+hcP15N0KeY+GIPli24O3FGD8VQAjN/Tc3i2ksyuu24RdbPjYAkFitvH+Zp2XWyCOMm8bg6Ll3qFib4Gw3nfwPhL9sitysXC34HHJvrlL+CegR+QnEYVuDbowvAfE9QiIy1ciAuqfSkS/W+dlInKUec8YFC33PhU7EbHe9/E+Ev3SJ9KmkxZGnjwczMkXMXjicKEdF5bwdggX9XIRGxdt7KT1IN8Vd16174k7P1lO0sb2P98kYSDAlAAA","proving_key":null,"verification_key":null}