🧪 Generate individual test functions per .ndct file for test isolation

Author: timfennis

tests/build.rs

@@ -0,0 +1,49 @@
+use std::env;
+use std::fs;
+use std::io::Write;
+use std::path::Path;
+
+fn main() {
+    let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
+    let programs_dir = Path::new(&manifest_dir).join("programs");
+
+    let out_dir = env::var("OUT_DIR").unwrap();
+    let dest_path = Path::new(&out_dir).join("generated_tests.rs");
+    let mut output = fs::File::create(dest_path).unwrap();
+
+    generate_tests(&mut output, &programs_dir, &programs_dir);
+
+    println!("cargo:rerun-if-changed=programs");
+}
+
+fn generate_tests(output: &mut impl Write, base: &Path, dir: &Path) {
+    let mut entries: Vec<_> = fs::read_dir(dir)
+        .unwrap()
+        .filter_map(|e| e.ok())
+        .collect();
+    entries.sort_by_key(|e| e.path());
+
+    for entry in entries {
+        let path = entry.path();
+        if path.is_dir() {
+            generate_tests(output, base, &path);
+        } else if path.extension().map_or(false, |e| e == "ndct") {
+            let relative = path.strip_prefix(base).unwrap();
+            let stem = relative
+                .with_extension("")
+                .to_string_lossy()
+                .replace(['/', '\\', '-', ' '], "_");
+            let test_name = format!("test_{stem}");
+            let path_str = relative.to_string_lossy();
+
+            writeln!(output, "#[test]").unwrap();
+            writeln!(output, "fn {test_name}() {{").unwrap();
+            writeln!(
+                output,
+                "    run_test(std::path::Path::new(env!(\"CARGO_MANIFEST_DIR\")).join(\"programs/{path_str}\")).expect(\"test failed\");",
+            )
+            .unwrap();
+            writeln!(output, "}}").unwrap();
+        }
+    }
+}

tests/src/programs.rs

@@ -1,39 +1,13 @@
 use ndc_lib::interpreter::Interpreter;
 use owo_colors::OwoColorize;
 use std::fs;
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 
-#[test]
-fn example_programs() {
-    run_dir(Path::new("../tests/programs/"));
-}
-
-fn run_dir<P: AsRef<Path>>(dir: P) {
-    match fs::read_dir(dir) {
-        Ok(files) => {
-            // Sort the files in to ensure the tests are executed in the intended order
-            let mut files = files.collect::<Vec<_>>();
-            assert_ne!(files.len(), 0, "no files were found at all");
-            files.sort_unstable_by_key(|r| r.as_ref().map(|e| e.path()).ok());
-
-            for file in files {
-                match file {
-                    Ok(file) if file.path().extension() == Some("ndct".as_ref()) => {
-                        run_test(file.path()).expect("something went wrong while running the test")
-                    }
-                    Ok(file) if file.path().is_dir() => {
-                        run_dir(file.path());
-                    }
-                    _ => panic!("invalid test file: {file:?}"),
-                }
-            }
-        }
-        Err(err) => panic!("Error reading test programs {err}"),
-    }
-}
+include!(concat!(env!("OUT_DIR"), "/generated_tests.rs"));
 
 fn run_test(path: PathBuf) -> Result<(), std::io::Error> {
-    let contents = fs::read_to_string(path.clone())?;
+    let contents = fs::read_to_string(&path)?;
+
     enum Mode {
         None,
         Program,
@@ -48,15 +22,9 @@ fn run_test(path: PathBuf) -> Result<(), std::io::Error> {
 
     for line in contents.split_inclusive('\n') {
         match line {
-            "--PROGRAM--\n" => {
-                mode = Mode::Program;
-            }
-            "--EXPECT--\n" => {
-                mode = Mode::Expect;
-            }
-            "--EXPECT-ERROR--\n" => {
-                mode = Mode::ExpectError;
-            }
+            "--PROGRAM--\n" => mode = Mode::Program,
+            "--EXPECT--\n" => mode = Mode::Expect,
+            "--EXPECT-ERROR--\n" => mode = Mode::ExpectError,
             _ => match mode {
                 Mode::None => panic!("unexpected line in file, not in mode"),
                 Mode::Program => program.push_str(line),
@@ -65,22 +33,20 @@ fn run_test(path: PathBuf) -> Result<(), std::io::Error> {
             },
         }
     }
-    // For now let's trim end both result and expect to ensure that any trailing line breaks don't cause issues
+
     print!("Running {path:?}...");
 
     let mut interpreter = Interpreter::new(Vec::new());
     let interpreter_result = interpreter.run_str(&program, false);
 
     let program_had_error = interpreter_result.is_err();
-
     let actual_error = interpreter_result.unwrap_or_else(|err| format!("{err:?}"));
 
     let environment = interpreter.environment();
     let environment = environment.borrow();
     let output = environment
         .get_output()
         .expect("interpreter must have output in test context");
-
     let output = String::from_utf8(output).expect("test output must be valid UTF-8");
 
     assert!(