test(cts): replace --quiet and --verbose with --print-output-when

Author: ErichDonGubler

xtask/src/cts.rs

@@ -30,7 +30,8 @@
 //! Lines starting with `//` or `#` in the test list are treated as comments and
 //! ignored.
 
-use anyhow::{bail, Context};
+use anyhow::{anyhow, bail, Context};
+use core::fmt;
 use pico_args::Arguments;
 use regex_lite::{Regex, RegexBuilder};
 use std::{ffi::OsString, sync::LazyLock};
@@ -56,6 +57,12 @@ struct TestLine {
     pub fails_if: Vec<String>,
 }
 
+#[derive(Clone, Copy, Debug)]
+enum PrintOutputWhen {
+    TestFails,
+    Always,
+}
+
 pub fn run_cts(
     shell: Shell,
     mut args: Arguments,
@@ -64,8 +71,31 @@ pub fn run_cts(
     let skip_checkout = args.contains("--skip-checkout");
     let llvm_cov = args.contains("--llvm-cov");
     let release = args.contains("--release");
-    let mut quiet = args.contains("--quiet");
-    let verbose = args.contains("--verbose");
+
+    let output_filter = args
+        .opt_value_from_str::<_, String>("--print-output-when")?
+        .map(|f| {
+            let values = [
+                ("test-fails", PrintOutputWhen::TestFails),
+                ("always", PrintOutputWhen::Always),
+            ];
+            let lowered = f.to_ascii_lowercase();
+            values
+                .iter()
+                .find_map(|(cli_str, enum_value)| (&*lowered == *cli_str).then_some(*enum_value))
+                .ok_or_else(|| {
+                    anyhow!(
+                        "`{f}` is not a valid `--print-output-when` value; expected one of {}",
+                        fmt::from_fn(|f| {
+                            f.debug_list()
+                                .entries(values.iter().map(|(cli, _enum)| cli))
+                                .finish()
+                        })
+                    )
+                })
+        })
+        .transpose()?;
+
     let running_on_backend = args.opt_value_from_str::<_, String>("--backend")?;
     let mut filter_pattern = args.opt_value_from_str::<_, String>("--filter")?;
     let mut filter_invert = false;
@@ -107,18 +137,21 @@ pub fn run_cts(
         })
         .collect::<Vec<_>>();
 
+    let mut default_output_filter = PrintOutputWhen::Always;
+
     if tests.is_empty() && list_files.is_empty() {
         if passthrough_args.is_none() {
             log::info!("Reading default test list from {CTS_DEFAULT_TEST_LIST}");
             list_files.push(OsString::from(CTS_DEFAULT_TEST_LIST));
 
-            // Reduce output, unless `--verbose` was specified.
-            quiet = !verbose;
+            default_output_filter = PrintOutputWhen::TestFails;
         }
     } else if passthrough_args.is_some() {
         bail!("Test(s) and test list(s) are incompatible with passthrough arguments.");
     }
 
+    let output_filter = output_filter.unwrap_or(default_output_filter);
+
     for file in list_files {
         tests.extend(shell.read_file(file)?.lines().filter_map(|line| {
             static TEST_LINE_REGEX: LazyLock<Regex> = LazyLock::new(|| {
@@ -311,7 +344,7 @@ pub fn run_cts(
             }
         }
 
-        if !quiet {
+        if let PrintOutputWhen::Always = output_filter {
             log::info!("Running {}", test.selector.to_string_lossy());
         }
 
@@ -330,26 +363,29 @@ pub fn run_cts(
             .args(["--", "./tools/run_deno", "--verbose"])
             .args([&test.selector]);
 
-        if quiet {
-            let output = cmd.ignore_status().output().context("Failed to run CTS")?;
-            let stdout = String::from_utf8_lossy(&output.stdout);
-            let stderr = String::from_utf8_lossy(&output.stderr);
-
-            if output.status.success() {
-                if let Some((_, summary)) = stdout.split_once("** Summary **") {
-                    println!("\n== Summary for {} ==", test.selector.to_string_lossy());
-                    println!("{}", summary.trim());
+        match output_filter {
+            PrintOutputWhen::TestFails => {
+                let output = cmd.ignore_status().output().context("Failed to run CTS")?;
+                let stdout = String::from_utf8_lossy(&output.stdout);
+                let stderr = String::from_utf8_lossy(&output.stderr);
+
+                if output.status.success() {
+                    if let Some((_, summary)) = stdout.split_once("** Summary **") {
+                        println!("\n== Summary for {} ==", test.selector.to_string_lossy());
+                        println!("{}", summary.trim());
+                    } else {
+                        print!("{}", stdout);
+                        eprint!("{}", stderr);
+                    }
                 } else {
                     print!("{}", stdout);
                     eprint!("{}", stderr);
+                    bail!("CTS failed");
                 }
-            } else {
-                print!("{}", stdout);
-                eprint!("{}", stderr);
-                bail!("CTS failed");
             }
-        } else {
-            cmd.run().context("CTS failed")?;
+            PrintOutputWhen::Always => {
+                cmd.run().context("CTS failed")?;
+            }
         }
     }
 

xtask/src/main.rs

@@ -22,8 +22,9 @@ Commands:
   cts [<options>] [<test selector...> | -f <test list file...> | -- <args...>]
     Check out, build, and run CTS tests
 
-    If no command-line arguments are specified, runs the default test list
-    in `cts_runner/test.lst`, with quiet mode enabled.
+    If no command-line arguments are specified, runs as if `cts
+    cts_runner/test.lst --print-output-when=test-fails` were
+    specified.
 
     --skip-checkout         Don't check out the pinned CTS version, use whatever
                             is already checked out.
@@ -34,8 +35,10 @@ Commands:
     --filter <regex>        Filter tests by selector using a regex pattern.
                             Prefix with '!' to invert (exclude matching tests).
                             Applied after all tests are collected.
-    --quiet                 Only show test counts for suites without failures.
-    --verbose               Show full output when running the default test list.
+    --print-output-when     One of `test-fails` or `always`. When no `test
+                            selector`, `test list file`, or `args` are
+                            specified, this defaults to `test-fails`. Otherwise,
+                            this defaults to `always`.
 
   run-wasm
     Build and run web examples