diff --git a/moz-webgpu-cts/src/main.rs b/moz-webgpu-cts/src/main.rs index 738aa2bf..7edec083 100644 --- a/moz-webgpu-cts/src/main.rs +++ b/moz-webgpu-cts/src/main.rs @@ -20,7 +20,7 @@ use std::{ fmt::{self, Debug, Display, Formatter}, fs, hash::Hash, - io::{self, BufReader, BufWriter}, + io::{self, BufReader, BufWriter, Read}, path::{Path, PathBuf}, process::ExitCode, sync::{mpsc::channel, Arc}, @@ -30,7 +30,7 @@ use camino::Utf8PathBuf; use clap::{Parser, ValueEnum}; use enumset::EnumSetType; use format::lazy_format; -use indexmap::{IndexMap, IndexSet}; +use indexmap::{Equivalent, IndexMap, IndexSet}; use joinery::JoinableIterator; use miette::{miette, Diagnostic, IntoDiagnostic, NamedSource, Report, SourceSpan, WrapErr}; use path_dsl::path; @@ -47,6 +47,8 @@ use whippit::{ struct Cli { #[clap(long)] gecko_checkout: Option, + #[clap(long)] + servo: bool, #[clap(subcommand)] subcommand: Subcommand, } @@ -104,6 +106,10 @@ enum ReportProcessingPreset { #[value(alias("same-fx"))] Merge, ResetAll, + /// Set's only reported results to new value + Set, + /// Resets only bad->good + SetGood, } #[derive(Clone, Copy, Debug, Default, ValueEnum)] @@ -125,6 +131,7 @@ fn run(cli: Cli) -> ExitCode { let Cli { gecko_checkout, subcommand, + servo, } = cli; let gecko_checkout = match gecko_checkout @@ -136,8 +143,11 @@ fn run(cli: Cli) -> ExitCode { }; let read_metadata = || -> Result<_, AlreadyReportedToCommandline> { - let webgpu_cts_meta_parent_dir = - { path!(&gecko_checkout | "testing" | "web-platform" | "mozilla" | "meta" | "webgpu") }; + let webgpu_cts_meta_parent_dir = if !servo { + path!(&gecko_checkout | "testing" | "web-platform" | "mozilla" | "meta" | "webgpu") + } else { + path!(&gecko_checkout | "tests" | "wpt" | "webgpu" | "meta" | "webgpu") + }; let mut found_err = false; let collected = @@ -421,11 +431,13 @@ fn run(cli: Cli) -> ExitCode { .into_par_iter() .for_each_with(exec_reports_sender, |sender, path| { let res = fs::File::open(&path) - .map(BufReader::new) .map_err(Report::msg) .wrap_err("failed to open file") - .and_then(|reader| { - serde_json::from_reader::<_, ExecutionReport>(reader) + .and_then(|mut f| { + let mut s = String::new(); + f.read_to_string(&mut s).unwrap(); + let first = s.lines().next().unwrap(); + serde_json::from_str::(first) .into_diagnostic() .wrap_err("failed to parse JSON") }) @@ -620,6 +632,22 @@ fn run(cli: Cli) -> ExitCode { Some(rep) => meta | rep, None => meta, }, + ReportProcessingPreset::Set => |meta, rep| match rep { + Some(rep) => rep, + None => meta, + }, + ReportProcessingPreset::SetGood => { + |meta: Expectation<_>, rep: Option>| match rep { + Some(rep) => { + if rep.equivalent(&Expectation::default()) { + rep + } else { + meta + } + } + None => meta, + } + } }; normalize( @@ -662,7 +690,7 @@ fn run(cli: Cli) -> ExitCode { if test_entry.reported.is_empty() { match preset { - ReportProcessingPreset::Merge => { + ReportProcessingPreset::Merge | ReportProcessingPreset::Set => { log::warn!("no entries found in reports for {test_path:?}") } ReportProcessingPreset::ResetAll @@ -673,6 +701,9 @@ fn run(cli: Cli) -> ExitCode { ); return None; } + ReportProcessingPreset::SetGood => { + log::warn!("no good results in {test_path:?}") + } } } @@ -707,7 +738,7 @@ fn run(cli: Cli) -> ExitCode { let mut files = BTreeMap::::new(); for (test_path, (properties, subtests)) in recombined_tests_iter { let name = test_path.test_name().to_string(); - let rel_path = Utf8PathBuf::from(test_path.rel_metadata_path_fx().to_string()); + let rel_path = Utf8PathBuf::from(test_path.rel_metadata_path_fx(servo).to_string()); let path = gecko_checkout.join(&rel_path); let file = files.entry(path).or_insert_with(|| File { properties: file_props_by_file @@ -926,6 +957,7 @@ fn run(cli: Cli) -> ExitCode { tests_with_runner_errors: TestSet, tests_with_disabled_or_skip: TestSet, tests_with_crashes: TestSet, + tests_with_fails: TestSet, subtests_with_failures_by_test: SubtestByTestSet, subtests_with_timeouts_by_test: SubtestByTestSet, } @@ -1087,6 +1119,15 @@ fn run(cli: Cli) -> ExitCode { outcome, ) }), + TestOutcome::Pass => (), + TestOutcome::Fail => receiver(&mut |analysis| { + insert_in_test_set( + &mut analysis.tests_with_fails, + test_name, + expectation, + outcome, + ) + }), } } } @@ -1261,6 +1302,7 @@ fn run(cli: Cli) -> ExitCode { tests_with_runner_errors, tests_with_disabled_or_skip, tests_with_crashes, + tests_with_fails, subtests_with_failures_by_test, subtests_with_timeouts_by_test, } = analysis; diff --git a/moz-webgpu-cts/src/metadata.rs b/moz-webgpu-cts/src/metadata.rs index 503d1df6..bc50541d 100644 --- a/moz-webgpu-cts/src/metadata.rs +++ b/moz-webgpu-cts/src/metadata.rs @@ -1122,6 +1122,8 @@ pub enum TestOutcome { Crash, Error, Skip, + Pass, + Fail, } impl Default for TestOutcome { @@ -1141,6 +1143,8 @@ impl Display for TestOutcome { Self::Crash => "CRASH", Self::Error => "ERROR", Self::Skip => "SKIP", + Self::Pass => "PASS", + Self::Fail => "FAIL", } ) } @@ -1159,6 +1163,8 @@ impl<'a> Properties<'a> for TestProps { keyword("TIMEOUT").to(TestOutcome::Timeout), keyword("ERROR").to(TestOutcome::Error), keyword("SKIP").to(TestOutcome::Skip), + keyword("PASS").to(TestOutcome::Pass), + keyword("FAIL").to(TestOutcome::Fail), )), ) .boxed() @@ -1364,24 +1370,6 @@ r#" assert_debug_snapshot!( parser().parse( r#" -[asdf] - # Incorrect; `PASS` isn't a valid test outcome (but it _is_ a valid subtest outcome). - expected: PASS -"# - ), - @r###" - ParseResult { - output: None, - errs: [ - found end of input at 108..112 expected something else, - ], - } - "### - ); - - assert_debug_snapshot!( - parser().parse( -r#" [asdf] [blarg] expected: [PASS, FAIL] @@ -1629,4 +1617,40 @@ r#" } "### ); + assert_debug_snapshot!( + parser().parse(r#" +[canvas_complex_rgba8unorm_store.https.html] + expected: [PASS, FAIL, CRASH] + +"#), + @r###" + ParseResult { + output: Some( + ( + "canvas_complex_rgba8unorm_store.https.html", + Test { + properties: TestProps { + is_disabled: false, + expectations: Some( + NormalizedExpectationPropertyValue( + Collapsed( + Collapsed( + [ + Crash, + Pass, + Fail, + ], + ), + ), + ), + ), + }, + subtests: {}, + }, + ), + ), + errs: [], + } + "### + ); } diff --git a/moz-webgpu-cts/src/shared.rs b/moz-webgpu-cts/src/shared.rs index 76acee52..13458e10 100644 --- a/moz-webgpu-cts/src/shared.rs +++ b/moz-webgpu-cts/src/shared.rs @@ -338,6 +338,8 @@ const SCOPE_DIR_FX_PRIVATE_STR: &str = "testing/web-platform/mozilla"; const SCOPE_DIR_FX_PRIVATE_COMPONENTS: &[&str] = &["testing", "web-platform", "mozilla"]; const SCOPE_DIR_FX_PUBLIC_STR: &str = "testing/web-platform"; const SCOPE_DIR_FX_PUBLIC_COMPONENTS: &[&str] = &["testing", "web-platform"]; +const SCOPE_DIR_SERVO_PUBLIC_STR: &str = "tests/wpt/webgpu"; +const SCOPE_DIR_SERVO_PUBLIC_COMPONENTS: &[&str] = &["tests", "wpt", "webgpu"]; impl<'a> TestPath<'a> { pub fn from_execution_report( @@ -349,8 +351,13 @@ impl<'a> TestPath<'a> { .map(|stripped| (TestScope::FirefoxPrivate, stripped)) .or_else(|| { test_url_path - .strip_prefix('/') + .strip_prefix("/_webgpu/") .map(|stripped| (TestScope::Public, stripped)) + .or_else(|| { + test_url_path + .strip_prefix('/') + .map(|stripped| (TestScope::Public, stripped)) + }) }) else { return Err(err()); @@ -403,6 +410,8 @@ impl<'a> TestPath<'a> { (TestScope::FirefoxPrivate, path) } else if let Ok(path) = rel_meta_file_path.strip_prefix(SCOPE_DIR_FX_PUBLIC_STR) { (TestScope::Public, path) + } else if let Ok(path) = rel_meta_file_path.strip_prefix(SCOPE_DIR_SERVO_PUBLIC_STR) { + (TestScope::Public, path) } else { return Err(err()); } @@ -484,16 +493,23 @@ impl<'a> TestPath<'a> { }) } - pub(crate) fn rel_metadata_path_fx(&self) -> impl Display + '_ { + pub(crate) fn rel_metadata_path_fx(&self, servo: bool) -> impl Display + '_ { let Self { path, variant: _, scope, } = self; - let scope_dir = match scope { - TestScope::Public => SCOPE_DIR_FX_PUBLIC_COMPONENTS, - TestScope::FirefoxPrivate => SCOPE_DIR_FX_PRIVATE_COMPONENTS, + let scope_dir = if !servo { + match scope { + TestScope::Public => SCOPE_DIR_FX_PUBLIC_COMPONENTS, + TestScope::FirefoxPrivate => SCOPE_DIR_FX_PRIVATE_COMPONENTS, + } + } else { + match scope { + TestScope::Public => SCOPE_DIR_SERVO_PUBLIC_COMPONENTS, + TestScope::FirefoxPrivate => todo!(), + } } .iter() .chain(&["meta"])