Skip to content

Add Servo integration via --browser=servo #92

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ErichDonGubler merged 4 commits into from
May 11, 2024
+224 −104
Merged

Add Servo integration via --browser=servo #92

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
289ecee
Add servo as browser option
sagudev Apr 16, 2024
0c32d99
fix: use more general language in repo-root finding
ErichDonGubler May 10, 2024
30e61d7
fix(update-expected): offer more general language and aliases
ErichDonGubler May 10, 2024
062b2b9
ci: try combining `push` and `pr` jobs without dupes (!)
ErichDonGubler May 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 52 additions & 43 deletions moz-webgpu-cts/src/main.rs
View file
Open in desktop
Comment thread
sagudev marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ use joinery::JoinableIterator;
use miette::{miette, Diagnostic, IntoDiagnostic, NamedSource, Report, SourceSpan, WrapErr};
use path_dsl::path;
use rayon::prelude::{IntoParallelIterator, ParallelIterator};
use shared::Browser;
use wax::Glob;
use whippit::{
metadata::SectionHeader,
Expand All @@ -51,29 +52,31 @@ use whippit::{
#[derive(Debug, Parser)]
#[command(about, version)]
struct Cli {
#[clap(long)]
gecko_checkout: Option<PathBuf>,
#[clap(long, alias = "gecko-checkout")]
checkout: Option<PathBuf>,
#[clap(value_enum, long, default_value = "firefox")]
browser: Browser,
#[clap(subcommand)]
subcommand: Subcommand,
}

#[derive(Debug, Parser)]
enum Subcommand {
/// Adjust expected test outcomes in metadata, optionally using `wptreport.json` reports from
/// CI runs covering Firefox's implementation of WebGPU.
/// CI runs covering your browser's implementation of WebGPU.
///
/// As Firefox's behavior changes, one generally expects CTS test outcomes to change. When you
/// are testing your own changes in CI, you can use this subcommand to update expected outcomes
/// automatically with the following steps:
/// As your browser's behavior changes, one generally expects CTS test outcomes to change. When
/// you are testing your own changes in CI, you can use this subcommand to update expected
/// outcomes automatically with the following steps:
///
/// 1. Run `moz-webgpu-cts update-expected --preset=new-fx …` against the first complete set of
/// reports you gather from CI with your new Firefox build. This will adjust for new
/// 1. Run `moz-webgpu-cts update-expected --preset=new-build …` against the first complete set
/// of reports you gather from CI with your new browser build. This will adjust for new
/// permanent outcomes, and may capture some (but not all) intermittent outcomes.
///
/// 2. There may still exist intermittent issues that you do not discover in CI run(s) from the
/// previous step. As you discover them in further CI runs on the same build of Firefox,
/// adjust expected outcomes to match by running `moz-webgpu-cts update-expected
/// --preset=same-fx …` against the runs' new reports. Repeat as necessary.
/// --preset=same-build …` against the runs' new reports. Repeat as necessary.
///
/// With both steps, you may delete the local copies of these reports after being processed
/// with `update-expected`. You should not need to re-process them unless you have made an
Expand Down Expand Up @@ -107,9 +110,11 @@ enum Subcommand {
enum ReportProcessingPreset {
/// alias: `new-fx`
#[value(alias("new-fx"))]
#[value(alias("new-build"))]
ResetContradictory,
/// alias: `same-fx`
#[value(alias("same-fx"))]
#[value(alias("same-build"))]
Merge,
ResetAll,
}
Expand All @@ -131,14 +136,12 @@ fn main() -> ExitCode {

fn run(cli: Cli) -> ExitCode {
let Cli {
gecko_checkout,
browser,
checkout,
subcommand,
} = cli;

let gecko_checkout = match gecko_checkout
.map(Ok)
.unwrap_or_else(search_for_moz_central_ckt)
{
let checkout = match checkout.map(Ok).unwrap_or_else(search_for_repo_root) {
Ok(ckt_path) => ckt_path,
Err(AlreadyReportedToCommandline) => return ExitCode::FAILURE,
};
Expand Down Expand Up @@ -244,7 +247,7 @@ fn run(cli: Cli) -> ExitCode {
log::trace!("working with the following WPT report files: {exec_report_paths:#?}");
log::info!("working with {} WPT report files", exec_report_paths.len());

let meta_files_by_path = match read_and_parse_all_metadata(&gecko_checkout)
let meta_files_by_path = match read_and_parse_all_metadata(browser, &checkout)
.collect::<Result<IndexMap<_, _>, _>>()
{
Ok(paths) => paths,
Expand Down Expand Up @@ -276,7 +279,7 @@ fn run(cli: Cli) -> ExitCode {
for (path, file) in meta_files_by_path {
let File { properties, tests } = file;

let file_rel_path = path.strip_prefix(&gecko_checkout).unwrap();
let file_rel_path = path.strip_prefix(&checkout).unwrap();

file_props_by_file.insert(
Utf8PathBuf::from(file_rel_path.to_str().unwrap()),
Expand All @@ -289,7 +292,8 @@ fn run(cli: Cli) -> ExitCode {
subtests,
} = test;

let test_path = TestPath::from_fx_metadata_test(file_rel_path, &name).unwrap();
let test_path =
TestPath::from_metadata_test(browser, file_rel_path, &name).unwrap();

let freak_out_do_nothing = |what: &dyn Display| {
log::error!("hoo boy, not sure what to do yet: {what}")
Expand Down Expand Up @@ -402,7 +406,7 @@ fn run(cli: Cli) -> ExitCode {
for entry in entries {
let TestExecutionEntry { test_name, result } = entry;

let test_path = TestPath::from_execution_report(&test_name).unwrap();
let test_path = TestPath::from_execution_report(browser, &test_name).unwrap();
let TestEntry {
entry: test_entry,
subtests: subtest_entries,
Expand Down Expand Up @@ -682,8 +686,8 @@ fn run(cli: Cli) -> ExitCode {
let mut files = BTreeMap::<PathBuf, File>::new();
for (test_path, (properties, subtests)) in recombined_tests_iter {
let name = test_path.test_name().to_string();
let rel_path = Utf8PathBuf::from(test_path.rel_metadata_path_fx().to_string());
let path = gecko_checkout.join(&rel_path);
let rel_path = Utf8PathBuf::from(test_path.rel_metadata_path().to_string());
let path = checkout.join(&rel_path);
let file = files.entry(path).or_insert_with(|| File {
properties: file_props_by_file
.get(&rel_path)
Expand Down Expand Up @@ -751,7 +755,7 @@ fn run(cli: Cli) -> ExitCode {
}
Subcommand::Fixup => {
log::info!("fixing up metadata in-place…");
let err_found = read_and_parse_all_metadata(&gecko_checkout)
let err_found = read_and_parse_all_metadata(browser, &checkout)
.map(|res| {
res.and_then(|(path, mut file)| {
for test in file.tests.values_mut() {
Expand Down Expand Up @@ -789,7 +793,7 @@ fn run(cli: Cli) -> ExitCode {
inner: Test,
}
let mut err_found = false;
let tests_by_name = read_and_parse_all_metadata(&gecko_checkout)
let tests_by_name = read_and_parse_all_metadata(browser, &checkout)
.map_ok(
|(
path,
Expand All @@ -799,11 +803,12 @@ fn run(cli: Cli) -> ExitCode {
},
)| {
tests.into_iter().map({
let gecko_checkout = &gecko_checkout;
let checkout = &checkout;
move |(name, inner)| {
let SectionHeader(name) = &name;
let test_path = TestPath::from_fx_metadata_test(
path.strip_prefix(gecko_checkout).unwrap(),
let test_path = TestPath::from_metadata_test(
browser,
path.strip_prefix(checkout).unwrap(),
name,
)
.unwrap();
Expand Down Expand Up @@ -1353,13 +1358,17 @@ fn run(cli: Cli) -> ExitCode {
}

fn read_and_parse_all_metadata(
gecko_checkout: &Path,
browser: Browser,
checkout: &Path,
) -> impl Iterator<Item = Result<(Arc<PathBuf>, metadata::File), AlreadyReportedToCommandline>> {
let webgpu_cts_meta_parent_dir =
path!(gecko_checkout | "testing" | "web-platform" | "mozilla" | "meta" | "webgpu");
let webgpu_cts_meta_parent_dir = match browser {
Browser::Firefox => {
path!(&checkout | "testing" | "web-platform" | "mozilla" | "meta" | "webgpu")
}
Browser::Servo => path!(&checkout | "tests" | "wpt" | "webgpu" | "meta" | "webgpu"),
};

let raw_metadata_files =
read_gecko_files_at(gecko_checkout, &webgpu_cts_meta_parent_dir, "**/*.ini");
let raw_metadata_files = read_files_at(checkout, &webgpu_cts_meta_parent_dir, "**/*.ini");

let mut started_parsing = false;
raw_metadata_files.filter_map(move |res| {
Expand Down Expand Up @@ -1420,20 +1429,20 @@ fn render_metadata_parse_errors<'a>(
}

/// Returns a "naturally" sorted list of files found by searching for `glob_pattern` in `base`.
/// `gecko_checkout` is stripped as a prefix from the absolute paths recorded into `log` entries
/// emitted by this function.
/// `checkout` is stripped as a prefix from the absolute paths recorded into `log` entries emitted
/// by this function.
///
/// # Returns
///
/// An iterator over [`Result`]s containing either a Gecko file's path and contents as a UTF-8
/// An iterator over [`Result`]s containing either a checkout file's path and contents as a UTF-8
/// string, or the sentinel of an error encountered for the same file that is already reported to
/// the command line.
///
/// # Panics
///
/// This function will panick if `gecko_checkout` cannot be stripped as a prefix of `base`.
fn read_gecko_files_at(
gecko_checkout: &Path,
/// This function will panick if `checkout` cannot be stripped as a prefix of `base`.
fn read_files_at(
checkout: &Path,
base: &Path,
glob_pattern: &str,
) -> impl Iterator<Item = Result<(PathBuf, String), AlreadyReportedToCommandline>> {
Expand All @@ -1447,7 +1456,7 @@ fn read_gecko_files_at(
Err(e) => {
let path_disp = e
.path()
.map(|p| format!(" in {}", p.strip_prefix(gecko_checkout).unwrap().display()));
.map(|p| format!(" in {}", p.strip_prefix(checkout).unwrap().display()));
let path_disp: &dyn Display = match path_disp.as_ref() {
Some(disp) => disp,
None => &"",
Expand All @@ -1469,7 +1478,7 @@ fn read_gecko_files_at(
"working with these files: {:#?}",
paths
.iter()
.map(|f| f.strip_prefix(gecko_checkout).unwrap())
.map(|f| f.strip_prefix(checkout).unwrap())
.collect::<std::collections::BTreeSet<_>>()
);

Expand All @@ -1494,19 +1503,19 @@ fn read_gecko_files_at(
.chain(file_read_iter.into_iter().flatten())
}

/// Search for a `mozilla-central` checkout either via Mercurial or Git, iterating from the CWD to
/// Search for source code repository root either via Mercurial or Git, iterating from the CWD to
/// its parent directories.
///
/// This function reports to `log` automatically, so no meaningful [`Err`] value is returned.
fn search_for_moz_central_ckt() -> Result<PathBuf, AlreadyReportedToCommandline> {
fn search_for_repo_root() -> Result<PathBuf, AlreadyReportedToCommandline> {
use lets_find_up::{find_up_with, FindUpKind, FindUpOptions};

let find_up_opts = || FindUpOptions {
cwd: Path::new("."),
kind: FindUpKind::Dir,
};
let find_up = |repo_tech_name, root_dir_name| {
log::debug!("searching for {repo_tech_name} checkout of `mozilla-central`…");
log::debug!("searching for {repo_tech_name} repository root…");
let err = || {
miette!(
"failed to find a {} repository ({:?}) in {}",
Expand All @@ -1533,13 +1542,13 @@ fn search_for_moz_central_ckt() -> Result<PathBuf, AlreadyReportedToCommandline>
Err(e2) => {
log::warn!("{e:?}");
log::warn!("{e2:?}");
log::error!("failed to find a Gecko repository root");
log::error!("failed to automatically find a repository root");
Err(AlreadyReportedToCommandline)
}
})?;

log::info!(
"detected Gecko repository root at {}",
"detected repository root at {}",
gecko_source_root.display()
);

Expand Down
Loading