Merge pull request #127 from ErichDonGubler/update-test-structure

Add `migrate` subcmd.

Author: ErichDonGubler

moz-webgpu-cts/src/main.rs

@@ -62,6 +62,33 @@ struct Cli {
 
 #[derive(Debug, Parser)]
 enum Subcommand {
+    /// Migrate old test structure in metadata to that found in `wptreport.json` reports.
+    ///
+    /// When a new version of CTS is run by your implementation of WebGPU, new execution reports
+    /// may differ from old ones in various ways:
+    ///
+    /// 1. Test may have been added, deleted, or moved.
+    ///
+    ///    It requires human judgment to determine what additions and deletions are actually
+    ///    movements of the same test coverage.
+    /// 2. Tests' actual outcomes on your implementation may change, since implementations of
+    ///    existing tests may have changed.
+    ///
+    /// This command implements (only) the changes from (1) in your metadata by using the reports
+    /// you provide to:
+    ///
+    /// 1. Remove _all_ metadata from test paths that are currently in metadata, but not observedn
+    ///    execution reports.
+    /// 2. Add empty metadata entries for test paths that are observed in execution reports, but
+    ///    absent from current metadata.
+    ///
+    /// The diff produced by the above changes makes it easier to determine what tests may have
+    /// moved, and, by extension, whether you should attempt to migrate metadata for subsequent
+    /// test runs.
+    Migrate {
+        #[clap(flatten)]
+        exec_report_spec: ExecReportSpec,
+    },
     /// Adjust expected test outcomes in metadata, optionally using `wptreport.json` reports from
     /// CI runs covering your browser's implementation of WebGPU.
     ///
@@ -292,6 +319,16 @@ fn run(cli: Cli) -> ExitCode {
     };
 
     match subcommand {
+        Subcommand::Migrate { exec_report_spec } => match process_reports(
+            browser,
+            &checkout,
+            exec_report_spec,
+            process_reports::ReportProcessingPreset::MigrateTestStructure,
+            &mut should_update_expected::NeverUpdateExpected,
+        ) {
+            Ok(()) => ExitCode::SUCCESS,
+            Err(AlreadyReportedToCommandline) => ExitCode::FAILURE,
+        },
         Subcommand::UpdateExpected {
             exec_report_spec,
             preset,

moz-webgpu-cts/src/process_reports.rs

@@ -65,6 +65,7 @@ pub(crate) enum ReportProcessingPreset {
     ResetContradictoryOutcomes,
     MergeOutcomes,
     ResetAllOutcomes,
+    MigrateTestStructure,
 }
 
 #[derive(Debug, Default)]
@@ -132,6 +133,7 @@ fn reconcile<Out>(
                 Some(rep) => meta | rep,
                 None => meta,
             },
+            ReportProcessingPreset::MigrateTestStructure => |meta, _rep| meta,
         };
 
         ExpandedPropertyValue::from_query(|platform, build_profile| {
@@ -447,6 +449,10 @@ pub(crate) fn process_reports(
                         log::warn!("removing metadata after {msg}");
                         return None;
                     }
+                    ReportProcessingPreset::MigrateTestStructure => {
+                        log::info!("removing metadata after {msg}");
+                        return None;
+                    }
                 }
             }
 
@@ -521,6 +527,10 @@ pub(crate) fn process_reports(
                                 log::warn!("removing metadata after {msg}");
                                 return None;
                             }
+                            ReportProcessingPreset::MigrateTestStructure => {
+                                log::info!("removing metadata after {msg}");
+                                return None;
+                            }
                         }
                     }
 

moz-webgpu-cts/src/process_reports/should_update_expected.rs

@@ -60,3 +60,27 @@ impl ShouldUpdateExpected for ImplementationStatusFilter {
         self.is_allowed(status)
     }
 }
+
+#[derive(Debug)]
+pub(crate) struct NeverUpdateExpected;
+
+impl ShouldUpdateExpected for NeverUpdateExpected {
+    fn test(
+        &mut self,
+        _meta_props: &TestProps<TestOutcome>,
+        _reported: &NonNormalizedPropertyValue<Expected<TestOutcome>>,
+        _key: (Platform, BuildProfile),
+    ) -> bool {
+        false
+    }
+
+    fn subtest(
+        &mut self,
+        _meta_props: &TestProps<SubtestOutcome>,
+        _reported: &NonNormalizedPropertyValue<Expected<SubtestOutcome>>,
+        _parent_meta_props: &TestProps<TestOutcome>,
+        _key: (Platform, BuildProfile),
+    ) -> bool {
+        false
+    }
+}