diff --git a/moz-webgpu-cts/src/main.rs b/moz-webgpu-cts/src/main.rs index f3cd9c1..1a69c7b 100644 --- a/moz-webgpu-cts/src/main.rs +++ b/moz-webgpu-cts/src/main.rs @@ -62,6 +62,33 @@ struct Cli { #[derive(Debug, Parser)] enum Subcommand { + /// Migrate old test structure in metadata to that found in `wptreport.json` reports. + /// + /// When a new version of CTS is run by your implementation of WebGPU, new execution reports + /// may differ from old ones in various ways: + /// + /// 1. Test may have been added, deleted, or moved. + /// + /// It requires human judgment to determine what additions and deletions are actually + /// movements of the same test coverage. + /// 2. Tests' actual outcomes on your implementation may change, since implementations of + /// existing tests may have changed. + /// + /// This command implements (only) the changes from (1) in your metadata by using the reports + /// you provide to: + /// + /// 1. Remove _all_ metadata from test paths that are currently in metadata, but not observedn + /// execution reports. + /// 2. Add empty metadata entries for test paths that are observed in execution reports, but + /// absent from current metadata. + /// + /// The diff produced by the above changes makes it easier to determine what tests may have + /// moved, and, by extension, whether you should attempt to migrate metadata for subsequent + /// test runs. + Migrate { + #[clap(flatten)] + exec_report_spec: ExecReportSpec, + }, /// Adjust expected test outcomes in metadata, optionally using `wptreport.json` reports from /// CI runs covering your browser's implementation of WebGPU. /// @@ -292,6 +319,16 @@ fn run(cli: Cli) -> ExitCode { }; match subcommand { + Subcommand::Migrate { exec_report_spec } => match process_reports( + browser, + &checkout, + exec_report_spec, + process_reports::ReportProcessingPreset::MigrateTestStructure, + &mut should_update_expected::NeverUpdateExpected, + ) { + Ok(()) => ExitCode::SUCCESS, + Err(AlreadyReportedToCommandline) => ExitCode::FAILURE, + }, Subcommand::UpdateExpected { exec_report_spec, preset, diff --git a/moz-webgpu-cts/src/process_reports.rs b/moz-webgpu-cts/src/process_reports.rs index 6273f7f..7b4758d 100644 --- a/moz-webgpu-cts/src/process_reports.rs +++ b/moz-webgpu-cts/src/process_reports.rs @@ -65,6 +65,7 @@ pub(crate) enum ReportProcessingPreset { ResetContradictoryOutcomes, MergeOutcomes, ResetAllOutcomes, + MigrateTestStructure, } #[derive(Debug, Default)] @@ -132,6 +133,7 @@ fn reconcile( Some(rep) => meta | rep, None => meta, }, + ReportProcessingPreset::MigrateTestStructure => |meta, _rep| meta, }; ExpandedPropertyValue::from_query(|platform, build_profile| { @@ -447,6 +449,10 @@ pub(crate) fn process_reports( log::warn!("removing metadata after {msg}"); return None; } + ReportProcessingPreset::MigrateTestStructure => { + log::info!("removing metadata after {msg}"); + return None; + } } } @@ -521,6 +527,10 @@ pub(crate) fn process_reports( log::warn!("removing metadata after {msg}"); return None; } + ReportProcessingPreset::MigrateTestStructure => { + log::info!("removing metadata after {msg}"); + return None; + } } } diff --git a/moz-webgpu-cts/src/process_reports/should_update_expected.rs b/moz-webgpu-cts/src/process_reports/should_update_expected.rs index b7453d6..46ea33e 100644 --- a/moz-webgpu-cts/src/process_reports/should_update_expected.rs +++ b/moz-webgpu-cts/src/process_reports/should_update_expected.rs @@ -60,3 +60,27 @@ impl ShouldUpdateExpected for ImplementationStatusFilter { self.is_allowed(status) } } + +#[derive(Debug)] +pub(crate) struct NeverUpdateExpected; + +impl ShouldUpdateExpected for NeverUpdateExpected { + fn test( + &mut self, + _meta_props: &TestProps, + _reported: &NonNormalizedPropertyValue>, + _key: (Platform, BuildProfile), + ) -> bool { + false + } + + fn subtest( + &mut self, + _meta_props: &TestProps, + _reported: &NonNormalizedPropertyValue>, + _parent_meta_props: &TestProps, + _key: (Platform, BuildProfile), + ) -> bool { + false + } +}