From 3f2c0735618830116f1e13b3459c1ee3891eaaea Mon Sep 17 00:00:00 2001 From: Erich Gubler Date: Fri, 9 Aug 2024 09:57:16 -0400 Subject: [PATCH 1/6] chore: alphabetize deps. in `moz-webgpu-cts/Cargo.toml` --- moz-webgpu-cts/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moz-webgpu-cts/Cargo.toml b/moz-webgpu-cts/Cargo.toml index 696d287..718b846 100644 --- a/moz-webgpu-cts/Cargo.toml +++ b/moz-webgpu-cts/Cargo.toml @@ -13,8 +13,9 @@ dist = true [dependencies] camino = { version = "1.1.6", features = ["serde1"] } clap = { version = "4.4.2", features = ["derive"] } -env_logger = "0.10.0" +enum-map = { version = "2.7.3", features = ["serde"] } enumset = { version = "1.1.3", features = ["serde"] } +env_logger = "0.10.0" format = { workspace = true } indexmap = { workspace = true } itertools = "0.11.0" @@ -31,7 +32,6 @@ strum = { version = "0.25.0", features = ["derive"] } thiserror = { workspace = true } wax = { version = "0.6.0", features = ["miette"], git = "https://github.com/ErichDonGubler/wax", branch = "static-miette-diags"} whippit = { version = "0.6.2", path = "../whippit", default-features = false, features = ["serde1"] } -enum-map = { version = "2.7.3", features = ["serde"] } [dev-dependencies] insta = { workspace = true } From 30db619411d99976b3aa343293aafc71d03def73 Mon Sep 17 00:00:00 2001 From: Erich Gubler Date: Fri, 9 Aug 2024 14:52:40 -0400 Subject: [PATCH 2/6] refactor: extract `{TestOutcome,SubtestOutcome}::parser` helpers --- moz-webgpu-cts/src/wpt/metadata.rs | 63 ++++++++++++++++++------------ 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/moz-webgpu-cts/src/wpt/metadata.rs b/moz-webgpu-cts/src/wpt/metadata.rs index f4cae61..5870227 100644 --- a/moz-webgpu-cts/src/wpt/metadata.rs +++ b/moz-webgpu-cts/src/wpt/metadata.rs @@ -22,7 +22,8 @@ use whippit::{ ParseError, SectionHeader, }, reexport::chumsky::{ - input::Emitter, + extra::ParserExtra, + input::{Emitter, Input, StrInput}, prelude::Rich, primitive::{any, choice, end, group, just, one_of}, span::SimpleSpan, @@ -1284,24 +1285,30 @@ impl Display for TestOutcome { } } +impl TestOutcome { + pub(crate) fn parser<'a, I, E>() -> impl Parser<'a, I, TestOutcome, E> + Clone + where + I: Input<'a, Token = char> + StrInput<'a, char>, + E: ParserExtra<'a, I>, + { + choice(( + keyword(OK).to(TestOutcome::Ok), + keyword(PASS).to(TestOutcome::Pass), + keyword(FAIL).to(TestOutcome::Fail), + keyword(CRASH).to(TestOutcome::Crash), + keyword(TIMEOUT).to(TestOutcome::Timeout), + keyword(ERROR).to(TestOutcome::Error), + keyword(SKIP).to(TestOutcome::Skip), + )) + } +} + impl<'a> Properties<'a> for TestProps { type ParsedProperty = TestProp; fn property_parser( helper: &mut PropertiesParseHelper<'a>, ) -> Boxed<'a, 'a, &'a str, Self::ParsedProperty, ParseError<'a>> { - TestProp::property_parser( - helper, - choice(( - keyword(OK).to(TestOutcome::Ok), - keyword(PASS).to(TestOutcome::Pass), - keyword(FAIL).to(TestOutcome::Fail), - keyword(CRASH).to(TestOutcome::Crash), - keyword(TIMEOUT).to(TestOutcome::Timeout), - keyword(ERROR).to(TestOutcome::Error), - keyword(SKIP).to(TestOutcome::Skip), - )), - ) - .boxed() + TestProp::property_parser(helper, TestOutcome::parser()).boxed() } fn add_property(&mut self, prop: Self::ParsedProperty, emitter: &mut Emitter>) { @@ -1343,22 +1350,28 @@ impl Display for SubtestOutcome { } } +impl SubtestOutcome { + pub(crate) fn parser<'a, I, E>() -> impl Parser<'a, I, SubtestOutcome, E> + Clone + where + I: Input<'a, Token = char> + StrInput<'a, char>, + E: ParserExtra<'a, I>, + { + choice(( + keyword(PASS).to(SubtestOutcome::Pass), + keyword(FAIL).to(SubtestOutcome::Fail), + keyword(TIMEOUT).to(SubtestOutcome::Timeout), + keyword(CRASH).to(SubtestOutcome::Crash), + keyword(NOTRUN).to(SubtestOutcome::NotRun), + )) + } +} + impl<'a> Properties<'a> for TestProps { type ParsedProperty = TestProp; fn property_parser( helper: &mut PropertiesParseHelper<'a>, ) -> Boxed<'a, 'a, &'a str, Self::ParsedProperty, ParseError<'a>> { - TestProp::property_parser( - helper, - choice(( - keyword(PASS).to(SubtestOutcome::Pass), - keyword(FAIL).to(SubtestOutcome::Fail), - keyword(TIMEOUT).to(SubtestOutcome::Timeout), - keyword(CRASH).to(SubtestOutcome::Crash), - keyword(NOTRUN).to(SubtestOutcome::NotRun), - )), - ) - .boxed() + TestProp::property_parser(helper, SubtestOutcome::parser()).boxed() } fn add_property(&mut self, prop: Self::ParsedProperty, emitter: &mut Emitter>) { From 3c91bac6d7982d1c65db3a56b81204ff69947eba Mon Sep 17 00:00:00 2001 From: Erich Gubler Date: Fri, 9 Aug 2024 14:52:40 -0400 Subject: [PATCH 3/6] fix: remove `SubtestOutcome::Crash` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This isn't actually a valid subtest outcome. 😅 Sorry for the confusion! --- moz-webgpu-cts/src/main.rs | 8 -------- moz-webgpu-cts/src/wpt/metadata.rs | 3 --- 2 files changed, 11 deletions(-) diff --git a/moz-webgpu-cts/src/main.rs b/moz-webgpu-cts/src/main.rs index ff4b974..eb77c5c 100644 --- a/moz-webgpu-cts/src/main.rs +++ b/moz-webgpu-cts/src/main.rs @@ -770,14 +770,6 @@ fn run(cli: Cli) -> ExitCode { ) }) } - SubtestOutcome::Crash => receiver(&mut |analysis| { - insert_in_test_set( - &mut analysis.tests_with_crashes, - test_name, - expected, - outcome, - ) - }), SubtestOutcome::Fail => receiver(&mut |analysis| { insert_in_subtest_by_test_set( &mut analysis.subtests_with_failures_by_test, diff --git a/moz-webgpu-cts/src/wpt/metadata.rs b/moz-webgpu-cts/src/wpt/metadata.rs index 5870227..b287d08 100644 --- a/moz-webgpu-cts/src/wpt/metadata.rs +++ b/moz-webgpu-cts/src/wpt/metadata.rs @@ -1324,7 +1324,6 @@ pub enum SubtestOutcome { Pass, Fail, Timeout, - Crash, NotRun, } @@ -1343,7 +1342,6 @@ impl Display for SubtestOutcome { Self::Pass => PASS, Self::Fail => FAIL, Self::Timeout => TIMEOUT, - Self::Crash => CRASH, Self::NotRun => NOTRUN, } ) @@ -1360,7 +1358,6 @@ impl SubtestOutcome { keyword(PASS).to(SubtestOutcome::Pass), keyword(FAIL).to(SubtestOutcome::Fail), keyword(TIMEOUT).to(SubtestOutcome::Timeout), - keyword(CRASH).to(SubtestOutcome::Crash), keyword(NOTRUN).to(SubtestOutcome::NotRun), )) } From a9dc78c79607a5bcd087907392eb9b9ee79c295a Mon Sep 17 00:00:00 2001 From: Erich Gubler Date: Fri, 2 Aug 2024 20:14:35 -0400 Subject: [PATCH 4/6] WIP: refactor: extract file-gathering into new internal `FileSpec` API --- moz-webgpu-cts/src/main.rs | 90 ++++++++++++++++++++++++-------------- 1 file changed, 58 insertions(+), 32 deletions(-) diff --git a/moz-webgpu-cts/src/main.rs b/moz-webgpu-cts/src/main.rs index eb77c5c..f502eed 100644 --- a/moz-webgpu-cts/src/main.rs +++ b/moz-webgpu-cts/src/main.rs @@ -155,16 +155,18 @@ struct ExecReportSpec { report_globs: Vec, } -impl ExecReportSpec { - fn paths(self) -> Result, AlreadyReportedToCommandline> { - let Self { - report_paths, - report_globs, - } = self; +struct FileSpec { + paths: Vec, + globs: Vec, +} - let report_globs = { +impl FileSpec { + fn into_paths(self, what: impl Display) -> Result, AlreadyReportedToCommandline> { + let Self { paths, globs } = self; + + let globs = { let mut found_glob_parse_err = false; - let globs = report_globs + let globs = globs .into_iter() .filter_map(|glob| match Glob::diagnosed(&glob) { Ok((glob, _diagnostics)) => Some(glob.into_owned().partition()), @@ -187,16 +189,16 @@ impl ExecReportSpec { .collect::>(); if found_glob_parse_err { - log::error!("failed to parse one or more WPT report globs; bailing"); + log::error!("failed to parse one or more globs for {what}; bailing"); return Err(AlreadyReportedToCommandline); } globs }; - let report_paths_from_glob = { + let paths_from_globs = { let mut found_glob_walk_err = false; - let files = report_globs + let files = globs .iter() .flat_map(|(base_path, glob)| { glob.walk(base_path) @@ -206,12 +208,12 @@ impl ExecReportSpec { found_glob_walk_err = true; let ctx_msg = if let Some(path) = e.path() { format!( - "failed to enumerate files for glob `{}` at path {}", + "failed to enumerate {what} from glob `{}` at path {}", glob, path.display() ) } else { - format!("failed to enumerate files for glob `{glob}`") + format!("failed to enumerate {what} from glob `{glob}`") }; let e = Report::msg(e).wrap_err(ctx_msg); eprintln!("{e:?}"); @@ -223,44 +225,68 @@ impl ExecReportSpec { .collect::>(); if found_glob_walk_err { - log::error!(concat!( - "failed to enumerate files with WPT report globs, ", - "see above for more details" - )); + log::error!( + concat!( + "failed to enumerate {} from globs, ", + "see above for more details" + ), + what + ); return Err(AlreadyReportedToCommandline); } files }; - if report_paths_from_glob.is_empty() && !report_globs.is_empty() { - if report_paths.is_empty() { - log::error!(concat!( - "reports were specified exclusively via glob search, ", - "but none were found; bailing" - )); + if paths_from_globs.is_empty() && !globs.is_empty() { + if paths.is_empty() { + log::error!( + concat!( + "{} were specified exclusively via glob search, ", + "but none were found; bailing" + ), + what + ); return Err(AlreadyReportedToCommandline); } else { - log::warn!(concat!( - "reports were specified via path and glob search, ", - "but none were found via glob; ", - "continuing with report paths" - )) + log::warn!( + concat!( + "{} were specified via path and glob search, ", + "but none were found via glob; ", + "continuing with direct paths" + ), + what + ) } } - let exec_report_paths = report_paths + let exec_report_paths = paths .into_iter() - .chain(report_paths_from_glob) + .chain(paths_from_globs) .collect::>(); - log::trace!("working with the following WPT report files: {exec_report_paths:#?}"); - log::info!("working with {} WPT report files", exec_report_paths.len()); + log::trace!("working with the following {what}: {exec_report_paths:#?}"); + log::info!("working with {} {what}", exec_report_paths.len()); Ok(exec_report_paths) } } +impl ExecReportSpec { + fn paths(self) -> Result, AlreadyReportedToCommandline> { + let Self { + report_paths, + report_globs, + } = self; + + FileSpec { + paths: report_paths, + globs: report_globs, + } + .into_paths("WPT report(s)") + } +} + #[derive(Clone, Copy, Debug, ValueEnum)] enum UpdateExpectedPreset { /// alias: `new-fx` From a1d69a7457e7573f3e0665d08b5d1173421374e7 Mon Sep 17 00:00:00 2001 From: Erich Gubler Date: Fri, 9 Aug 2024 08:31:10 -0400 Subject: [PATCH 5/6] =?UTF-8?q?build:=20upgrade=20`miette`=205.1.0=20?= =?UTF-8?q?=E2=86=92=207.2.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 175 ++++++++++++++++++++++++++++--------- moz-webgpu-cts/Cargo.toml | 4 +- moz-webgpu-cts/src/main.rs | 50 ++++++----- 3 files changed, 163 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c52351a..6e51aa0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -79,7 +79,7 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -89,7 +89,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -116,6 +116,12 @@ dependencies = [ "backtrace", ] +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + [[package]] name = "camino" version = "1.1.6" @@ -205,7 +211,7 @@ dependencies = [ "encode_unicode", "lazy_static", "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -361,6 +367,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "fnv" version = "1.0.7" @@ -470,7 +486,7 @@ checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ "hermit-abi", "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -524,6 +540,12 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + [[package]] name = "log" version = "0.4.21" @@ -538,15 +560,14 @@ checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "miette" -version = "5.10.0" +version = "7.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59bb584eaeeab6bd0226ccf3509a69d7936d148cf3d036ad350abe35e8c6856e" +checksum = "4edc8853320c2a0dab800fbda86253c8938f6ea88510dc92c5f1ed20e794afc1" dependencies = [ "backtrace", "backtrace-ext", - "is-terminal", + "cfg-if", "miette-derive", - "once_cell", "owo-colors", "supports-color", "supports-hyperlinks", @@ -559,9 +580,9 @@ dependencies = [ [[package]] name = "miette-derive" -version = "5.10.0" +version = "7.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c" +checksum = "dcf09caffaac8068c346b6df2a7fc27a177fd20b39421a39ce0a211bde679a6c" dependencies = [ "proc-macro2", "quote", @@ -644,9 +665,9 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "owo-colors" -version = "3.5.0" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" +checksum = "caff54706df99d2a78a5a4e3455ff45448d81ef1bb63c22cd14052ca0e993a3f" [[package]] name = "path-dsl" @@ -762,6 +783,19 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustix" +version = "0.38.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + [[package]] name = "rustversion" version = "1.0.14" @@ -869,31 +903,24 @@ dependencies = [ [[package]] name = "supports-color" -version = "2.1.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6398cde53adc3c4557306a96ce67b302968513830a77a95b2b17305d9719a89" +checksum = "9829b314621dfc575df4e409e79f9d6a66a3bd707ab73f23cb4aa3a854ac854f" dependencies = [ - "is-terminal", "is_ci", ] [[package]] name = "supports-hyperlinks" -version = "2.1.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84231692eb0d4d41e4cdd0cabfdd2e6cd9e255e65f80c9aa7c98dd502b4233d" -dependencies = [ - "is-terminal", -] +checksum = "2c0a1e5168041f5f3ff68ff7d95dcb9c8749df29f6e7e89ada40dd4c9de404ee" [[package]] name = "supports-unicode" -version = "2.1.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f850c19edd184a205e883199a261ed44471c81e39bd95b1357f5febbef00e77a" -dependencies = [ - "is-terminal", -] +checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" [[package]] name = "syn" @@ -920,7 +947,7 @@ dependencies = [ [[package]] name = "tardar" version = "0.1.0" -source = "git+https://github.com/ErichDonGubler/tardar?branch=static-diags#8dddc68b9f1ad730f3a97b1819333e2a6769ccb7" +source = "git+https://github.com/ErichDonGubler/tardar?branch=static-diags-miette-0.7#e6d2383f519030c6fe8618182995c409ed3b1383" dependencies = [ "miette", "vec1", @@ -937,19 +964,19 @@ dependencies = [ [[package]] name = "terminal_size" -version = "0.1.17" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" +checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7" dependencies = [ - "libc", - "winapi", + "rustix", + "windows-sys 0.48.0", ] [[package]] name = "textwrap" -version = "0.15.2" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7b3e525a49ec206798b40326a44121291b530c963cfb01018f63e135bac543d" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" dependencies = [ "smawk", "unicode-linebreak", @@ -1031,7 +1058,7 @@ dependencies = [ [[package]] name = "wax" version = "0.6.0" -source = "git+https://github.com/ErichDonGubler/wax?branch=static-miette-diags#b606968c386f98dba23c15f681d8afdc40142b11" +source = "git+https://github.com/ErichDonGubler/wax?branch=static-miette-diags-0.7#63542ef14cd5ac824f78cf6c2864ca4b031a0ce6" dependencies = [ "const_format", "itertools", @@ -1086,13 +1113,37 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.4", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] @@ -1101,51 +1152,93 @@ version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.4" diff --git a/moz-webgpu-cts/Cargo.toml b/moz-webgpu-cts/Cargo.toml index 718b846..7637834 100644 --- a/moz-webgpu-cts/Cargo.toml +++ b/moz-webgpu-cts/Cargo.toml @@ -22,7 +22,7 @@ itertools = "0.11.0" joinery = "3.1.0" lets_find_up = "0.0.3" log = { workspace = true } -miette = { version = "5.10.0", features = ["fancy"] } +miette = { version = "7.2.0", features = ["fancy"] } natord = "1.0.9" path-dsl = "0.6.1" rayon = "1.8.0" @@ -30,7 +30,7 @@ serde = { workspace = true, features = ["derive"] } serde_json = "1.0.107" strum = { version = "0.25.0", features = ["derive"] } thiserror = { workspace = true } -wax = { version = "0.6.0", features = ["miette"], git = "https://github.com/ErichDonGubler/wax", branch = "static-miette-diags"} +wax = { version = "0.6.0", features = ["miette"], git = "https://github.com/ErichDonGubler/wax", branch = "static-miette-diags-0.7"} whippit = { version = "0.6.2", path = "../whippit", default-features = false, features = ["serde1"] } [dev-dependencies] diff --git a/moz-webgpu-cts/src/main.rs b/moz-webgpu-cts/src/main.rs index f502eed..af4b776 100644 --- a/moz-webgpu-cts/src/main.rs +++ b/moz-webgpu-cts/src/main.rs @@ -41,7 +41,7 @@ use process_reports::{ should_update_expected::{self, ShouldUpdateExpected}, ProcessReportsArgs, }; -use wax::Glob; +use wax::{walk::Entry as _, Glob}; use whippit::{ metadata::SectionHeader, reexport::chumsky::{self, prelude::Rich}, @@ -179,7 +179,7 @@ impl FileSpec { diag.severity() .map_or(true, |sev| sev == miette::Severity::Error) }) - .map(Report::new_boxed); + .map(|diag| Report::new_boxed(diag)); for report in error_reports { eprintln!("{report:?}"); } @@ -201,26 +201,30 @@ impl FileSpec { let files = globs .iter() .flat_map(|(base_path, glob)| { - glob.walk(base_path) - .filter_map(|entry| match entry { - Ok(entry) => Some(entry.into_path()), - Err(e) => { - found_glob_walk_err = true; - let ctx_msg = if let Some(path) = e.path() { - format!( - "failed to enumerate {what} from glob `{}` at path {}", - glob, - path.display() - ) - } else { - format!("failed to enumerate {what} from glob `{glob}`") - }; - let e = Report::msg(e).wrap_err(ctx_msg); - eprintln!("{e:?}"); - None - } - }) - .collect::>() // OPT: Can we get rid of this somehow? + if let Some(glob) = glob { + glob.walk(base_path) + .filter_map(|entry| match entry { + Ok(entry) => Some(entry.into_path()), + Err(e) => { + found_glob_walk_err = true; + let ctx_msg = if let Some(path) = e.path() { + format!( + "failed to enumerate {what} from glob `{}` at path {}", + glob, + path.display() + ) + } else { + format!("failed to enumerate {what} from glob `{glob}`") + }; + let e = Report::msg(e).wrap_err(ctx_msg); + eprintln!("{e:?}"); + None + } + }) + .collect::>() // OPT: Can we get rid of this somehow? + } else { + vec![base_path.to_owned()] + } }) .collect::>(); @@ -1255,7 +1259,7 @@ fn render_metadata_parse_errors<'a>( #[label] span: SourceSpan, #[source_code] - source_code: NamedSource, + source_code: NamedSource>, inner: Rich<'static, char>, } let source_code = file_contents.clone(); From f2ea983022f3fc6817e2a1ce27b8e37d05b9fb3d Mon Sep 17 00:00:00 2001 From: Erich Gubler Date: Fri, 2 Aug 2024 20:14:35 -0400 Subject: [PATCH 6/6] WIP: feat: `aggregate-timings-from-log` subcmd. --- Cargo.lock | 152 ++++ moz-webgpu-cts/Cargo.toml | 1 + .../src/aggregate_timings_from_logs.rs | 413 +++++++++++ .../log_line_reader.rs | 688 ++++++++++++++++++ moz-webgpu-cts/src/main.rs | 26 + moz-webgpu-cts/src/wpt/path.rs | 2 +- 6 files changed, 1281 insertions(+), 1 deletion(-) create mode 100644 moz-webgpu-cts/src/aggregate_timings_from_logs.rs create mode 100644 moz-webgpu-cts/src/aggregate_timings_from_logs/log_line_reader.rs diff --git a/Cargo.lock b/Cargo.lock index 6e51aa0..78b6eb0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -44,6 +44,21 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "0.6.13" @@ -92,6 +107,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "autocfg" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" + [[package]] name = "backtrace" version = "0.3.71" @@ -122,6 +143,12 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + [[package]] name = "camino" version = "1.1.6" @@ -143,6 +170,20 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets 0.52.4", +] + [[package]] name = "chumsky" version = "1.0.0-alpha.6" @@ -234,6 +275,12 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + [[package]] name = "crossbeam-deque" version = "0.8.5" @@ -450,6 +497,29 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -516,6 +586,15 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57d8bde02bbf44a562cf068a8ff4a68842df387e302a03a4de4a57fcf82ec377" +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -609,6 +688,7 @@ name = "moz-webgpu-cts" version = "2.0.0" dependencies = [ "camino", + "chrono", "clap", "enum-map", "enumset", @@ -648,6 +728,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "object" version = "0.32.2" @@ -1055,6 +1144,60 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.58", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.58", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" + [[package]] name = "wax" version = "0.6.0" @@ -1113,6 +1256,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.4", +] + [[package]] name = "windows-sys" version = "0.48.0" diff --git a/moz-webgpu-cts/Cargo.toml b/moz-webgpu-cts/Cargo.toml index 7637834..c330d8e 100644 --- a/moz-webgpu-cts/Cargo.toml +++ b/moz-webgpu-cts/Cargo.toml @@ -12,6 +12,7 @@ dist = true [dependencies] camino = { version = "1.1.6", features = ["serde1"] } +chrono = "0.4.38" clap = { version = "4.4.2", features = ["derive"] } enum-map = { version = "2.7.3", features = ["serde"] } enumset = { version = "1.1.3", features = ["serde"] } diff --git a/moz-webgpu-cts/src/aggregate_timings_from_logs.rs b/moz-webgpu-cts/src/aggregate_timings_from_logs.rs new file mode 100644 index 0000000..91b4437 --- /dev/null +++ b/moz-webgpu-cts/src/aggregate_timings_from_logs.rs @@ -0,0 +1,413 @@ +mod log_line_reader; + +use std::{ + fs::File, + io::BufReader, + ops::ControlFlow, + path::{Path, PathBuf}, + time::Duration, +}; + +use format::lazy_format; +use indexmap::IndexMap; +use log_line_reader::{ + LogLine, LogLineKind, LogLineReader, LogLineSpans, ParseExpectedTestEndError, TestLogLine, + TestLogLineKind, TestLogLineParseError, TestLogLineParseErrorKind, TestPathParseError, + TookParseError, +}; +use miette::{IntoDiagnostic, LabeledSpan, Report, SourceSpan}; + +use crate::{ + wpt::{ + metadata::{SubtestOutcome, TestOutcome}, + path::Browser, + }, + AlreadyReportedToCommandline, +}; + +pub(crate) fn aggregate_timings_from_logs( + browser: Browser, + log_paths: Vec, +) -> Result<(), AlreadyReportedToCommandline> { + if log_paths.is_empty() { + log::error!(concat!( + "no log file(s) specified; ", + "this command doesn't make sense without them!" + )); + return Err(AlreadyReportedToCommandline); + } + + enum TestLogParserState { + ReadyForTest, + StartedTest { + test_name: LogLineSpans, + // reported_subtest: bool, + }, + } + + #[derive(Clone, Debug)] + enum TestLogEvent { + Test { + test_name: String, + outcome: TestOutcome, + duration: Duration, + subtests_noted: IndexMap, + }, + } + + // fn find(line_handler: impl FindLogLine) -> Result, E> {} + + #[derive(Clone, Copy, Debug)] + enum BufferClear { + All, + Truncate(usize), + No, + } + + trait FindLogLine { + fn handle_line( + &mut self, + buf_len_before_read: usize, + log_line: LogLine, + ) -> ControlFlow, E>>; + fn handle_eof(&mut self); + } + + // TODO: Do 'em all in parallel! + + let log_path = log_paths.first().unwrap(); + let mut reader = LogLineReader::new(browser, BufReader::new(File::open(log_path).unwrap())); + + let mut state = TestLogParserState::ReadyForTest; + let mut buf = String::with_capacity(512); + let mut errs = Vec::new(); + let mut read_line = |buf: &mut String| { + let buf_len_before_read = buf.len(); + + let line_res = reader + .next_log_line(buf, &mut |e| errs.push(e)) + .map(|res| res.into_diagnostic()); + match line_res { + Some(Ok(line)) => { + assert!(errs.is_empty()); + Ok(Some(line)) + } + Some(Err(e)) => { + for err in errs.drain(..) { + render_test_log_line_err(&log_path, &buf, err); + } + log::error!("{e}"); + buf.truncate(buf_len_before_read); + return Err(AlreadyReportedToCommandline); + } + None => { + assert!(errs.is_empty()); + Ok(None) + } + } + }; + // let mut last_test_span = None; + 'read_loop: loop { + let (started_test_span, started_test_line_num, started_test_line_start_idx_in_buf) = 'start_test: loop { + let len = buf.len(); + let line_start_idx_in_buf = len; + let LogLine { line_num, kind } = match read_line(&mut buf)? { + Some(some) => some, + None => break 'read_loop, + }; + match kind { + LogLineKind::Other => (/* s' cool */), + LogLineKind::Test(TestLogLine { timestamp, kind }) => match kind { + TestLogLineKind::StartTest { test_name } => { + break 'start_test (test_name, line_num, line_start_idx_in_buf) + } + TestLogLineKind::LeakCheck => { + log::warn!("bruh wat is happening, y u no not leakcheck (line {line_num})"); + } + TestLogLineKind::FinishTestExpected { + test_name, + outcome, + took: _, + } => { + log::error!( + "ocraps, ended a test with {outcome} with no test previously noted as started on line {line_num}: {}", + test_name.get_from(&buf) + ); + return Err(AlreadyReportedToCommandline); + } + TestLogLineKind::FinishSubtest { + test_name, + subtest_name, + outcome, + } => { + log::error!( + "ocraps, ended a subtest with {outcome} with no test previously noted as started on line {line_num}: {} | {}", + test_name.get_from(&buf), + subtest_name.get_from(&buf) + ); + return Err(AlreadyReportedToCommandline); + } + TestLogLineKind::FinishTestUnexpected { test_name, outcome } => { + log::error!( + "ocraps, ended a test with {outcome} with no test previously noted as started on line {line_num}: {}", + test_name.get_from(&buf) + ); + return Err(AlreadyReportedToCommandline); + } + TestLogLineKind::InfoTook { took } => { + log::error!("ocraps, ended _something_ with no test previously noted as started on line {line_num}"); + return Err(AlreadyReportedToCommandline); + } + }, + } + }; + + // Read zero or more subtest results until we get a test result. + let mut subtests_noted = IndexMap::new(); + let (outcome, duration) = 'end_test: loop { + let LogLine { line_num, kind } = match read_line(&mut buf)? { + Some(some) => some, + None => { + // TODO: Maybe this might happen when the task times out? + log::error!( + "ocraps, log ended before finishing test started on line {}", + started_test_line_num + ); + return Err(AlreadyReportedToCommandline); + } + }; + match kind { + LogLineKind::Other => (/* s' cool */), + LogLineKind::Test(TestLogLine { timestamp, kind }) => match kind { + TestLogLineKind::StartTest { test_name } => { + log::error!( + concat!( + "ocraps, started a test on line {} ", + "before ending the test that started on line {}: {}", + ), + line_num, + started_test_line_num, + test_name.get_from(&buf) + ); + return Err(AlreadyReportedToCommandline); + } + TestLogLineKind::LeakCheck => { + log::warn!("leakcheck on line {line_num} good? teach me plz"); + } + TestLogLineKind::FinishTestExpected { + test_name, + outcome, + took, + } => { + { + let started_test_name = started_test_span.get_from(&buf); + let finished_test_name = test_name.get_from(&buf); + if started_test_name != finished_test_name { + log::error!( + concat!( + "bruh wtf, ", + "started test {:?}, ", + "but then the end event says it's {:?}" + ), + started_test_name, + finished_test_name, + ) + } + } + break 'end_test (outcome, took); + } + TestLogLineKind::FinishSubtest { + test_name, + subtest_name, + outcome, + } => { + { + let started_test_name = started_test_span.get_from(&buf); + let finished_test_name = test_name.get_from(&buf); + if started_test_name != finished_test_name { + log::error!( + concat!( + "bruh wtf, ", + "started test {:?}, ", + "but then the end event says it's {:?}" + ), + started_test_name, + finished_test_name, + ) + } + } + let subtest_name = subtest_name.get_from(&buf).to_owned(); + subtests_noted.insert(subtest_name, outcome); + return Err(AlreadyReportedToCommandline); + } + TestLogLineKind::FinishTestUnexpected { test_name, outcome } => { + { + let started_test_name = started_test_span.get_from(&buf); + let finished_test_name = test_name.get_from(&buf); + if started_test_name != finished_test_name { + log::error!( + concat!( + "bruh wtf, ", + "started test {:?}, ", + "but then the end event says it's {:?}" + ), + started_test_name, + finished_test_name, + ) + } + } + loop { + let LogLine { line_num, kind } = match read_line(&mut buf)? { + Some(some) => some, + None => { + // TODO: Maybe this might happen when the task times out? + log::error!( + concat!( + "ocraps, log EOF reached before finishing test ", + "started on line {}", + ), + started_test_line_num + ); + return Err(AlreadyReportedToCommandline); + } + }; + match kind { + LogLineKind::Test(TestLogLine { timestamp, kind }) => match kind { + TestLogLineKind::InfoTook { took } => { + break 'end_test (outcome, took); + } + TestLogLineKind::StartTest { test_name } => todo!(), + TestLogLineKind::LeakCheck => todo!(), + TestLogLineKind::FinishTestExpected { + test_name, + outcome, + took, + } => todo!(), + TestLogLineKind::FinishSubtest { + test_name, + subtest_name, + outcome, + } => todo!(), + TestLogLineKind::FinishTestUnexpected { + test_name, + outcome, + } => todo!(), + }, + LogLineKind::Other => (), + } + } + } + TestLogLineKind::InfoTook { took } => { + log::error!( + concat!( + "ocraps, ended _something_ ", + "with no test previously noted as started on line {}" + ), + line_num + ); + return Err(AlreadyReportedToCommandline); + } + }, + } + }; + println!( + "{:?}", + TestLogEvent::Test { + test_name: started_test_span.get_from(&buf).to_owned(), + outcome, + duration, + subtests_noted, + } + ); + buf.truncate(started_test_line_start_idx_in_buf); + } + + buf.clear(); + errs.clear(); + + Ok(()) +} + +fn render_test_log_line_err(log_path: &Path, buf: &str, e: TestLogLineParseError) { + impl From for SourceSpan { + fn from(value: LogLineSpans) -> Self { + value.buf_slice_idx().into() + } + } + + let TestLogLineParseError { line_num, kind } = e; + // TODO: use `camino` paths, save everyone some pain 😭 + let log_and_line_prepend = + lazy_format!("{log_path:?}:{line_num}: failed to parse `TEST` log line: "); + let test_path_parse_labels = |err| { + let TestPathParseError { + discriminant_span, + test_path_span, + msg, + } = err; + vec![ + LabeledSpan::at( + discriminant_span, + "indicates that the test path will be started", + ), + LabeledSpan::new_primary_with_span(Some(msg), test_path_span), + ] + }; + let diagnostic = match kind { + TestLogLineParseErrorKind::ParseTimestamp { source, span } => { + miette::diagnostic!( + labels = vec![LabeledSpan::new_primary_with_span(None, span)], + "{log_and_line_prepend}{source}" + ) + } + TestLogLineParseErrorKind::UnrecognizedDiscriminant { span } => { + let discriminant = span.get_from(&buf); + miette::diagnostic!( + labels = vec![LabeledSpan::new_primary_with_span(None, span)], + "{log_and_line_prepend}unrecognized discriminant {discriminant:?}" + ) + } + TestLogLineParseErrorKind::ParseStartTestPath(inner) => miette::diagnostic!( + labels = test_path_parse_labels(inner), + "{log_and_line_prepend}failed to parse `START`ed test path" + ), + TestLogLineParseErrorKind::ParseExpectedTestEnd(e) => match e { + ParseExpectedTestEndError::SplitDivider { span } => { + miette::diagnostic!( + labels = vec![LabeledSpan::new_primary_with_span(None, span)], + // TODO: share constant, probably via `Display`? + "{log_and_line_prepend}failed to find dividing split (` | `) between a presumed test path and `took` duration", + ) + } + ParseExpectedTestEndError::ParseTestPath { inner } => miette::diagnostic!( + labels = test_path_parse_labels(inner), + "{log_and_line_prepend}failed to parse `OK`'d test path" + ), + ParseExpectedTestEndError::ParseTook { inner } => { + let log_and_line_prepend = lazy_format!("{log_and_line_prepend}`took` duration "); + match inner { + TookParseError::ParseMillis { span, source } => miette::diagnostic!( + labels = vec![LabeledSpan::new_primary_with_span( + Some(source.to_string()), + span + )], + "{log_and_line_prepend}had invalid milliseconds count" + ), + TookParseError::ParseUnit { expected_ms_span } => miette::diagnostic!( + labels = vec![LabeledSpan::new_primary_with_span( + Some("expected here".to_owned()), + expected_ms_span + )], + "{log_and_line_prepend}of the form `took ms` not found" + ), + } + } + }, + }.with_help(concat!( + "If this isn't a malformed edit of yours, it's likely a bug in `", + env!("CARGO_BIN_NAME"), + "`. You should file an issue upstream!" + )); + let diagnostic = Report::new(diagnostic).with_source_code(buf.to_owned()); + eprintln!("{diagnostic:?}") +} diff --git a/moz-webgpu-cts/src/aggregate_timings_from_logs/log_line_reader.rs b/moz-webgpu-cts/src/aggregate_timings_from_logs/log_line_reader.rs new file mode 100644 index 0000000..f9c4e6b --- /dev/null +++ b/moz-webgpu-cts/src/aggregate_timings_from_logs/log_line_reader.rs @@ -0,0 +1,688 @@ +use std::{ + io::{self, BufRead}, + ops::Range, + time::Duration, +}; + +use chrono::{DateTime, FixedOffset}; +use miette::Diagnostic; +use whippit::reexport::chumsky::{ + error::{EmptyErr, Simple}, + extra::{self, Full, ParserExtra}, + input::{Input, StrInput}, + primitive::{any, choice, just}, + span::SimpleSpan, + text::{ascii, digits}, + IterParser, Parser, +}; + +use crate::wpt::{ + metadata::{SubtestOutcome, TestOutcome}, + path::{Browser, TestEntryPath}, +}; + +pub(super) struct LogLineReader { + browser: Browser, + next_line_idx: u64, + reader: R, +} + +impl LogLineReader { + pub fn new(browser: Browser, reader: R) -> Self { + Self { + browser, + next_line_idx: 1, + reader, + } + } +} + +impl LogLineReader +where + R: BufRead, +{ + pub fn next_log_line<'a>( + &mut self, + buf: &'a mut String, + test_log_line_parse_error_sink: &mut dyn FnMut(TestLogLineParseError), + ) -> Option> { + let line_offset_in_buf = buf.len(); + let mut should_keep_line = false; + let ret = self.read_line(buf)?.and_then(|(line, line_idx)| { + let (res, saved_spans) = classify_log_line( + self.browser, + line_idx, + line, + line_offset_in_buf, + test_log_line_parse_error_sink, + ); + should_keep_line = saved_spans; + let kind = res?; + Ok(LogLine { + line_num: line_idx, + kind, + }) + }); + if !should_keep_line { + buf.truncate(line_offset_in_buf); + } + Some(ret) + } + + fn read_line<'a>( + &mut self, + buf: &'a mut String, + ) -> Option> { + let Self { + next_line_idx, + reader, + .. + } = self; + + let start = buf.len(); + match reader + .read_line(buf) + .map_err(|source| LogLineReadError::Io { source }) + { + Ok(0) => None, + Ok(bytes_read) => { + let mut line = &buf[start..buf.len()]; + line = line.strip_suffix('\n').unwrap_or(line); + + let extracted = match bytes_read { + 0 => None, + _ => Some(Ok((line, *next_line_idx))), + }; + *next_line_idx = next_line_idx.checked_add(1).unwrap(); + extracted + } + Err(e) => Some(Err(e)), + } + } +} + +#[derive(Debug, Diagnostic, thiserror::Error)] +#[error("failed to parse log line")] +pub(super) enum LogLineReadError { + Io { + source: io::Error, + }, + ClassifyTestLogLine { + #[from] + source: CheckErrorSink, + }, +} + +#[derive(Clone, Debug)] +pub(super) struct LogLine { + pub line_num: u64, + pub kind: LogLineKind, +} + +#[derive(Clone, Debug)] +#[cfg_attr(test, derive(Eq, PartialEq))] +pub(super) enum LogLineKind { + Test(TestLogLine), + Other, +} + +#[derive(Clone, Debug)] +#[cfg_attr(test, derive(Eq, PartialEq))] +pub(super) struct TestLogLine { + pub timestamp: DateTime, + pub kind: TestLogLineKind, +} + +#[derive(Clone, Debug)] +#[cfg_attr(test, derive(Eq, PartialEq))] +pub(super) struct LogLineSpans { + offset_from_start_of_line: usize, + offset_in_buf: usize, + length: usize, +} + +impl LogLineSpans { + pub fn buf_slice_idx(&self) -> Range { + let &Self { + offset_in_buf, + length, + .. + } = self; + offset_in_buf..(offset_in_buf + length) + } + + #[track_caller] + pub fn get_from<'a>(&self, s: &'a str) -> &'a str { + s.get(self.buf_slice_idx()).unwrap() + } + + #[track_caller] + pub fn truncate_before_in(&self, s: &mut String) { + s.truncate(self.offset_in_buf) + } +} + +#[derive(Clone, Debug)] +#[cfg_attr(test, derive(Eq, PartialEq))] +pub(super) enum TestLogLineKind { + StartTest { + test_name: LogLineSpans, + }, + LeakCheck, + FinishTestExpected { + test_name: LogLineSpans, + outcome: TestOutcome, + took: Duration, + }, + FinishSubtest { + test_name: LogLineSpans, + subtest_name: LogLineSpans, + outcome: SubtestOutcome, + }, + FinishTestUnexpected { + test_name: LogLineSpans, + outcome: TestOutcome, + }, + InfoTook { + took: Duration, + }, +} + +enum TestLogLineDiscriminant { + Start, + Info, + Expected(Outcome), + Unexpected(Outcome), +} + +impl TestLogLineDiscriminant { + pub fn new(s: &str) -> Option { + let outcome = Outcome::from_ambiguous::<_, extra::Default>; + + match s { + "START" => Some(Self::Start), + "INFO" => Some(Self::Info), + _ => { + if let Some(s) = s.strip_prefix("UNEXPECTED-") { + outcome(s).map(Self::Unexpected) + } else if let Some(s) = s.strip_prefix("KNOWN-INTERMITTENT-") { + outcome(s).map(Self::Expected) + } else { + outcome(s).map(Self::Expected) + } + } + } + } +} + +/// The combined sets of [`TestOutcome`] and [`SubtestOutcome`]. Used in [`classify_log_line`] +/// while the set to which an outcome should belong is ambiguous. +#[derive(Clone, Copy, Debug)] +enum Outcome { + Pass, + Fail, + Skip, + Crash, + Timeout, + Error, + Ok, + NotRun, +} + +impl Outcome { + pub fn from_ambiguous<'a, I, E>(input: I) -> Option + where + I: Input<'a, Token = char> + StrInput<'a, char>, + E: ParserExtra<'a, I>, + E::Context: Default, + E::State: Default, + { + choice(( + TestOutcome::parser::().map(|o| match o { + TestOutcome::Pass => Self::Pass, + TestOutcome::Fail => Self::Fail, + TestOutcome::Timeout => Self::Timeout, + TestOutcome::Crash => Self::Crash, + TestOutcome::Error => Self::Error, + TestOutcome::Skip => Self::Skip, + TestOutcome::Ok => Self::Ok, + }), + SubtestOutcome::parser::().map(|o| match o { + SubtestOutcome::Pass => Self::Pass, + SubtestOutcome::Fail => Self::Fail, + SubtestOutcome::Timeout => Self::Timeout, + SubtestOutcome::NotRun => Self::NotRun, + }), + )) + .parse(input) + .into_output() + } + + pub fn to_test_outcome(self) -> Option { + Some(match self { + Self::Pass => TestOutcome::Pass, + Self::Fail => TestOutcome::Fail, + Self::Skip => TestOutcome::Skip, + Self::Crash => TestOutcome::Crash, + Self::Timeout => TestOutcome::Timeout, + Self::Error => TestOutcome::Error, + Self::Ok => TestOutcome::Ok, + Self::NotRun => return None, + }) + } + + pub fn to_subtest_outcome(self) -> Option { + Some(match self { + Self::Pass => SubtestOutcome::Pass, + Self::Fail => SubtestOutcome::Fail, + Self::Timeout => SubtestOutcome::Timeout, + Self::NotRun => SubtestOutcome::NotRun, + Self::Crash | Self::Skip | Self::Error | Self::Ok => return None, + }) + } +} + +#[derive(Clone, Debug)] +#[cfg_attr(test, derive(Eq, PartialEq))] +pub(super) struct TestLogLineParseError { + pub line_num: u64, + pub kind: TestLogLineParseErrorKind, +} + +#[derive(Clone, Debug)] +#[cfg_attr(test, derive(Eq, PartialEq))] +pub(super) enum TestLogLineParseErrorKind { + ParseTimestamp { + source: chrono::ParseError, + span: LogLineSpans, + }, + UnrecognizedDiscriminant { + span: LogLineSpans, + }, + ParseStartTestPath(TestPathParseError), + ParseExpectedTestEnd(ParseExpectedTestEndError), +} + +#[derive(Clone, Debug)] +#[cfg_attr(test, derive(Eq, PartialEq))] +pub(super) enum ParseExpectedTestEndError { + SplitDivider { span: LogLineSpans }, + ParseTestPath { inner: TestPathParseError }, + ParseTook { inner: TookParseError }, +} + +#[derive(Clone, Debug)] +#[cfg_attr(test, derive(Eq, PartialEq))] +pub(super) struct TestPathParseError { + pub discriminant_span: LogLineSpans, + pub test_path_span: LogLineSpans, + pub msg: String, +} + +#[derive(Clone, Debug)] +#[cfg_attr(test, derive(Eq, PartialEq))] +pub(super) enum TookParseError { + ParseUnit { + expected_ms_span: LogLineSpans, + }, + ParseMillis { + span: LogLineSpans, + source: std::num::ParseIntError, + }, +} + +#[derive(Debug, Eq, PartialEq, thiserror::Error)] +#[error("see above errors for more details")] +pub(super) struct CheckErrorSink; + +fn classify_log_line<'a>( + browser: Browser, + line_num: u64, + s: &'a str, + slice_start: usize, // TODO: maybe confusing with `s`' start? + unrecoverable_err_sink: &'a mut dyn FnMut(TestLogLineParseError), +) -> (Result, bool) { + fn mozlog_test_message_section_divider<'a, I, E>() -> impl Parser<'a, I, (), E> + Copy + Clone + where + I: Input<'a, Token = char>, + E: ParserExtra<'a, I>, + { + just(" | ").to(()) + } + + let test_log_line = { + // i.e., something of the form `[task 2024-08-02T22:11:54.874Z] ` + let first_log_layer = any::< + &str, + Full, (&mut bool, &mut dyn FnMut(TestLogLineParseError)), ()>, + >() // TODO: ew, dis bad, better plz? + .and_is(just("]").not()) + .repeated() + .to_slice() + .map_with(|raw, e| (raw, e.span())) + .delimited_by(just("[task "), just("] ")); + + // i.e., something of the form `22:11:54 INFO - ` + let second_log_layer = any() + .repeated() + .exactly(2) + .separated_by(just(":")) + .exactly(3) + .then(just(" INFO - ")) + .ignored(); + + let mozlog_test_message_layer = just("TEST-") + .ignore_then(ascii::ident().map_with(|ident, e| (ident, e.span()))) + .then_ignore(mozlog_test_message_section_divider()); + + first_log_layer + .then_ignore(second_log_layer) + .then(mozlog_test_message_layer) + .then( + any() + .repeated() + .to_slice() + .map_with(|rest, e| (rest, e.span())), + ) + .map_with( + move |( + ((raw_timestamp, timestamp_span), (discriminant, discriminant_span)), + (rest, rest_span), + ), + e| { + let (should_save_spans, unrecoverable_err_sink) = e.state(); + + let mut save_span = |simple_span: SimpleSpan| { + **should_save_spans = true; + LogLineSpans { + offset_from_start_of_line: simple_span.start, + offset_in_buf: slice_start + simple_span.start, + length: simple_span.end - simple_span.start, + } + }; + + let mut unrecoverable_err_sink = + |kind| unrecoverable_err_sink(TestLogLineParseError { line_num, kind }); + + let timestamp = match DateTime::parse_from_rfc3339(raw_timestamp) { + Ok(ok) => Some(ok), + Err(source) => { + unrecoverable_err_sink(TestLogLineParseErrorKind::ParseTimestamp { + span: save_span(timestamp_span), + source, + }); + None + } + }; + + let kind = 'kind: { + match TestLogLineDiscriminant::new(discriminant) { + Some(TestLogLineDiscriminant::Start) => { + if let Err(e) = TestEntryPath::from_execution_report(browser, rest) + { + unrecoverable_err_sink( + TestLogLineParseErrorKind::ParseStartTestPath( + TestPathParseError { + discriminant_span: save_span(discriminant_span), + test_path_span: save_span(rest_span), + msg: e.to_string(), + }, + ), + ); + None + } else { + Some(TestLogLineKind::StartTest { + test_name: save_span(rest_span), + }) + } + } + Some(TestLogLineDiscriminant::Info) => { + if rest.starts_with("leakcheck") { + Some(TestLogLineKind::LeakCheck) + } else { + todo!("bruh IDK what to do with `INFO` lines yet") + } + } + Some(TestLogLineDiscriminant::Expected(outcome)) => { + if rest.starts_with("leakcheck") { + break 'kind Some(TestLogLineKind::LeakCheck); + } + + let rest = rest.map_span(|span| { + SimpleSpan::new( + rest_span.start + span.start, + rest_span.start + span.end, + ) + }); + let [(test_path, test_path_span), (took_section, took_span)] = { + let res = any::<_, Full, &str, ()>>() + .and_is(mozlog_test_message_section_divider().not()) + .repeated() + .to_slice() + .map_with(|section, e| (section, e.span())) + .separated_by(mozlog_test_message_section_divider()) + .collect_exactly() + .parse(rest) + .into_result() + .map_err(|_e| { + unrecoverable_err_sink( + TestLogLineParseErrorKind::ParseExpectedTestEnd( + ParseExpectedTestEndError::SplitDivider { + span: save_span(rest_span), + }, + ), + ); + CheckErrorSink + }); + match res { + Ok(ok) => ok, + Err(CheckErrorSink) => break 'kind None, + } + }; + + if let Err(e) = + TestEntryPath::from_execution_report(browser, test_path) + { + unrecoverable_err_sink( + TestLogLineParseErrorKind::ParseExpectedTestEnd( + ParseExpectedTestEndError::ParseTestPath { + inner: TestPathParseError { + discriminant_span: save_span(discriminant_span), + test_path_span: save_span(test_path_span), + msg: e.to_string(), + }, + }, + ), + ); + break 'kind None; + } + + let took_section = took_section.map_span(|span| { + SimpleSpan::new( + took_span.start + span.start, + took_span.start + span.end, + ) + }); + + let took = { + let took_res = digits::<_, _, Full>(10) + .to_slice() + .map_with(|millis, e| (millis, e.span())) + .delimited_by(just("took "), just("ms")) + .parse(took_section) + .into_result() + .map_err(|_e| TookParseError::ParseUnit { + expected_ms_span: save_span(took_span), + }) + .and_then(|(millis, millis_span)| { + millis.parse().map(Duration::from_millis).map_err( + |source| TookParseError::ParseMillis { + span: save_span(millis_span), + source, + }, + ) + }) + .map_err(|inner| { + TestLogLineParseErrorKind::ParseExpectedTestEnd( + ParseExpectedTestEndError::ParseTook { inner }, + ) + }); + match took_res { + Ok(some) => some, + Err(e) => { + unrecoverable_err_sink(e); + break 'kind None; + } + } + }; + + let outcome = match outcome.to_test_outcome() { + Some(some) => some, + None => todo!("bruh IDK what to do with {outcome:?} yet"), + }; + + Some(TestLogLineKind::FinishTestExpected { + test_name: save_span(test_path_span), + outcome, + took, + }) + } + Some(TestLogLineDiscriminant::Unexpected(outcome)) => { + todo!("haven't done `UNEXPECTED` stuff yet, soz") + } + None => { + unrecoverable_err_sink( + TestLogLineParseErrorKind::UnrecognizedDiscriminant { + span: save_span(discriminant_span.clone()), + }, + ); + None + } + } + }; + match (timestamp, kind) { + (Some(timestamp), Some(kind)) => { + Ok(LogLineKind::Test(TestLogLine { timestamp, kind })) + } + _ => Err(CheckErrorSink), + } + }, + ) + }; + + let mut should_save_spans = false; + let res = test_log_line + .parse_with_state(s, &mut (&mut should_save_spans, unrecoverable_err_sink)) + .into_output() + .unwrap_or(Ok(LogLineKind::Other)); + (res, should_save_spans) +} + +#[test] +fn classify_good_lines() { + macro_rules! assert_good_parse_eq { + ($line:expr, $should_save_spans:expr, $expected:expr) => { + let mut errs = vec![]; + let res = classify_log_line(Browser::Firefox, 0, $line, 0, &mut |e| errs.push(e)); + if !errs.is_empty() { + for err in &errs { + eprintln!("got unexpected test log line error: {err:#?}"); + } + } + assert_eq!(res, (Ok($expected), $should_save_spans)); + assert!(errs.is_empty()); + }; + } + + let line = "[task 2024-08-02T22:11:54.874Z] 22:11:54 INFO - TEST-START | /_mozilla/webgpu/cts/webgpu/shader/validation/decl/var/cts.https.html?q=webgpu:shader,validation,decl,var:initializer_kind:*"; + assert_good_parse_eq!( + line, + true, + LogLineKind::Test(TestLogLine { + timestamp: DateTime::parse_from_rfc3339("2024-08-02T22:11:54.874Z").unwrap(), + kind: TestLogLineKind::StartTest { + test_name: LogLineSpans { + offset_from_start_of_line: 65, + offset_in_buf: 65, + length: 124, + } + } + }) + ); + + let line = "[task 2024-08-02T22:17:15.803Z] 22:17:15 INFO - TEST-OK | /_mozilla/webgpu/cts/webgpu/api/operation/shader_module/compilation_info/cts.https.html?q=webgpu:api,operation,shader_module,compilation_info:getCompilationInfo_returns:* | took 9443ms"; + assert_good_parse_eq!( + line, + true, + LogLineKind::Test(TestLogLine { + timestamp: DateTime::parse_from_rfc3339("2024-08-02T22:17:15.803Z").unwrap(), + kind: TestLogLineKind::FinishTestExpected { + test_name: LogLineSpans { + offset_from_start_of_line: 62, + offset_in_buf: 62, + length: 170, + }, + outcome: TestOutcome::Ok, + took: Duration::from_millis(9443), + }, + }) + ); +} + +#[test] +fn classify_bad_lines() { + let mut errs = Vec::new(); + macro_rules! assert_errs { + ($line:expr, $should_save_spans:expr, $errs:expr) => { + errs.clear(); + assert_eq!( + classify_log_line(Browser::Firefox, 0, $line, 0, &mut |e| errs.push(e)), + (Err(CheckErrorSink), $should_save_spans) + ); + assert_eq!(errs, $errs); + }; + } + + let line = + "[task 2024-08-02T22:11:54.874Z] 22:11:54 INFO - TEST-DERP | /valid/test/path.https.html"; + assert_errs!( + line, + true, + vec![TestLogLineParseError { + line_num: 0, + kind: TestLogLineParseErrorKind::UnrecognizedDiscriminant { + span: LogLineSpans { + offset_from_start_of_line: 57, + offset_in_buf: 57, + length: 4, + } + } + }] + ); + + let line = + "[task 2024-08-02T22:11:54.874Z] 22:11:54 INFO - TEST-START | bruh idk this ain't valid"; + assert_errs!( + line, + true, + vec![TestLogLineParseError { + line_num: 0, + kind: TestLogLineParseErrorKind::ParseStartTestPath(TestPathParseError { + discriminant_span: LogLineSpans { + offset_from_start_of_line: 57, + offset_in_buf: 57, + length: 5, + }, + test_path_span: LogLineSpans { + offset_from_start_of_line: 65, + offset_in_buf: 65, + length: 25, + }, + msg: crate::wpt::path::ExecutionReportPathError { + test_url_path: "bruh idk this ain't valid" + } + .to_string() + }) + }] + ); +} diff --git a/moz-webgpu-cts/src/main.rs b/moz-webgpu-cts/src/main.rs index af4b776..8b49544 100644 --- a/moz-webgpu-cts/src/main.rs +++ b/moz-webgpu-cts/src/main.rs @@ -1,3 +1,4 @@ +mod aggregate_timings_from_logs; mod process_reports; mod report; mod wpt; @@ -120,6 +121,11 @@ enum Subcommand { #[clap(value_enum, long)] implementation_status: Vec, }, + AggregateTimingsFromLogs { + log_paths: Vec, + #[clap(long = "glob", value_name = "LOG_GLOB")] + log_globs: Vec, + }, /// Parse test metadata, apply automated fixups, and re-emit it in normalized form. #[clap(name = "fixup", alias = "fmt")] Fixup, @@ -408,6 +414,26 @@ fn run(cli: Cli) -> ExitCode { Err(AlreadyReportedToCommandline) => ExitCode::FAILURE, } } + Subcommand::AggregateTimingsFromLogs { + log_paths, + log_globs, + } => { + let log_paths_res = FileSpec { + paths: log_paths, + globs: log_globs, + } + .into_paths("log file(s)"); + + let log_paths = match log_paths_res { + Ok(ok) => ok, + Err(AlreadyReportedToCommandline) => return ExitCode::FAILURE, + }; + + match aggregate_timings_from_logs::aggregate_timings_from_logs(browser, log_paths) { + Ok(()) => ExitCode::SUCCESS, + Err(AlreadyReportedToCommandline) => ExitCode::FAILURE, + } + } Subcommand::Fixup => { log::info!("fixing up metadata in-place…"); let err_found = read_and_parse_all_metadata(browser, &checkout) diff --git a/moz-webgpu-cts/src/wpt/path.rs b/moz-webgpu-cts/src/wpt/path.rs index 7748fb9..41bdefb 100644 --- a/moz-webgpu-cts/src/wpt/path.rs +++ b/moz-webgpu-cts/src/wpt/path.rs @@ -447,7 +447,7 @@ impl<'a> TestEntryPath<'a> { /// An error encountered during [`TestEntryPath::from_execution_report`]. #[derive(Debug)] pub struct ExecutionReportPathError<'a> { - test_url_path: &'a str, + pub(crate) test_url_path: &'a str, } impl Display for ExecutionReportPathError<'_> {