From c886ac2ffde88c24e85ad6b3d53c7e4bce23a4b8 Mon Sep 17 00:00:00 2001 From: Erich Gubler Date: Fri, 27 Oct 2023 16:06:32 -0400 Subject: [PATCH] refactor(cli): `process-reports`: use `rayon` and channels to parallelize report parsing --- Cargo.lock | 75 ++++++++++++++++++++++++++++++++++++++ moz-webgpu-cts/Cargo.toml | 1 + moz-webgpu-cts/src/main.rs | 56 +++++++++++++++------------- 3 files changed, 106 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6c527f4..aa09dca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,6 +91,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + [[package]] name = "backtrace" version = "0.3.69" @@ -233,6 +239,39 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + [[package]] name = "darling" version = "0.20.3" @@ -516,6 +555,15 @@ version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + [[package]] name = "miette" version = "5.10.0" @@ -581,6 +629,7 @@ dependencies = [ "miette", "natord", "path-dsl", + "rayon", "regex", "serde", "serde_json", @@ -669,6 +718,26 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "regex" version = "1.9.5" @@ -738,6 +807,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "serde" version = "1.0.188" diff --git a/moz-webgpu-cts/Cargo.toml b/moz-webgpu-cts/Cargo.toml index 2a875e9..d594129 100644 --- a/moz-webgpu-cts/Cargo.toml +++ b/moz-webgpu-cts/Cargo.toml @@ -21,6 +21,7 @@ log = { workspace = true } miette = { version = "5.10.0", features = ["fancy"] } natord = "1.0.9" path-dsl = "0.6.1" +rayon = "1.8.0" regex = "1.9.5" serde = { version = "1.0.188", features = ["derive"] } serde_json = "1.0.107" diff --git a/moz-webgpu-cts/src/main.rs b/moz-webgpu-cts/src/main.rs index b182ff0..5200eb5 100644 --- a/moz-webgpu-cts/src/main.rs +++ b/moz-webgpu-cts/src/main.rs @@ -22,7 +22,7 @@ use std::{ io::{self, BufReader, BufWriter}, path::{Path, PathBuf}, process::ExitCode, - sync::Arc, + sync::{mpsc::channel, Arc}, }; use clap::Parser; @@ -31,7 +31,7 @@ use format::lazy_format; use indexmap::{IndexMap, IndexSet}; use miette::{miette, Diagnostic, IntoDiagnostic, NamedSource, Report, SourceSpan, WrapErr}; use path_dsl::path; - +use rayon::prelude::{IntoParallelIterator, ParallelIterator}; use regex::Regex; use wax::Glob; use whippit::{metadata::SectionHeader, reexport::chumsky::prelude::Rich}; @@ -344,30 +344,34 @@ fn run(cli: Cli) -> ExitCode { log::info!("gathering reported test outcomes for reconciliation with metadata…"); - let exec_reports_iter = exec_report_paths.iter().map(|path| { - fs::File::open(path) - .map(BufReader::new) - .map_err(Report::msg) - .wrap_err("failed to open file") - .and_then(|reader| { - serde_json::from_reader::<_, ExecutionReport>(reader) - .into_diagnostic() - .wrap_err("failed to parse JSON") - }) - .map(|parsed| (path, parsed)) - .wrap_err_with(|| { - format!( - "failed to read WPT execution report from {}", - path.display() - ) - }) - .map_err(|e| { - log::error!("{e:?}"); - AlreadyReportedToCommandline - }) - }); - - for res in exec_reports_iter { + let (exec_reports_sender, exec_reports_receiver) = channel(); + exec_report_paths + .into_par_iter() + .for_each_with(exec_reports_sender, |sender, path| { + let res = fs::File::open(&path) + .map(BufReader::new) + .map_err(Report::msg) + .wrap_err("failed to open file") + .and_then(|reader| { + serde_json::from_reader::<_, ExecutionReport>(reader) + .into_diagnostic() + .wrap_err("failed to parse JSON") + }) + .wrap_err_with(|| { + format!( + "failed to read WPT execution report from {}", + path.display() + ) + }) + .map(|parsed| (path, parsed)) + .map_err(|e| { + log::error!("{e:?}"); + AlreadyReportedToCommandline + }); + let _ = sender.send(res); + }); + + for res in exec_reports_receiver { let (_path, exec_report) = match res { Ok(ok) => ok, Err(AlreadyReportedToCommandline) => return ExitCode::FAILURE,