diff --git a/lib/bolero-engine/src/target_location.rs b/lib/bolero-engine/src/target_location.rs index 54af956..533fb0d 100644 --- a/lib/bolero-engine/src/target_location.rs +++ b/lib/bolero-engine/src/target_location.rs @@ -175,7 +175,7 @@ impl TargetLocation { components.join("__") } - fn item_path(&self) -> String { + pub fn item_path(&self) -> String { Self::format_symbol_name(self.item_path) } diff --git a/lib/bolero/src/test/mod.rs b/lib/bolero/src/test/mod.rs index f832e49..d2a5f50 100644 --- a/lib/bolero/src/test/mod.rs +++ b/lib/bolero/src/test/mod.rs @@ -10,6 +10,7 @@ use std::path::PathBuf; type ExhastiveDriver = Box>; mod input; +mod outcome; mod report; /// Engine implementation which mimics Rust's default test @@ -352,17 +353,23 @@ impl TestEngine { report.spawn_timer(); } + let mut outcome = outcome::Outcome::new(&self.location, start_time); + bolero_engine::panic::set_hook(); bolero_engine::panic::forward_panic(false); for input in tests { if let Some(test_time) = test_time { if start_time.elapsed() > test_time { + outcome.on_exit(outcome::ExitReason::MaxDurationExceeded { + limit: test_time, + default: self.rng_cfg.test_time.is_none(), + }); break; } } - progress(); + outcome.on_named_test(&input.data); match testfn(&mut state, &input.data) { Ok(is_valid) => { @@ -370,6 +377,8 @@ impl TestEngine { } Err(err) => { bolero_engine::panic::forward_panic(true); + outcome.on_exit(outcome::ExitReason::TestFailure); + drop(outcome); eprintln!("{}", err); panic!("test failed"); } @@ -393,13 +402,21 @@ impl TestEngine { // when running exhaustive tests, it's nice to have the progress displayed report.spawn_timer(); + let mut outcome = outcome::Outcome::new(&self.location, start_time); + while driver.step().is_continue() { if let Some(test_time) = test_time { if start_time.elapsed() > test_time { + outcome.on_exit(outcome::ExitReason::MaxDurationExceeded { + limit: test_time, + default: false, + }); break; } } + outcome.on_exhaustive_input(); + let (drvr, result) = testfn(driver, &mut state); driver = drvr; @@ -410,6 +427,8 @@ impl TestEngine { } Err(error) => { bolero_engine::panic::forward_panic(true); + outcome.on_exit(outcome::ExitReason::TestFailure); + drop(outcome); eprintln!("{error}"); panic!("test failed"); } @@ -444,13 +463,3 @@ impl bolero_engine::ScopedEngine for TestEngine { bolero_engine::panic::forward_panic(true); } } - -fn progress() { - if cfg!(miri) { - use std::io::{stderr, Write}; - - // miri doesn't capture explicit writes to stderr - #[allow(clippy::explicit_write)] - let _ = write!(stderr(), "."); - } -} diff --git a/lib/bolero/src/test/outcome.rs b/lib/bolero/src/test/outcome.rs new file mode 100644 index 0000000..34e5057 --- /dev/null +++ b/lib/bolero/src/test/outcome.rs @@ -0,0 +1,130 @@ +use bolero_engine::TargetLocation; +use core::{fmt, time::Duration}; +use std::time::Instant; + +pub enum ExitReason { + MaxDurationExceeded { limit: Duration, default: bool }, + TestFailure, +} + +impl fmt::Display for ExitReason { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ExitReason::MaxDurationExceeded { limit, default } => { + write!( + f, + "max duration ({:?}{}) exceeded", + limit, + if *default { " - default" } else { "" } + ) + } + ExitReason::TestFailure => write!(f, "test failure"), + } + } +} + +pub struct Outcome<'a> { + location: &'a TargetLocation, + start_time: Instant, + corpus_input: u64, + rng_input: u64, + exhaustive_input: u64, + total: u64, + exit_reason: Option, +} + +impl fmt::Display for Outcome<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let runtime = self.start_time.elapsed(); + + if let Some(name) = self.location.test_name.as_ref() { + write!(f, "test {name} ...\t")?; + } else { + write!(f, "test {} ...\t", self.location.item_path())?; + } + + write!(f, "run time: {runtime:?} | ")?; + + let mut ips = self.total as f64 / runtime.as_secs_f64(); + if ips > 10.0 { + ips = ips.round(); + write!(f, "iterations/s: {ips}")?; + } else { + write!(f, "iterations/s: {ips:0.2}")?; + } + + for (label, count) in [ + ("corpus inputs", self.corpus_input), + ("rng inputs", self.rng_input), + ("exhaustive inputs", self.exhaustive_input), + ] { + if count > 0 { + write!(f, " | {label}: {count}")?; + } + } + + if let Some(reason) = &self.exit_reason { + write!(f, " | exit reason: {}", reason)?; + } + + Ok(()) + } +} + +impl<'a> Outcome<'a> { + pub fn new(location: &'a TargetLocation, start_time: Instant) -> Self { + Self { + location, + start_time, + corpus_input: 0, + rng_input: 0, + exhaustive_input: 0, + total: 0, + exit_reason: None, + } + } + + pub fn on_named_test(&mut self, test: &super::input::Test) { + match test { + super::input::Test::Rng(_) => self.on_rng_input(), + super::input::Test::File(_) => self.on_corpus_input(), + } + } + + pub fn on_corpus_input(&mut self) { + progress(); + self.corpus_input += 1; + self.total += 1; + } + + pub fn on_rng_input(&mut self) { + progress(); + self.rng_input += 1; + self.total += 1; + } + + pub fn on_exhaustive_input(&mut self) { + self.exhaustive_input += 1; + self.total += 1; + } + + pub fn on_exit(&mut self, reason: ExitReason) { + self.exit_reason = Some(reason); + } +} + +impl Drop for Outcome<'_> { + fn drop(&mut self) { + eprintln!("{}", self.to_string()); + } +} + +fn progress() { + if cfg!(miri) { + use std::io::{stderr, Write}; + + // miri doesn't capture explicit writes to stderr + #[allow(clippy::explicit_write)] + let _ = write!(stderr(), "."); + } +}