windsock: nextest compatibility

.github/workflows/build_and_test.yaml

@@ -71,7 +71,7 @@ jobs:
  - name: Build tests
  run: |
  cargo test --doc ${{ matrix.cargo_flags }} --all-features -- --show-output --nocapture
- cargo nextest archive --archive-file nextest-${{ matrix.profile }}.tar.zst ${{ matrix.cargo_flags }} --all-features
+ cargo nextest archive --archive-file nextest-${{ matrix.profile }}.tar.zst ${{ matrix.cargo_flags }} --all-features --all-targets
  - name: Upload built tests to workflow
  uses: actions/upload-artifact@v3
  with:

.github/workflows/windsock_benches.yaml

@@ -32,7 +32,7 @@ jobs:
  save-if: ${{ github.ref == 'refs/head/main' }}
  - name: Ensure that custom benches run
  run: |
- cargo windsock --bench-length-seconds 5 --operations-per-second 100
+ # run some extra cases that arent handled by nextest
  cargo windsock --bench-length-seconds 5 --operations-per-second 100 --profilers flamegraph --name cassandra,compression=none,driver=scylla,operation=read_i64,protocol=v4,shotover=standard,topology=single
  cargo windsock --bench-length-seconds 5 --operations-per-second 100 --profilers sys_monitor --name kafka,shotover=standard,size=1B,topology=single
 

windsock/src/cli.rs

@@ -1,3 +1,4 @@
+use anyhow::{anyhow, Error};
 use clap::Parser;
 
 const ABOUT: &str = r#"Bench Names:
@@ -110,4 +111,51 @@ pub struct Args {
  /// Not for human use. Call this from your bench orchestration method to launch your bencher.
  #[clap(long, verbatim_doc_comment)]
  pub internal_run: Option<String>,
+
+ /// Not for human use. For nextest compatibility only.
+ #[clap(long, verbatim_doc_comment)]
+ format: Option<NextestFormat>,
+
+ /// Not for human use. For nextest compatibility only.
+ #[clap(long, verbatim_doc_comment)]
+ ignored: bool,
+
+ /// Not for human use. For nextest compatibility only.
+ #[clap(long, verbatim_doc_comment)]
+ exact: bool,
+
+ /// Not for human use. For nextest compatibility only.
+ #[clap(long, verbatim_doc_comment)]
+ nocapture: bool,
+}
+
+#[derive(clap::ValueEnum, Clone, Copy)]
+enum NextestFormat {
+ Terse,
+}
+
+impl Args {
+ pub fn nextest_list_all(&self) -> bool {
+ self.list && matches!(&self.format, Some(NextestFormat::Terse)) && !self.ignored
+ }
+
+ pub fn nextest_list_ignored(&self) -> bool {
+ self.list && matches!(&self.format, Some(NextestFormat::Terse)) && self.ignored
+ }
+
+ pub fn nextest_run_by_name(&self) -> bool {
+ self.nocapture && self.exact
+ }
+
+ pub fn nextest_invalid_args(&self) -> Option<Error> {
+ if self.format.is_some() && self.list {
+ Some(anyhow!("`--format` only exists for nextest compatibility and is not supported without `--list`"))
+ } else if self.nocapture && !self.exact {
+ Some(anyhow!("`--nocapture` only exists for nextest compatibility and is not supported without `--exact`"))
+ } else if self.exact && !self.nocapture {
+ Some(anyhow!("`--nocapture` only exists for nextest compatibility and is not supported without `--nocapture`"))
+ } else {
+ None
+ }
+ }
 }

windsock/src/lib.rs

@@ -3,6 +3,7 @@ mod cli;
 pub mod cloud;
 mod data;
 mod filter;
+mod list;
 mod report;
 mod tables;
 
@@ -84,10 +85,11 @@ impl Windsock {
  ReportArchive::clear_baseline();
  println!("Baseline cleared");
  } else if args.list {
- println!("Benches:");
- for bench in &self.benches {
- println!("{}", bench.tags.get_name());
- }
+ list::list(&args, &self.benches);
+ } else if args.nextest_run_by_name() {
+ create_runtime(None).block_on(self.run_nextest(args, running_in_release))?;
+ } else if let Some(err) = args.nextest_invalid_args() {
+ return Err(err);
  } else if let Some(internal_run) = &args.internal_run {
  self.internal_run(&args, internal_run, running_in_release)?;
  } else if let Some(name) = args.name.clone() {
@@ -126,6 +128,15 @@ impl Windsock {
  }
  }
 
+ async fn run_nextest(&mut self, mut args: Args, running_in_release: bool) -> Result<()> {
+ // This is not a real bench we are just testing that it works,
+ // so set some really minimal runtime values
+ args.bench_length_seconds = Some(2);
+ args.operations_per_second = Some(100);
+
+ let name = args.filter.as_ref().unwrap().clone();
+ self.run_named_bench(args, name, running_in_release).await
+ }
  async fn run_named_bench(
  &mut self,
  args: Args,

windsock/src/list.rs

@@ -0,0 +1,18 @@
+use crate::{bench::BenchState, cli::Args};
+
+pub fn list(args: &Args, benches: &[BenchState]) {
+ if args.nextest_list_all() {
+ // list all windsock benches in nextest format
+ for bench in benches {
+ println!("{}: benchmark", bench.tags.get_name());
+ }
+ } else if args.nextest_list_ignored() {
+ // windsock does not support ignored tests
+ } else {
+ // regular usage
+ println!("Benches:");
+ for bench in benches {
+ println!("{}", bench.tags.get_name());
+ }
+ }
+}