Support multi-line run/fail directives in CLI tests (#1782)

Also add a `readme.wat` test explainer as to what's going on.

Cargo.toml

@@ -178,6 +178,7 @@ tempfile = "3.1"
 wast = { path = 'crates/wast' }
 pretty_assertions = { workspace = true }
 libtest-mimic = { workspace = true }
+indexmap = { workspace = true }
 
 [[test]]
 name = "cli"

tests/cli.rs

@@ -1,31 +1,12 @@
 //! A test suite to test the `wasm-tools` CLI itself.
 //!
-//! This test suite will look for `*.wat` files in the `tests/cli/**` directory,
-//! recursively. Each wat file must have a directive of the form:
-//!
-//! ;; RUN: ...
-//!
-//! where `...` is a space-separate set of command to pass to the `wasm-tools`
-//! CLI. The `%` argument is replaced with the path to the current file. For
-//! example:
-//!
-//! ;; RUN: dump %
-//!
-//! would execute `wasm-tools dump the-current-file.wat`. The `cli` directory
-//! additionally contains `*.stdout` and `*.stderr` files to assert the output
-//! of the subcommand. Files are not present if the stdout/stderr are empty.
-//!
-//! This also supports a limited form of piping along the lines of:
-//!
-//! ;; RUN: strip % | objdump
-//!
-//! where a `|` will execute the first subcommand and pipe its stdout into the
-//! stdin of the next command.
-//!
-//! Use `BLESS=1` in the environment to auto-update expectation files. Be sure
-//! to look at the diff!
+//! This test suite will look for `*.wat` and `*.wit` files in the
+//! `tests/cli/**` directory, recursively. For more information about supported
+//! directives and features of this test suite see the `tests/cli/readme.wat`
+//! file which has an explanatory comment at the top for what's going on.
 
-use anyhow::{anyhow, bail, Context, Result};
+use anyhow::{bail, Context, Result};
+use indexmap::IndexMap;
 use libtest_mimic::{Arguments, Trial};
 use pretty_assertions::StrComparison;
 use std::env;
@@ -59,54 +40,96 @@ fn main() {
  libtest_mimic::run(&args, trials).exit();
 }
 
-fn wasm_tools_exe() -> Command {
- Command::new(env!("CARGO_BIN_EXE_wasm-tools"))
-}
-
 fn run_test(test: &Path, bless: bool) -> Result<()> {
  let contents = std::fs::read_to_string(test)?;
- let (line, should_fail) = contents
+
+ let mut directives = contents
  .lines()
- .filter_map(|l| {
- let run = l.strip_prefix(";; RUN: ").or(l.strip_prefix("// RUN: "));
- let fail = l.strip_prefix(";; FAIL: ").or(l.strip_prefix("// FAIL: "));
- run.map(|l| (l, false)).or(fail.map(|l| (l, true)))
- })
- .next()
- .ok_or_else(|| anyhow!("no line found with `;; RUN: ` directive"))?;
-
- let mut cmd = wasm_tools_exe();
- let mut stdin = None;
- let tempdir = TempDir::new()?;
- for arg in line.split_whitespace() {
- let arg = arg.replace("%tmpdir", tempdir.path().to_str().unwrap());
- if arg == "|" {
- let output = execute(&mut cmd, stdin.as_deref(), false)?;
- stdin = Some(output.stdout);
- cmd = wasm_tools_exe();
- } else if arg == "%" {
- cmd.arg(test);
- } else {
- cmd.arg(arg);
+ .enumerate()
+ .filter(|(_, l)| !l.is_empty())
+ .filter_map(|(i, l)| {
+ l.strip_prefix("// ")
+ .or(l.strip_prefix(";; "))
+ .map(|l| (i + 1, l))
+ });
+
+ let mut commands = IndexMap::new();
+
+ while let Some((i, line)) = directives.next() {
+ let run = line.strip_prefix("RUN");
+ let fail = line.strip_prefix("FAIL");
+ let (directive, should_fail) = match run.map(|l| (l, false)).or(fail.map(|l| (l, true))) {
+ Some(pair) => pair,
+ None => continue,
+ };
+ let (cmd, name) = match directive.strip_prefix("[") {
+ Some(prefix) => match prefix.find("]:") {
+ Some(i) => (&prefix[i + 2..], &prefix[..i]),
+ None => bail!("line {i}: failed to find `]:` after `[`"),
+ },
+ None => match directive.strip_prefix(":") {
+ Some(cmd) => (cmd, ""),
+ None => bail!("line {i}: failed to find `:` after `RUN` or `FAIL`"),
+ },
+ };
+ let mut cmd = cmd.to_string();
+ while cmd.ends_with("\\") {
+ cmd.pop();
+ match directives.next() {
+ Some((_, line)) => cmd.push_str(line),
+ None => bail!("line {i}: directive ends in `\\` but nothing on next line"),
+ }
+ }
+
+ match commands.insert(name, (cmd, should_fail)) {
+ Some(_) => bail!("line {i}: duplicate directive named {name:?}"),
+ None => {}
  }
  }
 
- let output = execute(&mut cmd, stdin.as_deref(), should_fail)?;
- let extension = test.extension().unwrap().to_str().unwrap();
- assert_output(
- bless,
- &output.stdout,
- &test.with_extension(&format!("{extension}.stdout")),
- &tempdir,
- )
- .context("failed to check stdout expectation (auto-update with BLESS=1)")?;
- assert_output(
- bless,
- &output.stderr,
- &test.with_extension(&format!("{extension}.stderr")),
- &tempdir,
- )
- .context("failed to check stderr expectation (auto-update with BLESS=1)")?;
+ if commands.is_empty() {
+ bail!("failed to find `// RUN: ...` or `// FAIL: ...` at the top of this file");
+ }
+ let exe = Path::new(env!("CARGO_BIN_EXE_wasm-tools"));
+ let tempdir = TempDir::new_in(exe.parent().unwrap())?;
+ for (name, (line, should_fail)) in commands {
+ let mut cmd = Command::new(exe);
+ let mut stdin = None;
+ for arg in line.split_whitespace() {
+ let arg = arg.replace("%tmpdir", tempdir.path().to_str().unwrap());
+ if arg == "|" {
+ let output = execute(&mut cmd, stdin.as_deref(), false)?;
+ stdin = Some(output.stdout);
+ cmd = Command::new(exe);
+ } else if arg == "%" {
+ cmd.arg(test);
+ } else {
+ cmd.arg(arg);
+ }
+ }
+
+ let output = execute(&mut cmd, stdin.as_deref(), should_fail)?;
+ let extension = test.extension().unwrap().to_str().unwrap();
+ let extension = if name.is_empty() {
+ extension.to_string()
+ } else {
+ format!("{extension}.{name}")
+ };
+ assert_output(
+ bless,
+ &output.stdout,
+ &test.with_extension(&format!("{extension}.stdout")),
+ &tempdir,
+ )
+ .context("failed to check stdout expectation (auto-update with BLESS=1)")?;
+ assert_output(
+ bless,
+ &output.stderr,
+ &test.with_extension(&format!("{extension}.stderr")),
+ &tempdir,
+ )
+ .context("failed to check stderr expectation (auto-update with BLESS=1)")?;
+ }
  Ok(())
 }
 

tests/cli/readme.wat

@@ -0,0 +1,76 @@
+;; This is intended to be a self-documenting test which explains what's
+;; possible in directives for this test suite in `tests/cli/*`. The purpose of
+;; this test suite is to make it as easy as dropping a file in this directory to
+;; test the `wasm-tools` CLI tool and its subcommands. The test file itself is
+;; generally the input to the test and what's being tested will be present in
+;; comments at the top of the file with directives.
+;;
+;; All test directives must come in comments at the start of the file:
+;;
+;; RUN: validate %
+;;
+;; The `RUN` prefix indicates that the specified `wasm-tools` subcommand should
+;; be executed. It's possible to have more than one test in a file by having
+;; named directives such as:
+;;
+;; RUN[validate-again]: validate %
+;;
+;; Directive names must be unique, so using `validate-again` would not be valid.
+;; Additionally you can't use an unprefixed directive more than once so using
+;; `RUN: ...` here again would not be allowed for example.
+;;
+;; You can also use the `FAIL` directive to indicate that the subcommand should
+;; fail rather than succeed.
+;;
+;; FAIL[should-fail]: validate % --features=-simd
+;;
+;; As you can see directives can have comments around them. Directives are
+;; identified as comment lines starting with `RUN` or `FAIL`.
+;;
+;; Within directives there are a few feature. First as you've seen the `%` value
+;; will be substituted with the current filename which means:
+;;
+;; RUN[subst]: validate %
+;;
+;; means to run `wasm-tools validate tests/cli/readme.wat` and test the result
+;; is successful.
+;;
+;; You can additionally use `|` to pipe commands together by feeding the stdout
+;; of the previous command into the stdin of the next command.
+;;
+;; RUN[pipe]: print % | validate
+;;
+;; Note that when piping commands the intermediate commands before the final
+;; one, in this case `print` being the intermediate, must all succeed.
+;;
+;; Tests also assert the stdout/stderr of the command being tested. For example
+;; if printing is tested:
+;;
+;; RUN[print]: print %
+;;
+;; then this tests that `tests/cli/readme.wat.print.stdout` is the result of
+;; `wasm-tools print tests/cli/readme.wat`. Note that this can be tedious to
+;; update so you can use the environment variable `BLESS=1` to automatically
+;; update all test assertions. This can then be reviewed after the test is
+;; passing for accuracy.
+;;
+;; Each test additionally can have a temporary directory available to it which
+;; is accessible with the `%tmpdir` substitution. For example:
+;;
+;; RUN[tmpdir]: print % -o %tmpdir/foo.wat | validate %tmpdir/foo.wat
+;;
+;; Note that temporary directories are persisted across tests in the same file,
+;; but different files all get different temporary directories.
+;;
+;; You can also split commands across multiple lines:
+;;
+;; RUN[multiline]: print % | \
+;; validate
+;;
+;; here the `\` character is deleted and the next line is concatenated.
+
+
+;; this is the contents of the test, mostly empty in this case.
+(module
+ (type (func (result v128)))
+)

tests/cli/readme.wat.print.stdout

@@ -0,0 +1,3 @@
+(module
+ (type (;0;) (func (result v128)))
+)

tests/cli/readme.wat.should-fail.stderr

@@ -0,0 +1 @@
+error: SIMD support is not enabled (at offset 0xb)