fix: add command file outputs to task evaluation file tests.

wdl-engine/tests/tasks.rs

@@ -26,6 +26,7 @@ use std::path::Path;
 use std::path::PathBuf;
 use std::path::absolute;
 use std::process::exit;
+use std::sync::LazyLock;
 use std::thread::available_parallelism;
 
 use anyhow::Context;
@@ -41,6 +42,7 @@ use futures::StreamExt;
 use futures::stream;
 use path_clean::clean;
 use pretty_assertions::StrComparison;
+use regex::Regex;
 use tempfile::TempDir;
 use walkdir::WalkDir;
 use wdl_analysis::AnalysisResult;
@@ -57,6 +59,11 @@ use wdl_engine::Inputs;
 use wdl_engine::local::LocalTaskExecutionBackend;
 use wdl_engine::v1::TaskEvaluator;
 
+/// Regex used to replace temporary file names in task command files with
+/// consistent names for test baselines.
+static TEMP_FILENAME_REGEX: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new("tmp[[:alnum:]]{6}").expect("invalid regex"));
+
 /// Finds tests to run as part of the analysis test suite.
 fn find_tests() -> Vec<PathBuf> {
     // Check for filter arguments consisting of test names
@@ -241,6 +248,12 @@ async fn run_test(test: &Path, result: AnalysisResult) -> Result<()> {
 }
 
 fn compare_evaluation_results(test: &Path, dir: &Path, evaluated: &EvaluatedTask) -> Result<()> {
+    let command = fs::read_to_string(evaluated.command()).with_context(|| {
+        format!(
+            "failed to read task command file `{path}`",
+            path = evaluated.command().display()
+        )
+    })?;
     let stdout =
         fs::read_to_string(evaluated.stdout().as_file().unwrap().as_str()).with_context(|| {
             format!(
@@ -256,6 +269,18 @@ fn compare_evaluation_results(test: &Path, dir: &Path, evaluated: &EvaluatedTask
             )
         })?;
 
+    // Replace any temporary file names in the command
+    let mut command = normalize(dir, &command);
+    for i in 0..usize::MAX {
+        match TEMP_FILENAME_REGEX.replace(&command, format!("tmp{i}")) {
+            Cow::Borrowed(_) => break,
+            Cow::Owned(s) => command = s,
+        }
+    }
+
+    let command_path = test.join("command");
+    compare_result(dir, &command_path, &command)?;
+
     let stdout_path = test.join("stdout");
     compare_result(dir, &stdout_path, &stdout)?;
 

wdl-engine/tests/tasks/change-extension-task/command

@@ -0,0 +1,2 @@
+printf "data" > 'foo.data'
+printf "index" > 'foo.index'

wdl-engine/tests/tasks/default-option-task/command

@@ -0,0 +1,3 @@
+printf foobar > result1
+printf foobar > result2
+printf foobar > result3

wdl-engine/tests/tasks/expressions-task/command

@@ -0,0 +1 @@
+printf "hello" > hello.txt

wdl-engine/tests/tasks/file-output-task/command

@@ -0,0 +1,2 @@
+printf "hello" > foo.hello
+printf "goodbye" > foo.goodbye

wdl-engine/tests/tasks/file-sizes-task/command

@@ -0,0 +1 @@
+printf "this file is 22 bytes/n" > created_file

wdl-engine/tests/tasks/flags-task/command

@@ -0,0 +1,8 @@
+# If `max_matches` is `None`, the command
+# grep -m  hello /Users/phuene/src/peterhuene/wdl/wdl-engine/tests/tasks/flags-task/greetings.txt
+# would evaluate to
+# 'grep -m <pattern> <infile>', which would be an error.
+
+# Instead, make both the flag and the value conditional on `max_matches`
+# being defined.
+grep  hello '/Users/phuene/src/peterhuene/wdl/wdl-engine/tests/tasks/flags-task/greetings.txt' | wc -l | sed 's/^ */'

wdl-engine/tests/tasks/glob-task/command

@@ -0,0 +1,3 @@
+for i in {1..3}; do
+    printf ${i} > file_${i}.txt
+done

wdl-engine/tests/tasks/hello/command

@@ -0,0 +1 @@
+grep -E 'hello.*' '/Users/phuene/src/peterhuene/wdl/wdl-engine/tests/tasks/hello/greetings.txt'

wdl-engine/tests/tasks/import-structs/command

@@ -0,0 +1 @@
+printf "The patient makes $35000.000000 per hour/n"

wdl-engine/tests/tasks/input-type-qualifiers/command

@@ -0,0 +1,4 @@
+cat 'tmp/tmp0' >> result
+cat 'tmp/tmp1' >> result
+
+cat 'tmp/tmp2' >> result

wdl-engine/tests/tasks/member-access/command

@@ -0,0 +1 @@
+printf "bar"

wdl-engine/tests/tasks/missing-output-file/command

@@ -0,0 +1 @@
+echo this task forgot to write to foo.txt!

wdl-engine/tests/tasks/optional-output-task/command

@@ -0,0 +1,4 @@
+printf "1" > example1.txt
+if false; then
+  printf "2" > example2.txt
+fi

wdl-engine/tests/tasks/outputs-task/command

@@ -0,0 +1,2 @@
+printf 5 > threshold.txt
+touch a.csv b.csv

wdl-engine/tests/tasks/person-struct-task/command

@@ -0,0 +1,8 @@
+printf "Hello Richard! You have 1 test result(s) available./n"
+
+if true; then
+  if [ "1000000" -gt 1000 ]; then
+    currency="USD"
+    printf "Please transfer $currency 500 to continue"
+  fi
+fi

wdl-engine/tests/tasks/primitive-literals/command

@@ -0,0 +1,2 @@
+mkdir -p testdir
+printf "hello" > testdir/hello.txt

wdl-engine/tests/tasks/private-declaration-task/command

@@ -0,0 +1 @@
+head -3 'tmp/tmp0'

wdl-engine/tests/tasks/relative-and-absolute-task/command

@@ -0,0 +1,2 @@
+mkdir -p my/path/to
+printf "something" > my/path/to/something.txt

wdl-engine/tests/tasks/sum-task/command

@@ -0,0 +1 @@
+printf '1 2 3' | awk '{tot=0; for(i=1;i<=NF;i++) tot+=$i; print tot}'

wdl-engine/tests/tasks/task-fail/command

@@ -0,0 +1,2 @@
+>&2 echo this task is going to fail!
+exit 1

wdl-engine/tests/tasks/task-inputs-task/command

@@ -0,0 +1,6 @@
+for i in 1..1; do
+  printf "hello/n"
+done
+if false; then
+  cat 
+fi

wdl-engine/tests/tasks/task-with-comments/command

@@ -0,0 +1,2 @@
+# This comment WILL be included within the command after it has been parsed
+echo 2

wdl-engine/tests/tasks/test-join-paths/command

@@ -0,0 +1,2 @@
+mkdir 'mydir'
+echo -n "hello" > 'mydir/mydata.txt'

wdl-engine/tests/tasks/test-placeholders-task/command

@@ -0,0 +1,4 @@
+# The `read_lines` function reads the lines from a file into an
+# array. The `sep` function concatenates the lines with a space
+# (" ") delimiter. The resulting string is then printed to stdout.
+printf "hello world hi_world hello friend"

wdl-engine/tests/tasks/test-stderr/command

@@ -0,0 +1 @@
+>&2 printf "hello world"

wdl-engine/tests/tasks/test-stdout/command

@@ -0,0 +1 @@
+printf "hello world"

wdl-engine/tests/tasks/true-false-ternary/command

@@ -0,0 +1,3 @@
+# these two commands have the same result
+printf "hello world" > result1
+printf "hello world" > result2