Add dumping of stdout and stderr along with data for test failures

awslabs · Feb 4, 2025 · d3d19a7 · d3d19a7
1 parent bb19edd
commit d3d19a7
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 5 deletions.
diff --git a/test/examples/runtests.jl b/test/examples/runtests.jl
@@ -35,7 +35,9 @@ else
     ]
 end
 
+@info "Julia version: $VERSION"
 @info "Starting regression tests using `$palace` on $numprocs core$(numprocs > 1 ? "s" : "")"
+@info "Number of julia threads: $(Threads.nthreads())"
 
 reltol = 1.0e-4
 abstol = 1.0e-16

diff --git a/test/examples/testcase.jl b/test/examples/testcase.jl
@@ -64,16 +64,33 @@ function testcase(
         (~, dirs, files) = first(walkdir(postprodir))
         (~, ~, filesref) = first(walkdir(refpostprodir))
         metafiles = filter(x -> last(splitext(x)) != ".csv", files)
-        @test length(dirs) == 1 && first(dirs) == "paraview"
-        @test length(metafiles) == 1 && first(metafiles) == "palace.json"
-        @test length(filter(x -> last(splitext(x)) == ".csv", files)) == length(filesref)
+        @test length(dirs) == 1 && first(dirs) == "paraview" || (@show dirs; false)
+        @test length(metafiles) == 1 && first(metafiles) == "palace.json" || (@show metafiles; false)
+        @test length(filter(x -> last(splitext(x)) == ".csv", files)) == length(filesref) || (@show filesref; false)
+
+        # Helper to extract the stdout and stderr files and dump their contents.
+        # Useful when debugging a failure
+        function logdump(data...)
+            @show data
+            logfile = "log.out"
+            errfile = "err.out"
+            if isfile(joinpath(exampledir, logdir, logfile))
+                @warn "Contents of stdout:"
+                println(String(read(joinpath(exampledir, logdir, logfile))))
+            end
+            if isfile(joinpath(exampledir, logdir, errfile))
+                @warn "Contents of stderr:"
+                println(String(read(joinpath(exampledir, logdir, errfile))))
+            end
+            return false
+        end
 
         # Test the simulation outputs
         for file in filesref
             data    = CSV.File(joinpath(postprodir, file); header=1) |> DataFrame
             dataref = CSV.File(joinpath(refpostprodir, file); header=1) |> DataFrame
             if !skip_rowcount
-                @test nrow(data) == nrow(dataref)
+                @test nrow(data) == nrow(dataref) || logdump(data, dataref)
             end
             data = data[1:min(nrow(data), nrow(dataref)), :]
 
@@ -86,7 +103,7 @@ function testcase(
             rename!(data, strip.(names(data)))
             rename!(dataref, strip.(names(dataref)))
 
-            @test names(data) == names(dataref)
+            @test names(data) == names(dataref) || logdump(names(data), names(dataref))
             test = isapprox.(data, dataref; rtol=rtol, atol=atol)
             for (row, rowdataref, rowdata) in
                 zip(eachrow(test), eachrow(dataref), eachrow(data))