Skip to content

Commit

Permalink
Improve the failure messages when tests fail. (#298)
Browse files Browse the repository at this point in the history
- The line numbers in the error messages now refer to the function that
  calls `shouldBe*`, instead of the location of the assertion inside of
  `shouldBe*`.

- The error messages now include the actual exit code, stdout and
  stderr from the failed invocation.

- The slight downside of this change is that the expected value for
  code/stdout/stderr no longer appears in the error message. Expected
  values are normally easily visible in the test code itself, so this
  should not be too painful.
  • Loading branch information
Dretch authored Jun 30, 2019
1 parent 127bac9 commit 7bf293a
Showing 1 changed file with 14 additions and 14 deletions.
28 changes: 14 additions & 14 deletions test/Utils.hs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import qualified Control.Exception as Exception
import Prelude hiding (FilePath)
import System.Directory (removePathForcibly)
import qualified System.Process as Process
import Test.Hspec (shouldBe)
import Test.Hspec (HasCallStack, shouldBe, shouldSatisfy)
import Turtle (ExitCode (..), FilePath, Text, cd, empty, encodeString,
procStrictWithErr, pwd, readTextFile)

Expand All @@ -35,33 +35,33 @@ runFor us cmd args = do
Concurrent.threadDelay us
Process.terminateProcess p

shouldBeSuccess :: (ExitCode, Text, Text) -> IO ()
shouldBeSuccess (code, _stdout, _stderr) = do
shouldBeSuccess :: HasCallStack => (ExitCode, Text, Text) -> IO ()
shouldBeSuccess result@(_code, _stdout, _stderr) = do
-- print $ "STDOUT: " <> _stdout
-- print $ "STDERR: " <> _stderr
code `shouldBe` ExitSuccess
result `shouldSatisfy` (\(code, _, _) -> code == ExitSuccess)

shouldBeSuccessOutput :: FilePath -> (ExitCode, Text, Text) -> IO ()
shouldBeSuccessOutput expected (code, out, _) = do
shouldBeSuccessOutput :: HasCallStack => FilePath -> (ExitCode, Text, Text) -> IO ()
shouldBeSuccessOutput expected result = do
expectedContent <- readFixture expected
(code, out) `shouldBe` (ExitSuccess, expectedContent)
result `shouldSatisfy` (\(code, stdout, _stderr) -> code == ExitSuccess && stdout == expectedContent)

shouldBeFailure :: (ExitCode, Text, Text) -> IO ()
shouldBeFailure (code, _stdout, _stderr) = do
shouldBeFailure :: HasCallStack => (ExitCode, Text, Text) -> IO ()
shouldBeFailure result@(_code, _stdout, _stderr) = do
-- print $ "STDOUT: " <> _stdout
-- print $ "STDERR: " <> _stderr
code `shouldBe` ExitFailure 1
result `shouldSatisfy` (\(code, _, _) -> code == ExitFailure 1)

shouldBeFailureOutput :: FilePath -> (ExitCode, Text, Text) -> IO ()
shouldBeFailureOutput expected (code, _, out) = do
shouldBeFailureOutput :: HasCallStack => FilePath -> (ExitCode, Text, Text) -> IO ()
shouldBeFailureOutput expected result = do
expectedContent <- readFixture expected
(code, out) `shouldBe` (ExitFailure 1, expectedContent)
result `shouldSatisfy` (\(code, _stdout, stderr) -> code == ExitFailure 1 && stderr == expectedContent)

readFixture :: FilePath -> IO Text
readFixture path =
readTextFile $ "../fixtures/" <> path

checkFixture :: FilePath -> IO ()
checkFixture :: HasCallStack => FilePath -> IO ()
checkFixture path = do
actual <- readTextFile path
expected <- readFixture path
Expand Down

0 comments on commit 7bf293a

Please sign in to comment.