Added FAIL_CHECK

Works like FAIL, but does not abort test. As proposed in #765
catchorg · Mar 8, 2017 · 5dd0639 · 5dd0639
1 parent a251575
commit 5dd0639
Show file tree

Hide file tree

Showing 8 changed files with 83 additions and 22 deletions.
diff --git a/docs/logging.md b/docs/logging.md
@@ -26,6 +26,10 @@ The message is always reported but does not fail the test.
 
 The message is reported and the test case fails.
 
+**FAIL_CHECK(** _message expression_ **)**
+
+AS `FAIL`, but does not abort the test
+
 ## Quickly capture a variable value
 
 **CAPTURE(** _expression_ **)**

diff --git a/include/catch.hpp b/include/catch.hpp
@@ -126,6 +126,7 @@ LeakDetector leakDetector;
     #define CATCH_REGISTER_TEST_CASE( Function, ... ) INTERNAL_CATCH_REGISTER_TESTCASE( Function, __VA_ARGS__ )
     #define CATCH_SECTION( ... ) INTERNAL_CATCH_SECTION( __VA_ARGS__ )
     #define CATCH_FAIL( ... ) INTERNAL_CATCH_MSG( Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::Normal, "CATCH_FAIL", __VA_ARGS__ )
+    #define CATCH_FAIL_CHECK( ... ) INTERNAL_CATCH_MSG( Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::ContinueOnFailure, "CATCH_FAIL_CHECK", __VA_ARGS__ )
     #define CATCH_SUCCEED( ... ) INTERNAL_CATCH_MSG( Catch::ResultWas::Ok, Catch::ResultDisposition::ContinueOnFailure, "CATCH_SUCCEED", __VA_ARGS__ )
 #else
     #define CATCH_TEST_CASE( name, description ) INTERNAL_CATCH_TESTCASE( name, description )
@@ -134,6 +135,7 @@ LeakDetector leakDetector;
     #define CATCH_REGISTER_TEST_CASE( function, name, description ) INTERNAL_CATCH_REGISTER_TESTCASE( function, name, description )
     #define CATCH_SECTION( name, description ) INTERNAL_CATCH_SECTION( name, description )
     #define CATCH_FAIL( msg ) INTERNAL_CATCH_MSG( Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::Normal, "CATCH_FAIL", msg )
+    #define CATCH_FAIL_CHECK( msg ) INTERNAL_CATCH_MSG( Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::ContinueOnFailure, "CATCH_FAIL_CHECK", msg )
     #define CATCH_SUCCEED( msg ) INTERNAL_CATCH_MSG( Catch::ResultWas::Ok, Catch::ResultDisposition::ContinueOnFailure, "CATCH_SUCCEED", msg )
 #endif
 #define CATCH_ANON_TEST_CASE() INTERNAL_CATCH_TESTCASE( "", "" )
@@ -189,20 +191,22 @@ LeakDetector leakDetector;
 #define SCOPED_CAPTURE( msg ) INTERNAL_CATCH_INFO( #msg " := " << Catch::toString(msg), "CAPTURE" )
 
 #ifdef CATCH_CONFIG_VARIADIC_MACROS
-    #define TEST_CASE( ... ) INTERNAL_CATCH_TESTCASE( __VA_ARGS__ )
-    #define TEST_CASE_METHOD( className, ... ) INTERNAL_CATCH_TEST_CASE_METHOD( className, __VA_ARGS__ )
-    #define METHOD_AS_TEST_CASE( method, ... ) INTERNAL_CATCH_METHOD_AS_TEST_CASE( method, __VA_ARGS__ )
-    #define REGISTER_TEST_CASE( Function, ... ) INTERNAL_CATCH_REGISTER_TESTCASE( Function, __VA_ARGS__ )
-    #define SECTION( ... ) INTERNAL_CATCH_SECTION( __VA_ARGS__ )
-    #define FAIL( ... ) INTERNAL_CATCH_MSG( Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::Normal, "FAIL", __VA_ARGS__ )
-    #define SUCCEED( ... ) INTERNAL_CATCH_MSG( Catch::ResultWas::Ok, Catch::ResultDisposition::ContinueOnFailure, "SUCCEED", __VA_ARGS__ )
+#define TEST_CASE( ... ) INTERNAL_CATCH_TESTCASE( __VA_ARGS__ )
+#define TEST_CASE_METHOD( className, ... ) INTERNAL_CATCH_TEST_CASE_METHOD( className, __VA_ARGS__ )
+#define METHOD_AS_TEST_CASE( method, ... ) INTERNAL_CATCH_METHOD_AS_TEST_CASE( method, __VA_ARGS__ )
+#define REGISTER_TEST_CASE( Function, ... ) INTERNAL_CATCH_REGISTER_TESTCASE( Function, __VA_ARGS__ )
+#define SECTION( ... ) INTERNAL_CATCH_SECTION( __VA_ARGS__ )
+#define FAIL( ... ) INTERNAL_CATCH_MSG( Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::Normal, "FAIL", __VA_ARGS__ )
+#define FAIL_CHECK( ... ) INTERNAL_CATCH_MSG( Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::ContinueOnFailure, "FAIL_CHECK", __VA_ARGS__ )
+#define SUCCEED( ... ) INTERNAL_CATCH_MSG( Catch::ResultWas::Ok, Catch::ResultDisposition::ContinueOnFailure, "SUCCEED", __VA_ARGS__ )
 #else
-    #define TEST_CASE( name, description ) INTERNAL_CATCH_TESTCASE( name, description )
+#define TEST_CASE( name, description ) INTERNAL_CATCH_TESTCASE( name, description )
     #define TEST_CASE_METHOD( className, name, description ) INTERNAL_CATCH_TEST_CASE_METHOD( className, name, description )
     #define METHOD_AS_TEST_CASE( method, name, description ) INTERNAL_CATCH_METHOD_AS_TEST_CASE( method, name, description )
     #define REGISTER_TEST_CASE( method, name, description ) INTERNAL_CATCH_REGISTER_TESTCASE( method, name, description )
     #define SECTION( name, description ) INTERNAL_CATCH_SECTION( name, description )
     #define FAIL( msg ) INTERNAL_CATCH_MSG( Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::Normal, "FAIL", msg )
+    #define FAIL_CHECK( msg ) INTERNAL_CATCH_MSG( Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::ContinueOnFailure, "FAIL_CHECK", msg )
     #define SUCCEED( msg ) INTERNAL_CATCH_MSG( Catch::ResultWas::Ok, Catch::ResultDisposition::ContinueOnFailure, "SUCCEED", msg )
 #endif
 #define ANON_TEST_CASE() INTERNAL_CATCH_TESTCASE( "", "" )

diff --git a/projects/SelfTest/Baselines/console.std.approved.txt b/projects/SelfTest/Baselines/console.std.approved.txt
@@ -287,6 +287,20 @@ MessageTests.cpp:<line number>
 
 MessageTests.cpp:<line number>: FAILED:
 
+-------------------------------------------------------------------------------
+FAIL_CHECK does not abort the test
+-------------------------------------------------------------------------------
+MessageTests.cpp:<line number>
+...............................................................................
+
+MessageTests.cpp:<line number>: FAILED:
+explicitly with message:
+  This is a failure
+
+MessageTests.cpp:<line number>:
+warning:
+  This message appears in the output
+
 -------------------------------------------------------------------------------
 INFO and WARN do not abort tests
 -------------------------------------------------------------------------------
@@ -901,6 +915,6 @@ with expansion:
   "first" == "second"
 
 ===============================================================================
-test cases: 165 | 119 passed | 44 failed |  2 failed as expected
-assertions: 957 | 852 passed | 87 failed | 18 failed as expected
+test cases: 166 | 119 passed | 44 failed |  3 failed as expected
+assertions: 958 | 852 passed | 87 failed | 19 failed as expected
 
diff --git a/projects/SelfTest/Baselines/console.sw.approved.txt b/projects/SelfTest/Baselines/console.sw.approved.txt
@@ -61,7 +61,7 @@ with expansion:
 
 MiscTests.cpp:<line number>:
 PASSED:
-  REQUIRE( (*__errno_location ()) == 1 )
+  REQUIRE( (*__error()) == 1 )
 with expansion:
   1 == 1
 
@@ -1216,6 +1216,20 @@ MessageTests.cpp:<line number>
 
 MessageTests.cpp:<line number>: FAILED:
 
+-------------------------------------------------------------------------------
+FAIL_CHECK does not abort the test
+-------------------------------------------------------------------------------
+MessageTests.cpp:<line number>
+...............................................................................
+
+MessageTests.cpp:<line number>: FAILED:
+explicitly with message:
+  This is a failure
+
+MessageTests.cpp:<line number>:
+warning:
+  This message appears in the output
+
 -------------------------------------------------------------------------------
 Factorials are computed
 -------------------------------------------------------------------------------
@@ -9367,6 +9381,6 @@ MiscTests.cpp:<line number>:
 PASSED:
 
 ===============================================================================
-test cases: 165 | 118 passed | 45 failed |  2 failed as expected
-assertions: 959 | 852 passed | 89 failed | 18 failed as expected
+test cases: 166 | 118 passed | 45 failed |  3 failed as expected
+assertions: 960 | 852 passed | 89 failed | 19 failed as expected
 
diff --git a/projects/SelfTest/Baselines/console.swa4.approved.txt b/projects/SelfTest/Baselines/console.swa4.approved.txt
@@ -61,7 +61,7 @@ with expansion:
 
 MiscTests.cpp:<line number>:
 PASSED:
-  REQUIRE( (*__errno_location ()) == 1 )
+  REQUIRE( (*__error()) == 1 )
 with expansion:
   1 == 1
 
@@ -82,7 +82,10 @@ ConditionTests.cpp:<line number>: FAILED:
 with expansion:
   false
 
+ConditionTests.cpp:<line number>: FAILED:
+  CHECK_FALSE( true )
+
 ===============================================================================
-test cases:  5 | 3 passed | 2 failed
-assertions: 11 | 7 passed | 4 failed
+test cases:  5 | 3 passed | 1 failed | 1 failed as expected
+assertions: 12 | 7 passed | 4 failed | 1 failed as expected
 
diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuitesspanner>
-  <testsuite name="<exe-name>" errors="13" failures="77" tests="960" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+  <testsuite name="<exe-name>" errors="13" failures="77" tests="961" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
     <testcase classname="global" name="# A test name that starts with a #" time="{duration}"/>
     <testcase classname="global" name="#542" time="{duration}"/>
     <testcase classname="global" name="#809" time="{duration}"/>
@@ -199,6 +199,12 @@ MessageTests.cpp:<line number>
     </testcase>
     <testcase classname="global" name="FAIL does not require an argument" time="{duration}">
       <failure type="FAIL">
+MessageTests.cpp:<line number>
+      </failure>
+    </testcase>
+    <testcase classname="global" name="FAIL_CHECK does not abort the test" time="{duration}">
+      <failure type="FAIL_CHECK">
+This is a failure
 MessageTests.cpp:<line number>
       </failure>
     </testcase>

diff --git a/projects/SelfTest/Baselines/xml.sw.approved.txt b/projects/SelfTest/Baselines/xml.sw.approved.txt
@@ -50,7 +50,7 @@
       </Expression>
       <OverallResult success="true"/>
     </TestCase>
-    <TestCase name="#835 -- errno should not be touched by Catch" filename="projects/<exe-name>/MiscTests.cpp" >
+    <TestCase name="#835 -- errno should not be touched by Catch" tags="[!shouldfail]" filename="projects/<exe-name>/MiscTests.cpp" >
       <Expression success="false" type="CHECK" filename="projects/<exe-name>/MiscTests.cpp" >
         <Original>
           f() == 0
@@ -61,13 +61,13 @@
       </Expression>
       <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/MiscTests.cpp" >
         <Original>
-          (*__errno_location ()) == 1
+          (*__error()) == 1
         </Original>
         <Expanded>
           1 == 1
         </Expanded>
       </Expression>
-      <OverallResult success="false"/>
+      <OverallResult success="true"/>
     </TestCase>
     <TestCase name="'Not' checks that should fail" tags="[.][failing][hide]" filename="projects/<exe-name>/ConditionTests.cpp" >
       <Expression success="false" type="CHECK" filename="projects/<exe-name>/ConditionTests.cpp" >
@@ -1378,6 +1378,15 @@
       <Failure filename="projects/<exe-name>/MessageTests.cpp" />
       <OverallResult success="false"/>
     </TestCase>
+    <TestCase name="FAIL_CHECK does not abort the test" tags="[.][failing][hide][messages]" filename="projects/<exe-name>/MessageTests.cpp" >
+      <Failure filename="projects/<exe-name>/MessageTests.cpp" >
+        This is a failure
+      </Failure>
+      <Warning>
+        This message appears in the output
+      </Warning>
+      <OverallResult success="false"/>
+    </TestCase>
     <TestCase name="Factorials are computed" tags="[factorial]" filename="projects/<exe-name>/MiscTests.cpp" >
       <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/MiscTests.cpp" >
         <Original>
@@ -10007,7 +10016,7 @@ spanner      <OverallResult success="true"/>
       </Section>
       <OverallResult success="true"/>
     </TestCase>
-    <OverallResults successes="852" failures="90" expectedFailures="18"/>
+    <OverallResults successes="852" failures="90" expectedFailures="19"/>
   </Group>
-  <OverallResults successes="852" failures="89" expectedFailures="18"/>
+  <OverallResults successes="852" failures="89" expectedFailures="19"/>
 </Catch>
diff --git a/projects/SelfTest/MessageTests.cpp b/projects/SelfTest/MessageTests.cpp
@@ -53,6 +53,13 @@ TEST_CASE( "INFO gets logged on failure, even if captured before successful asse
 TEST_CASE( "FAIL aborts the test", "[failing][messages][.]" )
 {
     FAIL( "This is a " << "failure" );    // This should output the message and abort
+    WARN( "We should never see this");
+}
+
+TEST_CASE( "FAIL_CHECK does not abort the test", "[failing][messages][.]" )
+{
+    FAIL_CHECK( "This is a " << "failure" );    // This should output the message then continue
+    WARN( "This message appears in the output");
 }
 
 #ifdef CATCH_CONFIG_VARIADIC_MACROS