Add more unit tests

* Call order of listeners/reporters in multireporter * Adding listeners/reporters properly updates reporter preferences
catchorg · Dec 29, 2021 · 3d5c13f · 3d5c13f
1 parent 4f10efe
commit 3d5c13f
Show file tree

Hide file tree

Showing 19 changed files with 836 additions and 19 deletions.
diff --git a/src/catch2/reporters/catch_reporter_listening.hpp b/src/catch2/reporters/catch_reporter_listening.hpp
@@ -30,11 +30,8 @@ namespace Catch {
 
     public:
         ListeningReporter( IConfig const* config ):
-            IStreamingReporter( config ) {
-            // We will assume that listeners will always want all assertions
-            m_preferences.shouldReportAllAssertions = true;
-        }
-
+            IStreamingReporter( config )
+        {}
 
         void addListener( IStreamingReporterPtr&& listener );
         void addReporter( IStreamingReporterPtr&& reporter );

diff --git a/tests/SelfTest/Baselines/automake.sw.approved.txt b/tests/SelfTest/Baselines/automake.sw.approved.txt
@@ -170,6 +170,8 @@ Nor would this
 :test-result: FAIL Matchers can be negated (Not) with the ! operator - failing
 :test-result: XFAIL Mayfail test case with nested sections
 :test-result: FAIL Mismatching exception messages failing the test
+:test-result: PASS Multireporter calls reporters and listeners in correct order
+:test-result: PASS Multireporter updates ReporterPreferences properly
 :test-result: PASS Nested generators and captured variables
 :test-result: FAIL Nice descriptive name
 :test-result: FAIL Non-std exceptions can be translated

diff --git a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt
@@ -168,6 +168,8 @@
 :test-result: FAIL Matchers can be negated (Not) with the ! operator - failing
 :test-result: XFAIL Mayfail test case with nested sections
 :test-result: FAIL Mismatching exception messages failing the test
+:test-result: PASS Multireporter calls reporters and listeners in correct order
+:test-result: PASS Multireporter updates ReporterPreferences properly
 :test-result: PASS Nested generators and captured variables
 :test-result: FAIL Nice descriptive name
 :test-result: FAIL Non-std exceptions can be translated

diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt
@@ -941,6 +941,25 @@ Condition.tests.cpp:<line number>: failed: explicitly
 Condition.tests.cpp:<line number>: failed: explicitly
 Exception.tests.cpp:<line number>: passed: thisThrows(), "expected exception" for: "expected exception" equals: "expected exception"
 Exception.tests.cpp:<line number>: failed: thisThrows(), "should fail" for: "expected exception" equals: "should fail"
+Reporters.tests.cpp:<line number>: passed: records == expected for: { "Hello", "world", "Goodbye", "world" }
+==
+{ "Hello", "world", "Goodbye", "world" }
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == false for: false == false
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == false for: false == false
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
 Generators.tests.cpp:<line number>: passed: values > -6 for: 3 > -6
 Generators.tests.cpp:<line number>: passed: values > -6 for: 4 > -6
 Generators.tests.cpp:<line number>: passed: values > -6 for: 5 > -6

diff --git a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt
@@ -939,6 +939,25 @@ Condition.tests.cpp:<line number>: failed: explicitly
 Condition.tests.cpp:<line number>: failed: explicitly
 Exception.tests.cpp:<line number>: passed: thisThrows(), "expected exception" for: "expected exception" equals: "expected exception"
 Exception.tests.cpp:<line number>: failed: thisThrows(), "should fail" for: "expected exception" equals: "should fail"
+Reporters.tests.cpp:<line number>: passed: records == expected for: { "Hello", "world", "Goodbye", "world" }
+==
+{ "Hello", "world", "Goodbye", "world" }
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == false for: false == false
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == false for: false == false
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
+Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
 Generators.tests.cpp:<line number>: passed: values > -6 for: 3 > -6
 Generators.tests.cpp:<line number>: passed: values > -6 for: 4 > -6
 Generators.tests.cpp:<line number>: passed: values > -6 for: 5 > -6

diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt
@@ -1426,6 +1426,6 @@ due to unexpected exception with message:
   Why would you throw a std::string?
 
 ===============================================================================
-test cases:  384 |  307 passed |  70 failed |  7 failed as expected
-assertions: 2196 | 2040 passed | 129 failed | 27 failed as expected
+test cases:  386 |  309 passed |  70 failed |  7 failed as expected
+assertions: 2213 | 2057 passed | 129 failed | 27 failed as expected
 
diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt
@@ -7199,6 +7199,125 @@ Exception.tests.cpp:<line number>: FAILED:
 with expansion:
   "expected exception" equals: "should fail"
 
+-------------------------------------------------------------------------------
+Multireporter calls reporters and listeners in correct order
+-------------------------------------------------------------------------------
+Reporters.tests.cpp:<line number>
+...............................................................................
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( records == expected )
+with expansion:
+  { "Hello", "world", "Goodbye", "world" }
+  ==
+  { "Hello", "world", "Goodbye", "world" }
+
+-------------------------------------------------------------------------------
+Multireporter updates ReporterPreferences properly
+-------------------------------------------------------------------------------
+Reporters.tests.cpp:<line number>
+...............................................................................
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == false )
+with expansion:
+  false == false
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false )
+with expansion:
+  false == false
+
+-------------------------------------------------------------------------------
+Multireporter updates ReporterPreferences properly
+  Adding listeners
+-------------------------------------------------------------------------------
+Reporters.tests.cpp:<line number>
+...............................................................................
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
+with expansion:
+  true == true
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false )
+with expansion:
+  false == false
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
+with expansion:
+  true == true
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true )
+with expansion:
+  true == true
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
+with expansion:
+  true == true
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true )
+with expansion:
+  true == true
+
+-------------------------------------------------------------------------------
+Multireporter updates ReporterPreferences properly
+-------------------------------------------------------------------------------
+Reporters.tests.cpp:<line number>
+...............................................................................
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == false )
+with expansion:
+  false == false
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false )
+with expansion:
+  false == false
+
+-------------------------------------------------------------------------------
+Multireporter updates ReporterPreferences properly
+  Adding reporters
+-------------------------------------------------------------------------------
+Reporters.tests.cpp:<line number>
+...............................................................................
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
+with expansion:
+  true == true
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false )
+with expansion:
+  false == false
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
+with expansion:
+  true == true
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true )
+with expansion:
+  true == true
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
+with expansion:
+  true == true
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true )
+with expansion:
+  true == true
+
 -------------------------------------------------------------------------------
 Nested generators and captured variables
 -------------------------------------------------------------------------------
@@ -17701,6 +17820,6 @@ Misc.tests.cpp:<line number>
 Misc.tests.cpp:<line number>: PASSED:
 
 ===============================================================================
-test cases:  384 |  291 passed |  86 failed |  7 failed as expected
-assertions: 2213 | 2040 passed | 146 failed | 27 failed as expected
+test cases:  386 |  293 passed |  86 failed |  7 failed as expected
+assertions: 2230 | 2057 passed | 146 failed | 27 failed as expected
 
diff --git a/tests/SelfTest/Baselines/console.sw.multi.approved.txt b/tests/SelfTest/Baselines/console.sw.multi.approved.txt
@@ -7197,6 +7197,125 @@ Exception.tests.cpp:<line number>: FAILED:
 with expansion:
   "expected exception" equals: "should fail"
 
+-------------------------------------------------------------------------------
+Multireporter calls reporters and listeners in correct order
+-------------------------------------------------------------------------------
+Reporters.tests.cpp:<line number>
+...............................................................................
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( records == expected )
+with expansion:
+  { "Hello", "world", "Goodbye", "world" }
+  ==
+  { "Hello", "world", "Goodbye", "world" }
+
+-------------------------------------------------------------------------------
+Multireporter updates ReporterPreferences properly
+-------------------------------------------------------------------------------
+Reporters.tests.cpp:<line number>
+...............................................................................
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == false )
+with expansion:
+  false == false
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false )
+with expansion:
+  false == false
+
+-------------------------------------------------------------------------------
+Multireporter updates ReporterPreferences properly
+  Adding listeners
+-------------------------------------------------------------------------------
+Reporters.tests.cpp:<line number>
+...............................................................................
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
+with expansion:
+  true == true
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false )
+with expansion:
+  false == false
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
+with expansion:
+  true == true
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true )
+with expansion:
+  true == true
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
+with expansion:
+  true == true
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true )
+with expansion:
+  true == true
+
+-------------------------------------------------------------------------------
+Multireporter updates ReporterPreferences properly
+-------------------------------------------------------------------------------
+Reporters.tests.cpp:<line number>
+...............................................................................
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == false )
+with expansion:
+  false == false
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false )
+with expansion:
+  false == false
+
+-------------------------------------------------------------------------------
+Multireporter updates ReporterPreferences properly
+  Adding reporters
+-------------------------------------------------------------------------------
+Reporters.tests.cpp:<line number>
+...............................................................................
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
+with expansion:
+  true == true
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false )
+with expansion:
+  false == false
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
+with expansion:
+  true == true
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true )
+with expansion:
+  true == true
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
+with expansion:
+  true == true
+
+Reporters.tests.cpp:<line number>: PASSED:
+  REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true )
+with expansion:
+  true == true
+
 -------------------------------------------------------------------------------
 Nested generators and captured variables
 -------------------------------------------------------------------------------
@@ -17693,6 +17812,6 @@ Misc.tests.cpp:<line number>
 Misc.tests.cpp:<line number>: PASSED:
 
 ===============================================================================
-test cases:  384 |  291 passed |  86 failed |  7 failed as expected
-assertions: 2213 | 2040 passed | 146 failed | 27 failed as expected
+test cases:  386 |  293 passed |  86 failed |  7 failed as expected
+assertions: 2230 | 2057 passed | 146 failed | 27 failed as expected
 
diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuitesloose text artifact
 >
-  <testsuite name="<exe-name>" errors="17" failures="129" tests="2213" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+  <testsuite name="<exe-name>" errors="17" failures="129" tests="2230" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
     <properties>
       <property name="random-seed" value="1"/>
       <property name="filters" value="~[!nonportable]~[!benchmark]~[approvals] *"/>
@@ -870,6 +870,10 @@ with expansion:
 Exception.tests.cpp:<line number>
       </failure>
     </testcase>
+    <testcase classname="<exe-name>.global" name="Multireporter calls reporters and listeners in correct order" time="{duration}" status="run"/>
+    <testcase classname="<exe-name>.global" name="Multireporter updates ReporterPreferences properly" time="{duration}" status="run"/>
+    <testcase classname="<exe-name>.global" name="Multireporter updates ReporterPreferences properly/Adding listeners" time="{duration}" status="run"/>
+    <testcase classname="<exe-name>.global" name="Multireporter updates ReporterPreferences properly/Adding reporters" time="{duration}" status="run"/>
     <testcase classname="<exe-name>.global" name="Nested generators and captured variables" time="{duration}" status="run"/>
     <testcase classname="<exe-name>.global" name="Nice descriptive name" time="{duration}" status="run"/>
     <testcase classname="<exe-name>.global" name="Non-std exceptions can be translated" time="{duration}" status="run">