From 974a807c5c9dd04d658b91f6621accf97cff37ac Mon Sep 17 00:00:00 2001
From: Marc Philipp <mail@marcphilipp.de>
Date: Sat, 19 Aug 2017 20:47:11 +0200
Subject: [PATCH] Surefire: Report tests with non-AssertionError causes as
 errors

Fixes #1019.
---
 .../src/docs/asciidoc/release-notes-5.0.0-RC3.adoc   |  2 ++
 .../surefire/provider/RunListenerAdapter.java        | 12 +++++++++++-
 .../surefire/provider/RunListenerAdapterTests.java   | 12 +++++++++---
 3 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/documentation/src/docs/asciidoc/release-notes-5.0.0-RC3.adoc b/documentation/src/docs/asciidoc/release-notes-5.0.0-RC3.adoc
index 786d0546d2e1..fa36921f1479 100644
--- a/documentation/src/docs/asciidoc/release-notes-5.0.0-RC3.adoc
+++ b/documentation/src/docs/asciidoc/release-notes-5.0.0-RC3.adoc
@@ -20,6 +20,8 @@ on GitHub.
 ===== Bug Fixes
 
 * Source JARs no longer contain every source file twice.
+* The Surefire provider now reports failed tests with a cause that is not instance of
+  `AssertionError` as errors instead of failures for compatibility reasons.
 
 ===== New Features and Improvements
 
diff --git a/junit-platform-surefire-provider/src/main/java/org/junit/platform/surefire/provider/RunListenerAdapter.java b/junit-platform-surefire-provider/src/main/java/org/junit/platform/surefire/provider/RunListenerAdapter.java
index 9f7ccc4f9ab9..716b7abce6a0 100644
--- a/junit-platform-surefire-provider/src/main/java/org/junit/platform/surefire/provider/RunListenerAdapter.java
+++ b/junit-platform-surefire-provider/src/main/java/org/junit/platform/surefire/provider/RunListenerAdapter.java
@@ -75,13 +75,23 @@ public void executionFinished(TestIdentifier testIdentifier, TestExecutionResult
 			runListener.testAssumptionFailure(createReportEntry(testIdentifier, testExecutionResult.getThrowable()));
 		}
 		else if (testExecutionResult.getStatus() == FAILED) {
-			runListener.testFailed(createReportEntry(testIdentifier, testExecutionResult.getThrowable()));
+			reportFailedTest(testIdentifier, testExecutionResult.getThrowable());
 		}
 		else if (testIdentifier.isTest()) {
 			runListener.testSucceeded(createReportEntry(testIdentifier, Optional.empty()));
 		}
 	}
 
+	private void reportFailedTest(TestIdentifier testIdentifier, Optional<Throwable> throwable) {
+		SimpleReportEntry reportEntry = createReportEntry(testIdentifier, throwable);
+		if (throwable.filter(AssertionError.class::isInstance).isPresent()) {
+			runListener.testFailed(reportEntry);
+		}
+		else {
+			runListener.testError(reportEntry);
+		}
+	}
+
 	private SimpleReportEntry createReportEntry(TestIdentifier testIdentifier, Optional<Throwable> throwable) {
 		Optional<String> className = getClassName(testIdentifier);
 
diff --git a/junit-platform-surefire-provider/src/test/java/org/junit/platform/surefire/provider/RunListenerAdapterTests.java b/junit-platform-surefire-provider/src/test/java/org/junit/platform/surefire/provider/RunListenerAdapterTests.java
index 36682fdbaf10..ccab8a373409 100644
--- a/junit-platform-surefire-provider/src/test/java/org/junit/platform/surefire/provider/RunListenerAdapterTests.java
+++ b/junit-platform-surefire-provider/src/test/java/org/junit/platform/surefire/provider/RunListenerAdapterTests.java
@@ -133,11 +133,17 @@ void notifiedWhenClassExecutionAborted() throws Exception {
 	}
 
 	@Test
-	void notifiedWhenMethodExecutionFailed() throws Exception {
-		adapter.executionFinished(newMethodIdentifier(), TestExecutionResult.failed(new RuntimeException()));
+	void notifiedWhenMethodExecutionFailedWithAnAssertionError() throws Exception {
+		adapter.executionFinished(newMethodIdentifier(), TestExecutionResult.failed(new AssertionError()));
 		verify(listener).testFailed(any());
 	}
 
+	@Test
+	void notifiedWhenMethodExecutionFailedWithANonAssertionError() throws Exception {
+		adapter.executionFinished(newMethodIdentifier(), TestExecutionResult.failed(new RuntimeException()));
+		verify(listener).testError(any());
+	}
+
 	@Test
 	void notifiedWithCorrectNamesWhenClassExecutionFailed() throws Exception {
 		ArgumentCaptor<ReportEntry> entryCaptor = ArgumentCaptor.forClass(ReportEntry.class);
@@ -145,7 +151,7 @@ void notifiedWithCorrectNamesWhenClassExecutionFailed() throws Exception {
 		adapter.testPlanExecutionStarted(testPlan);
 
 		adapter.executionFinished(identifiersAsParentOnTestPlan(testPlan, newEngineDescriptor(), newClassDescriptor()),
-			TestExecutionResult.failed(new RuntimeException()));
+			TestExecutionResult.failed(new AssertionError()));
 		verify(listener).testFailed(entryCaptor.capture());
 
 		ReportEntry entry = entryCaptor.getValue();