Use matrix outcome to set exit code

Flank · Oct 8, 2019 · cd8b77f · cd8b77f
1 parent 060a15b
commit cd8b77f
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 14 deletions.
diff --git a/release_notes.md b/release_notes.md
@@ -1,6 +1,9 @@
 ## next (unreleased)
 
 - [#595](https://github.com/TestArmada/flank/pull/595) Rename `flaky-test-attempts` to `num-flaky-test-attempts`. Rename `repeat-tests` to `num-test-runs`. ([bootstraponline](https://github.com/bootstraponline))
+- [#605](https://github.com/TestArmada/flank/pull/605) Improve exit code logging. Use matrix outcome to set exit code. ([bootstraponline](https://github.com/bootstraponline))
+- [#597](https://github.com/TestArmada/flank/pull/597) Support parsing testLabExecutionId. [yogurtearl](https://github.com/yogurtearl)
+- [#599](https://github.com/TestArmada/flank/pull/599) Disable FAIL_ON_UNKNOWN_PROPERTIES for forward compatibility. [narenkmanoharan](https://github.com/narenkmanoharan)
 
 ## v7.0.2
 

diff --git a/test_runner/src/main/kotlin/ftl/json/MatrixMap.kt b/test_runner/src/main/kotlin/ftl/json/MatrixMap.kt
@@ -36,16 +36,20 @@ class MatrixMap(
         var exitCode = 0
 
         map.values.forEach { matrix ->
-            if (matrix.state != MatrixState.FINISHED)  {
-                println("Error: Matrix not finished: ${matrix.matrixId} ${matrix.state}")
+            if (matrix.state != MatrixState.FINISHED) {
+                matrix.logError("not finished")
                 return 2
             }
             if (matrix.failed()) {
-                println("Error: Matrix failed: ${matrix.matrixId} ${matrix.state}")
+                matrix.logError("failed")
                 exitCode = 1
             }
         }
 
         return exitCode
     }
+
+    private fun SavedMatrix.logError(message: String) {
+        println("Error: Matrix $message: ${this.matrixId} ${this.state} ${this.webLink}")
+    }
 }
diff --git a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt
@@ -97,16 +97,14 @@ object ReportManager {
         val useFlakyTests = args.flakyTestAttempts > 0
         if (useFlakyTests) JUnitDedupe.modify(testSuite)
 
-        val testSuccessful = if (useFlakyTests) testSuite?.successful() ?: false else matrices.allSuccessful()
-
         listOf(
             CostReport,
             MatrixResultsReport
         ).map {
             it.run(matrices, testSuite, printToStdout = true, args = args)
         }
 
-        if (!testSuccessful) {
+        if (!matrices.allSuccessful()) {
             listOf(
                 HtmlErrorReport
             ).map { it.run(matrices, testSuite, printToStdout = false, args = args) }
@@ -115,14 +113,7 @@ object ReportManager {
         JUnitReport.run(matrices, testSuite, printToStdout = false, args = args)
         processJunitXml(testSuite, args, testShardChunks)
 
-        // FTL has a bug with matrix roll-up when using flakyTestAttempts
-        // as a work around, we calculate the success based on JUnit XML results.
-        val exitCode = if (useFlakyTests) {
-            if (testSuccessful) 0 else 1
-        } else {
-            matrices.exitCode()
-        }
-        return exitCode
+        return matrices.exitCode()
     }
 
     data class ShardEfficiency(