test: Improve integration tests (#1719)

Fixes #1616 ## Test Plan > How do we know the code works? All integration tests succeed ## Checklist - [x] Integration tests updated
Flank · Mar 23, 2021 · 756d76c · 756d76c
1 parent fdce797
commit 756d76c
Show file tree

Hide file tree

Showing 32 changed files with 343 additions and 126 deletions.
diff --git a/common/src/main/kotlin/flank/common/OutputReportCostNode.kt b/common/src/main/kotlin/flank/common/OutputReportCostNode.kt
@@ -0,0 +1,9 @@
+package flank.common
+
+import java.math.BigDecimal
+
+data class OutputReportCostNode(
+    val physical: BigDecimal,
+    val virtual: BigDecimal,
+    val total: BigDecimal
+)
diff --git a/common/src/main/kotlin/flank/common/OutputReportKeys.kt b/common/src/main/kotlin/flank/common/OutputReportKeys.kt
@@ -0,0 +1,7 @@
+package flank.common
+
+const val OUTPUT_ARGS = "args"
+const val OUTPUT_WEBLINKS = "weblinks"
+const val OUTPUT_TEST_RESULTS = "test_results"
+const val OUTPUT_COST = "cost"
+const val OUTPUT_ERROR = "error"
diff --git a/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt b/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt
@@ -6,6 +6,17 @@ import flank.common.isWindows
 import org.junit.Assume.assumeFalse
 import org.junit.Test
 import run
+import utils.CONFIGS_PATH
+import utils.FLANK_JAR_PATH
+import utils.androidRunCommands
+import utils.asOutputReport
+import utils.assertExitCode
+import utils.assertNoOutcomeSummary
+import utils.findTestDirectoryFromOutput
+import utils.iosRunCommands
+import utils.json
+import utils.removeUnicode
+import utils.toOutputReportFile
 
 class AllTestFilteredIT {
     private val name = this::class.java.simpleName
@@ -25,8 +36,15 @@ class AllTestFilteredIT {
         assertExitCode(result, 1)
 
         val resOutput = result.output.removeUnicode()
-        assertThat(resOutput).containsMatch(findInCompare(name))
+
+        val outputReport = resOutput.findTestDirectoryFromOutput().toOutputReportFile().json().asOutputReport()
+
         assertNoOutcomeSummary(resOutput)
+
+        assertThat(outputReport.error).contains("There are no tests to run.")
+        assertThat(outputReport.cost).isNull()
+        assertThat(outputReport.testResults).isEmpty()
+        assertThat(outputReport.weblinks).isEmpty()
     }
 
     @Test
@@ -45,7 +63,16 @@ class AllTestFilteredIT {
         assertExitCode(result, 1)
 
         val resOutput = result.output.removeUnicode()
-        assertThat(resOutput).containsMatch(findInCompare(name))
+
         assertNoOutcomeSummary(resOutput)
+
+        val outputReport = resOutput.findTestDirectoryFromOutput().toOutputReportFile().json().asOutputReport()
+
+        assertNoOutcomeSummary(resOutput)
+
+        assertThat(outputReport.error).contains("Empty shards. Cannot match any method to [nonExisting/Class]")
+        assertThat(outputReport.cost).isNull()
+        assertThat(outputReport.testResults).isEmpty()
+        assertThat(outputReport.weblinks).isEmpty()
     }
 }
diff --git a/integration_tests/src/test/kotlin/integration/DumpShardsIT.kt b/integration_tests/src/test/kotlin/integration/DumpShardsIT.kt
@@ -1,14 +1,22 @@
 package integration
 
 import FlankCommand
-import com.google.common.truth.Truth
+import com.google.common.truth.Truth.assertThat
 import flank.common.isWindows
 import org.junit.Assume
 import org.junit.Test
 import run
+import utils.CONFIGS_PATH
+import utils.FLANK_JAR_PATH
+import utils.androidRunCommands
+import utils.assertExitCode
+import utils.assertNoOutcomeSummary
 import utils.containsAll
+import utils.findInCompare
+import utils.iosRunCommands
 import utils.loadAndroidDumpShards
 import utils.loadIosDumpShards
+import utils.removeUnicode
 import java.io.File
 
 class DumpShardsIT {
@@ -29,24 +37,24 @@ class DumpShardsIT {
         assertExitCode(result, 0)
 
         val resOutput = result.output.removeUnicode()
-        Truth.assertThat(resOutput).containsMatch(findInCompare(name))
+        assertThat(resOutput).containsMatch(findInCompare(name))
         assertNoOutcomeSummary(resOutput)
 
         val matrix = File("android_shards.json").loadAndroidDumpShards()
 
-        Truth.assertThat(matrix.shards.count()).isEqualTo(2)
+        assertThat(matrix.shards.count()).isEqualTo(2)
 
-        Truth.assertThat(matrix.shards.values.flatten()).containsAll(
+        assertThat(matrix.shards.values.flatten()).containsAll(
             "class com.example.test_app.parametrized.EspressoParametrizedClassParameterizedNamed",
             "class com.example.test_app.parametrized.EspressoParametrizedClassTestParameterized",
             "class com.example.test_app.ParameterizedTest",
             "class com.example.test_app.parametrized.EspressoParametrizedMethodTestJUnitParamsRunner",
         )
 
-        Truth.assertThat(matrix.junitIgnored.count()).isEqualTo(4)
-        Truth.assertThat(matrix.junitIgnored).containsNoDuplicates()
+        assertThat(matrix.junitIgnored.count()).isEqualTo(4)
+        assertThat(matrix.junitIgnored).containsNoDuplicates()
 
-        Truth.assertThat(matrix.junitIgnored)
+        assertThat(matrix.junitIgnored)
             .containsExactly(
                 "class com.example.test_app.InstrumentedTest#ignoredTestWitSuppress",
                 "class com.example.test_app.InstrumentedTest#ignoredTestWithIgnore",
@@ -71,16 +79,16 @@ class DumpShardsIT {
         assertExitCode(result, 0)
 
         val resOutput = result.output.removeUnicode()
-        Truth.assertThat(resOutput).containsMatch(findInCompare(name))
+        assertThat(resOutput).containsMatch(findInCompare(name))
         assertNoOutcomeSummary(resOutput)
 
         val shards = File("ios_shards.json").loadIosDumpShards()
 
-        Truth.assertThat(shards.count()).isEqualTo(2)
+        assertThat(shards.count()).isEqualTo(2)
 
         shards.first().let { firstShard ->
-            Truth.assertThat(firstShard.count()).isEqualTo(8)
-            Truth.assertThat(firstShard)
+            assertThat(firstShard.count()).isEqualTo(8)
+            assertThat(firstShard)
                 .contains("EarlGreyExampleSwiftTests/testWithCustomFailureHandler")
         }
     }

diff --git a/integration_tests/src/test/kotlin/integration/GameloopIT.kt b/integration_tests/src/test/kotlin/integration/GameloopIT.kt
@@ -1,13 +1,24 @@
 package integration
 
 import FlankCommand
-import com.google.common.truth.Truth
+import com.google.common.truth.Truth.assertThat
 import flank.common.isLinux
 import flank.common.isMacOS
 import flank.common.isWindows
 import org.junit.Assume.assumeFalse
 import org.junit.Test
 import run
+import utils.CONFIGS_PATH
+import utils.FLANK_JAR_PATH
+import utils.androidRunCommands
+import utils.asOutputReport
+import utils.assertCostMatches
+import utils.assertExitCode
+import utils.findTestDirectoryFromOutput
+import utils.iosRunCommands
+import utils.json
+import utils.removeUnicode
+import utils.toOutputReportFile
 
 class GameloopIT {
 
@@ -28,10 +39,19 @@ class GameloopIT {
         assertExitCode(result, 0)
 
         val resOutput = result.output.removeUnicode()
-        Truth.assertThat(resOutput).containsMatch(findInCompare(name))
-        assertContainsOutcomeSummary(resOutput) {
-            success = 1
-        }
+
+        val outputReport = resOutput.findTestDirectoryFromOutput().toOutputReportFile().json().asOutputReport()
+
+        assertThat(outputReport.error).isEmpty()
+        assertThat(outputReport.cost).isNotNull()
+
+        outputReport.assertCostMatches()
+
+        assertThat(outputReport.testResults.count()).isEqualTo(1)
+        assertThat(outputReport.weblinks.count()).isEqualTo(1)
+
+        val testAxis = outputReport.testResults.values.first().testAxises.first()
+        assertThat(testAxis.outcome).isEqualTo("success")
     }
 
     @Test
@@ -52,6 +72,18 @@ class GameloopIT {
         assertExitCode(result, 0)
 
         val resOutput = result.output.removeUnicode()
-        Truth.assertThat(resOutput).containsMatch(findInCompare(name))
+
+        val outputReport = resOutput.findTestDirectoryFromOutput().toOutputReportFile().json().asOutputReport()
+
+        assertThat(outputReport.error).isEmpty()
+        assertThat(outputReport.cost).isNotNull()
+
+        outputReport.assertCostMatches()
+
+        assertThat(outputReport.testResults.count()).isEqualTo(1)
+        assertThat(outputReport.weblinks.count()).isEqualTo(1)
+
+        val testAxis = outputReport.testResults.values.first().testAxises.first()
+        assertThat(testAxis.outcome).isEqualTo("success")
     }
 }
diff --git a/integration_tests/src/test/kotlin/integration/IgnoreFailedIT.kt b/integration_tests/src/test/kotlin/integration/IgnoreFailedIT.kt
@@ -4,9 +4,22 @@ import FlankCommand
 import com.google.common.truth.Truth.assertThat
 import org.junit.Test
 import run
+import utils.CONFIGS_PATH
+import utils.FLANK_JAR_PATH
+import utils.androidRunCommands
+import utils.asOutputReport
+import utils.assertCostMatches
+import utils.assertExitCode
+import utils.assertTestCountMatches
+import utils.findTestDirectoryFromOutput
+import utils.firstTestSuiteOverview
+import utils.json
+import utils.removeUnicode
+import utils.toOutputReportFile
 
 class IgnoreFailedIT {
     private val name = this::class.java.simpleName
+
     @Test
     fun `return with exit code 0 for failed tests`() {
         val result = FlankCommand(
@@ -21,9 +34,21 @@ class IgnoreFailedIT {
         assertExitCode(result, 0)
 
         val resOutput = result.output.removeUnicode()
-        assertThat(resOutput).containsMatch(findInCompare(name))
-        assertContainsOutcomeSummary(resOutput) {
-            failure = 1
-        }
+        val outputReport = resOutput.findTestDirectoryFromOutput().toOutputReportFile().json().asOutputReport()
+
+        assertThat(outputReport.error).isEmpty()
+        assertThat(outputReport.cost).isNotNull()
+
+        outputReport.assertCostMatches()
+
+        assertThat(outputReport.testResults.count()).isEqualTo(1)
+        assertThat(outputReport.weblinks.count()).isEqualTo(1)
+
+        val testSuiteOverview = outputReport.firstTestSuiteOverview
+
+        testSuiteOverview.assertTestCountMatches(
+            total = 1,
+            failures = 1
+        )
     }
 }
diff --git a/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt b/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt
@@ -4,12 +4,24 @@ import FlankCommand
 import com.google.common.truth.Truth.assertThat
 import org.junit.Test
 import run
+import utils.CONFIGS_PATH
+import utils.FLANK_JAR_PATH
+import utils.androidRunCommands
+import utils.asOutputReport
+import utils.assertContainsUploads
+import utils.assertCostMatches
+import utils.assertExitCode
 import utils.assertTestFail
 import utils.assertTestPass
 import utils.assertTestResultContainsWebLinks
 import utils.findTestDirectoryFromOutput
+import utils.json
 import utils.loadAsTestSuite
+import utils.multipleFailedTests
+import utils.multipleSuccessfulTests
+import utils.removeUnicode
 import utils.toJUnitXmlFile
+import utils.toOutputReportFile
 
 class MultipleApksIT {
     private val name = this::class.java.simpleName
@@ -25,23 +37,36 @@ class MultipleApksIT {
         assertExitCode(result, 10)
 
         val resOutput = result.output.removeUnicode()
-        assertThat(resOutput).containsMatch(findInCompare(name))
+
         assertContainsUploads(
             resOutput,
             "app-multiple-success-debug-androidTest.apk",
             "app-multiple-error-debug-androidTest.apk",
             "MainActivity_robo_script.json"
         )
 
-        assertContainsOutcomeSummary(resOutput) {
-            success = 3
-            failure = 1
-        }
-
         resOutput.findTestDirectoryFromOutput().toJUnitXmlFile().loadAsTestSuite().run {
             assertTestResultContainsWebLinks()
             assertTestPass(multipleSuccessfulTests)
             assertTestFail(multipleFailedTests)
         }
+
+        val outputReport = resOutput.findTestDirectoryFromOutput().toOutputReportFile().json().asOutputReport()
+
+        assertThat(outputReport.error).isEmpty()
+        assertThat(outputReport.cost).isNotNull()
+
+        outputReport.assertCostMatches()
+
+        assertThat(outputReport.testResults.count()).isEqualTo(4)
+        assertThat(outputReport.weblinks.count()).isEqualTo(4)
+
+        val testsResults = outputReport.testResults
+            .map { it.value }
+            .map { it.testAxises }
+            .flatten()
+
+        assertThat(testsResults.sumBy { it.testSuiteOverview.failures }).isEqualTo(5)
+        assertThat(testsResults.sumBy { it.testSuiteOverview.total }).isEqualTo(41)
     }
 }
diff --git a/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt b/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt
@@ -4,12 +4,16 @@ import FlankCommand
 import com.google.common.truth.Truth.assertThat
 import org.junit.Test
 import run
-import utils.assertTestFail
-import utils.assertTestPass
-import utils.assertTestResultContainsWebLinks
+import utils.CONFIGS_PATH
+import utils.FLANK_JAR_PATH
+import utils.androidRunCommands
+import utils.asOutputReport
+import utils.assertCostMatches
+import utils.assertExitCode
 import utils.findTestDirectoryFromOutput
-import utils.loadAsTestSuite
-import utils.toJUnitXmlFile
+import utils.json
+import utils.removeUnicode
+import utils.toOutputReportFile
 
 class MultipleDevicesIT {
     private val name = this::class.java.simpleName
@@ -29,20 +33,19 @@ class MultipleDevicesIT {
         assertExitCode(result, 10)
 
         val resOutput = result.output.removeUnicode()
-        assertThat(resOutput).containsMatch(findInCompare(name))
-        assertContainsUploads(
-            resOutput,
-            "app-multiple-success-debug-androidTest.apk",
-            "app-multiple-error-debug-androidTest.apk",
-        )
-        assertContainsOutcomeSummary(resOutput) {
-            success = 6
-            failure = 3
-        }
-        resOutput.findTestDirectoryFromOutput().toJUnitXmlFile().loadAsTestSuite().run {
-            assertTestResultContainsWebLinks()
-            assertTestPass(multipleSuccessfulTests)
-            assertTestFail(multipleFailedTests)
-        }
+
+        val outputReport = resOutput.findTestDirectoryFromOutput().toOutputReportFile().json().asOutputReport()
+        assertThat(outputReport.weblinks.size).isEqualTo(3)
+        assertThat(outputReport.error).isEmpty()
+
+        outputReport.assertCostMatches()
+
+        val testsResults = outputReport.testResults
+            .map { it.value }
+            .map { it.testAxises }
+            .flatten()
+
+        assertThat(testsResults.sumBy { it.testSuiteOverview.failures }).isEqualTo(15)
+        assertThat(testsResults.sumBy { it.testSuiteOverview.total }).isEqualTo(123)
     }
 }