From 93da9354c00185a732e48c443bfbdf3f03bbe868 Mon Sep 17 00:00:00 2001 From: bootstraponline Date: Sun, 4 Aug 2019 21:20:57 -0400 Subject: [PATCH] Poll all test executions --- .../src/main/kotlin/ftl/run/RunningDevices.kt | 74 +++++++++++++++++++ .../src/main/kotlin/ftl/run/TestRunner.kt | 69 ++++------------- .../main/kotlin/ftl/util/StopWatchMatrix.kt | 11 +++ 3 files changed, 99 insertions(+), 55 deletions(-) create mode 100644 test_runner/src/main/kotlin/ftl/run/RunningDevices.kt create mode 100644 test_runner/src/main/kotlin/ftl/util/StopWatchMatrix.kt diff --git a/test_runner/src/main/kotlin/ftl/run/RunningDevices.kt b/test_runner/src/main/kotlin/ftl/run/RunningDevices.kt new file mode 100644 index 0000000000..b73ebb5a5e --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/RunningDevices.kt @@ -0,0 +1,74 @@ +package ftl.run + +import com.google.api.services.testing.model.Environment +import com.google.api.services.testing.model.TestDetails +import com.google.api.services.testing.model.TestExecution +import com.google.api.services.testing.model.TestMatrix +import ftl.util.MatrixState +import ftl.util.StopWatch +import ftl.util.StopWatchMatrix + +class RunningDevice(private val stopwatch: StopWatch, val id: String) { + var lastState = "" + var lastError = "" + var progress = listOf() + var lastProgressLen = 0 + val details: TestDetails? = null + var complete = false + + private fun device(testExecution: TestExecution): String { + val env: Environment? = testExecution.environment + val device = env?.androidDevice?.androidModelId ?: env?.iosDevice?.iosModelId + val deviceVersion = env?.androidDevice?.androidVersionId ?: env?.iosDevice?.iosVersionId + return "$device-$deviceVersion" + } + + fun poll(matrix: TestMatrix) { + val testExecution = matrix.testExecutions.single { it.id == id } + val watch = StopWatchMatrix(stopwatch, "${testExecution.matrixId} ${device(testExecution)}") + val details: TestDetails? = testExecution.testDetails + + if (details != null) { + // Error message is never reset. Track last error to only print new messages. + val errorMessage = details.errorMessage + if ( + errorMessage != null && + errorMessage != lastError + ) { + // Note: After an error (infrastructure failure), FTL will retry 3x + lastError = errorMessage + watch.puts("Error: $lastError") + } + progress = details.progressMessages ?: progress + } + + // flaky-test-attempts restarts progress array at size 1 + if (lastProgressLen > progress.size) { + lastProgressLen = 0 + } + + // Progress contains all messages. only print new updates + for (msg in progress.listIterator(lastProgressLen)) { + watch.puts(msg) + } + + lastProgressLen = progress.size + + if (testExecution.state != lastState) { + lastState = testExecution.state + watch.puts(lastState) + } + + if (MatrixState.completed(testExecution.state)) { + complete = true + } + } +} + +class RunningDevices(stopwatch: StopWatch, testExecutions: List) { + private val devices = testExecutions.map { RunningDevice(stopwatch, it.id) } + + fun next(): RunningDevice? = devices.firstOrNull { it.complete.not() } + + fun allComplete(): Boolean = devices.all { it.complete } +} diff --git a/test_runner/src/main/kotlin/ftl/run/TestRunner.kt b/test_runner/src/main/kotlin/ftl/run/TestRunner.kt index 93bb1f1507..01bb1eab48 100644 --- a/test_runner/src/main/kotlin/ftl/run/TestRunner.kt +++ b/test_runner/src/main/kotlin/ftl/run/TestRunner.kt @@ -1,6 +1,5 @@ package ftl.run -import com.google.api.services.testing.model.TestDetails import com.google.api.services.testing.model.TestMatrix import com.google.cloud.storage.Storage import com.google.gson.GsonBuilder @@ -25,6 +24,7 @@ import ftl.util.Artifacts import ftl.util.MatrixState import ftl.util.ObjPath import ftl.util.StopWatch +import ftl.util.StopWatchMatrix import ftl.util.Utils import ftl.util.Utils.fatalError import java.nio.file.Files @@ -146,8 +146,8 @@ object TestRunner { if (!resultsFile.exists()) return null val scheduledRuns = resultsFile.listFiles() - .filter { it.isDirectory } - .sortedByDescending { it.lastModified() } + .filter { it.isDirectory } + .sortedByDescending { it.lastModified() } if (scheduledRuns.isEmpty()) return null return scheduledRuns.first().name @@ -282,70 +282,29 @@ object TestRunner { // Port of MonitorTestExecutionProgress // gcloud-cli/googlecloudsdk/api_lib/firebase/test/matrix_ops.py private fun pollMatrix(matrixId: String, stopwatch: StopWatch, args: IArgs, matrices: MatrixMap) { - var lastState = "" - var lastError = "" - var progress = listOf() - var lastProgressLen = 0 - var refreshedMatrix: TestMatrix - - fun puts(msg: String) { - val timestamp = stopwatch.check(indent = true) - println("$indent$timestamp $matrixId $msg") - } + var refreshedMatrix = GcTestMatrix.refresh(matrixId, args) + val watch = StopWatchMatrix(stopwatch, matrixId) + val runningDevices = RunningDevices(stopwatch, refreshedMatrix.testExecutions) while (true) { - refreshedMatrix = GcTestMatrix.refresh(matrixId, args) - // Update the matrix file when the matrix is updated if (matrices.map[matrixId]?.update(refreshedMatrix) == true) updateMatrixFile(matrices, args) - val firstTestStatus = refreshedMatrix.testExecutions.first() - - val details: TestDetails? = firstTestStatus.testDetails - if (details != null) { - // Error message is never reset. Track last error to only print new messages. - val errorMessage = details.errorMessage - if ( - errorMessage != null && - errorMessage != lastError - ) { - // Note: After an error (infrastructure failure), FTL will retry 3x - lastError = errorMessage - puts("Error: $lastError") - } - progress = details.progressMessages ?: progress - } + val nextDevice = runningDevices.next() ?: return + nextDevice.poll(refreshedMatrix) - if (MatrixState.completed(refreshedMatrix.state)) { + // Matrix has 0 or more devices (test executions) + if (runningDevices.allComplete()) { break } - // flaky-test-attempts restarts progress array at size 1 - if (lastProgressLen > progress.size) { - lastProgressLen = 0 - } - - // Progress contains all messages. only print new updates - for (msg in progress.listIterator(lastProgressLen)) { - puts(msg) - // There may be significant time lag between 'Done' and the matrix actually finishing. - if (msg.contains("Done. Test time=")) { - puts("Waiting for post-processing service to finish") - } - } - lastProgressLen = progress.size - - if (firstTestStatus.state != lastState) { - lastState = firstTestStatus.state - puts(lastState) - } - // GetTestMatrix is not designed to handle many requests per second. - // Sleep 15s to avoid overloading the system. + // Sleep to avoid overloading the system. Utils.sleep(15) + refreshedMatrix = GcTestMatrix.refresh(matrixId, args) } // Print final matrix state with timestamp. May be many minutes after the 'Done.' progress message. - puts(refreshedMatrix.state) + watch.puts(refreshedMatrix.state) } // used to update and poll the results from an async run @@ -379,7 +338,7 @@ object TestRunner { fetchArtifacts(matrixMap, args) val exitCode = ReportManager.generate(matrixMap, args, testShardChunks) - System.exit(exitCode) + exitProcess(exitCode) } } } diff --git a/test_runner/src/main/kotlin/ftl/util/StopWatchMatrix.kt b/test_runner/src/main/kotlin/ftl/util/StopWatchMatrix.kt new file mode 100644 index 0000000000..7379ebde41 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/util/StopWatchMatrix.kt @@ -0,0 +1,11 @@ +package ftl.util + +import ftl.config.FtlConstants + +class StopWatchMatrix(private val stopwatch: StopWatch, private val matrixId: String) { + + fun puts(msg: String) { + val timestamp = stopwatch.check(indent = true) + println("${FtlConstants.indent}$timestamp $matrixId $msg") + } +}