From e3ea3f43050b8357564ca18ce95d152ee8cf5fd3 Mon Sep 17 00:00:00 2001 From: bootstraponline Date: Fri, 15 Mar 2019 16:20:50 -0400 Subject: [PATCH] Add --local-result-dir Use exit code 3 on fatal error instead of -1 --- README.md | 6 ++ release_notes.md | 1 + test_runner/flank.ios.yml | 3 + test_runner/flank.yml | 4 ++ .../src/main/kotlin/ftl/args/AndroidArgs.kt | 7 +++ test_runner/src/main/kotlin/ftl/args/IArgs.kt | 1 + .../src/main/kotlin/ftl/args/IosArgs.kt | 2 + .../kotlin/ftl/args/yml/AndroidGcloudYml.kt | 2 +- .../src/main/kotlin/ftl/args/yml/FlankYml.kt | 9 ++- .../kotlin/ftl/cli/firebase/CancelCommand.kt | 3 +- .../kotlin/ftl/cli/firebase/RefreshCommand.kt | 3 +- .../test/android/AndroidRunCommand.kt | 6 ++ .../cli/firebase/test/ios/IosRunCommand.kt | 6 ++ .../main/kotlin/ftl/config/FtlConstants.kt | 2 - .../src/main/kotlin/ftl/reports/CostReport.kt | 9 +-- .../kotlin/ftl/reports/HtmlErrorReport.kt | 5 +- .../main/kotlin/ftl/reports/JUnitReport.kt | 9 +-- .../kotlin/ftl/reports/MatrixResultsReport.kt | 9 +-- .../main/kotlin/ftl/reports/util/IReport.kt | 7 ++- .../kotlin/ftl/reports/util/ReportManager.kt | 18 +++--- .../main/kotlin/ftl/run/GenericTestRunner.kt | 24 ++++++- .../src/main/kotlin/ftl/run/TestRunner.kt | 63 +++++++++---------- .../src/main/kotlin/ftl/util/MatrixUtil.kt | 5 +- test_runner/src/main/kotlin/ftl/util/Utils.kt | 2 +- .../test/kotlin/ftl/args/AndroidArgsTest.kt | 20 +++++- .../src/test/kotlin/ftl/args/IosArgsTest.kt | 17 +++++ .../ftl/cli/firebase/CancelCommandTest.kt | 4 +- .../ftl/cli/firebase/RefreshCommandTest.kt | 15 ++++- .../test/android/AndroidRunCommandTest.kt | 13 +++- .../firebase/test/ios/IosRunCommandTest.kt | 14 ++++- .../src/test/kotlin/ftl/fixtures/android.yml | 3 + .../src/test/kotlin/ftl/fixtures/ios.yml | 3 + .../ftl/reports/utils/ReportManagerTest.kt | 4 +- .../test/kotlin/ftl/util/MatrixUtilTest.kt | 5 +- 34 files changed, 221 insertions(+), 83 deletions(-) create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/android.yml create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/ios.yml diff --git a/README.md b/README.md index 0d4ead4e66..572c887ec9 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,9 @@ flank: ## The billing enabled Google Cloud Platform project name to use # project: delta-essence-114723 + + ## Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved. + # local-result-dir: flank ``` ### Android example @@ -285,6 +288,9 @@ flank: ## The billing enabled Google Cloud Platform project name to use # project: delta-essence-114723 + + ## Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved. + # local-result-dir: flank ``` ### Android code coverage diff --git a/release_notes.md b/release_notes.md index c72dc37f3a..41ac72a6a8 100644 --- a/release_notes.md +++ b/release_notes.md @@ -5,6 +5,7 @@ - [#514](https://github.com/TestArmada/flank/pull/514) Rename `testShards` to `maxTestShards` ([miguelslemos](https://github.com/miguelslemos)) - [#518](https://github.com/TestArmada/flank/pull/518) Add deprecation warnings when old key names are used. `flank android doctor --fix` will auto fix the YAML file. ([bootstraponline](https://github.com/bootstraponline)) - [#519](https://github.com/TestArmada/flank/pull/519) Rename `maxTestShards` to `max-test-shards`, `shardTime` to `shard-time`, `repeatTests` to `repeat-tests`, `smartFlankGcsPath` to `smart-flank-gcs-path`, `disableSharding` to `disable-sharding`. Moved `project` from `gcloud` to `flank` ([bootstraponline](https://github.com/bootstraponline)) +- [#523](https://github.com/TestArmada/flank/pull/523) Add `--local-result-dir` to make it easy to find the test result at a fixed path. ([bootstraponline](https://github.com/bootstraponline)) ## v4.4.0 diff --git a/test_runner/flank.ios.yml b/test_runner/flank.ios.yml index 435bb8cb21..2705bd3130 100644 --- a/test_runner/flank.ios.yml +++ b/test_runner/flank.ios.yml @@ -102,3 +102,6 @@ flank: ## The billing enabled Google Cloud Platform project name to use # project: delta-essence-114723 + + ## Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved. + # local-result-dir: flank diff --git a/test_runner/flank.yml b/test_runner/flank.yml index d5f0cce66f..fa88e67848 100644 --- a/test_runner/flank.yml +++ b/test_runner/flank.yml @@ -121,3 +121,7 @@ flank: ## The billing enabled Google Cloud Platform project name to use # project: delta-essence-114723 + + ## Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved. + # local-result-dir: flank + \ No newline at end of file diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index cf79898023..be321bc3d7 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -19,6 +19,7 @@ import ftl.args.ArgsToString.devicesToString import ftl.args.ArgsToString.listToString import ftl.args.ArgsToString.mapToString import ftl.args.yml.AndroidGcloudYml +import ftl.args.yml.AndroidGcloudYmlParams import ftl.args.yml.FlankYml import ftl.args.yml.GcloudYml import ftl.args.yml.YamlDeprecated @@ -72,6 +73,7 @@ class AndroidArgs( override val filesToDownload = cli?.filesToDownload ?: flank.filesToDownload override val disableSharding = cli?.disableSharding ?: flank.disableSharding override val project = cli?.project ?: flank.project + override val localResultDir = cli?.localResultDir ?: flank.localResultDir // computed properties not specified in yaml override val testShardChunks: List> by lazy { @@ -173,6 +175,7 @@ ${listToString(filesToDownload)} ${listToString(testTargetsAlwaysRun)} disable-sharding: $disableSharding project: $project + local-result-dir: $localResultDir """.trimIndent() } @@ -199,5 +202,9 @@ ${listToString(testTargetsAlwaysRun)} cli ) } + + fun default(): AndroidArgs { + return AndroidArgs(GcloudYml(), AndroidGcloudYml(AndroidGcloudYmlParams(app = ".", test = ".")), FlankYml(), "", AndroidRunCommand()) + } } } diff --git a/test_runner/src/main/kotlin/ftl/args/IArgs.kt b/test_runner/src/main/kotlin/ftl/args/IArgs.kt index 925674fb46..97c3920d85 100644 --- a/test_runner/src/main/kotlin/ftl/args/IArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IArgs.kt @@ -22,6 +22,7 @@ interface IArgs { val testTargetsAlwaysRun: List val filesToDownload: List val disableSharding: Boolean + val localResultDir: String? // computed property val testShardChunks: List> diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index 49b7637875..3dc74891e5 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -56,6 +56,7 @@ class IosArgs( override val filesToDownload = cli?.filesToDownload ?: flank.filesToDownload override val disableSharding = cli?.disableSharding ?: flank.disableSharding override val project = cli?.project ?: flank.project + override val localResultDir = cli?.localResultsDir ?: flank.localResultDir private val iosFlank = iosFlankYml.flank val testTargets = cli?.testTargets ?: iosFlank.testTargets @@ -132,6 +133,7 @@ ${listToString(filesToDownload)} ${listToString(testTargets)} disable-sharding: $disableSharding project: $project + local-result-dir: $localResultDir """.trimIndent() } diff --git a/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt index cbb934eaad..5d727c8034 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt @@ -52,7 +52,7 @@ class AndroidGcloudYmlParams( @JsonIgnoreProperties(ignoreUnknown = true) class AndroidGcloudYml( - val gcloud: AndroidGcloudYmlParams + val gcloud: AndroidGcloudYmlParams = AndroidGcloudYmlParams() ) { companion object : IYmlMap { diff --git a/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt index 7cf41a6ae3..edfe4be219 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt @@ -32,13 +32,18 @@ class FlankYmlParams( @field:JsonProperty("files-to-download") val filesToDownload: List = emptyList(), - val project: String = ArgsHelper.getDefaultProjectId() ?: "" + val project: String = ArgsHelper.getDefaultProjectId() ?: "", + + @field:JsonProperty("local-result-dir") + val localResultDir: String = defaultLocalResultDir ) { companion object : IYmlKeys { override val keys = listOf( "max-test-shards", "shard-time", "repeat-tests", "smart-flank-gcs-path", "disable-sharding", - "test-targets-always-run", "files-to-download", "project" + "test-targets-always-run", "files-to-download", "project", "local-result-dir" ) + + const val defaultLocalResultDir = "results" } init { diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/CancelCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/CancelCommand.kt index e3ed8bd512..72f64918c2 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/CancelCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/CancelCommand.kt @@ -1,5 +1,6 @@ package ftl.cli.firebase +import ftl.args.AndroidArgs import ftl.run.TestRunner import picocli.CommandLine @@ -18,7 +19,7 @@ Reads in the matrix_ids.json file. Cancels any incomplete matrices. ) class CancelCommand : Runnable { override fun run() { - TestRunner.cancelLastRun() + TestRunner.cancelLastRun(AndroidArgs.default()) } @CommandLine.Option(names = ["-h", "--help"], usageHelp = true, description = ["Prints this help message"]) diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/RefreshCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/RefreshCommand.kt index e71b20486c..e0bb83b9c9 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/RefreshCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/RefreshCommand.kt @@ -1,5 +1,6 @@ package ftl.cli.firebase +import ftl.args.AndroidArgs import ftl.run.TestRunner import kotlinx.coroutines.runBlocking import picocli.CommandLine.Command @@ -21,7 +22,7 @@ Reads in the matrix_ids.json file. Refreshes any incomplete matrices. class RefreshCommand : Runnable { override fun run() { runBlocking { - TestRunner.refreshLastRun() + TestRunner.refreshLastRun(AndroidArgs.default()) } } diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt index 5f099e4529..7ad768a039 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt @@ -273,4 +273,10 @@ class AndroidRunCommand : Runnable { "fail for any reason. The maximum number of reruns allowed is 10. Default is 0, which implies no reruns."] ) var flakyTestAttempts: Int? = null + + @Option( + names = ["--local-result-dir"], + description = ["Saves test result to this local folder. Deleted before each run."] + ) + var localResultDir: String? = null } diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt index 6e8ab92c94..a9966a4aa9 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt @@ -214,4 +214,10 @@ class IosRunCommand : Runnable { "fail for any reason. The maximum number of reruns allowed is 10. Default is 0, which implies no reruns."] ) var flakyTestAttempts: Int? = null + + @Option( + names = ["--local-result-dir"], + description = ["Saves test result to this local folder. Deleted before each run."] + ) + var localResultsDir: String? = null } diff --git a/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt b/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt index 9326c7c7f4..1f9315a259 100644 --- a/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt +++ b/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt @@ -90,8 +90,6 @@ object FtlConstants { } } - const val localResultsDir = "results" - fun configFileName(args: IArgs): String { return when (args) { is IosArgs -> defaultIosConfig diff --git a/test_runner/src/main/kotlin/ftl/reports/CostReport.kt b/test_runner/src/main/kotlin/ftl/reports/CostReport.kt index bca1764196..66b35d6ad5 100644 --- a/test_runner/src/main/kotlin/ftl/reports/CostReport.kt +++ b/test_runner/src/main/kotlin/ftl/reports/CostReport.kt @@ -1,5 +1,6 @@ package ftl.reports +import ftl.args.IArgs import ftl.config.FtlConstants.indent import ftl.json.MatrixMap import ftl.reports.util.IReport @@ -36,14 +37,14 @@ object CostReport : IReport { } } - private fun write(matrices: MatrixMap, output: String) { - val reportPath = reportPath(matrices) + private fun write(matrices: MatrixMap, output: String, args: IArgs) { + val reportPath = reportPath(matrices, args) reportPath.write(output) } - override fun run(matrices: MatrixMap, testSuite: JUnitTestResult?, printToStdout: Boolean) { + override fun run(matrices: MatrixMap, testSuite: JUnitTestResult?, printToStdout: Boolean, args: IArgs) { val output = generate(matrices) if (printToStdout) print(output) - write(matrices, output) + write(matrices, output, args) } } diff --git a/test_runner/src/main/kotlin/ftl/reports/HtmlErrorReport.kt b/test_runner/src/main/kotlin/ftl/reports/HtmlErrorReport.kt index 88d80c4e3d..ff43779248 100644 --- a/test_runner/src/main/kotlin/ftl/reports/HtmlErrorReport.kt +++ b/test_runner/src/main/kotlin/ftl/reports/HtmlErrorReport.kt @@ -1,6 +1,7 @@ package ftl.reports import com.google.gson.Gson +import ftl.args.IArgs import ftl.json.MatrixMap import ftl.reports.util.IReport import ftl.reports.xml.model.JUnitTestCase @@ -77,7 +78,7 @@ object HtmlErrorReport : IReport { return groupJson to itemJson } - override fun run(matrices: MatrixMap, testSuite: JUnitTestResult?, printToStdout: Boolean) { + override fun run(matrices: MatrixMap, testSuite: JUnitTestResult?, printToStdout: Boolean, args: IArgs) { if (testSuite == null) return val reactJson = reactJson(testSuite) ?: return val newGroupJson = reactJson.first @@ -88,7 +89,7 @@ object HtmlErrorReport : IReport { templateData = replaceRange(templateData, findGroupRange(templateData), newGroupJson) templateData = replaceRange(templateData, findItemRange(templateData), newItemsJson) - val writePath = Paths.get(reportPath(matrices)) + val writePath = Paths.get(reportPath(matrices, args)) Files.write(writePath, templateData.toByteArray()) } diff --git a/test_runner/src/main/kotlin/ftl/reports/JUnitReport.kt b/test_runner/src/main/kotlin/ftl/reports/JUnitReport.kt index f8534071b3..688373c57d 100644 --- a/test_runner/src/main/kotlin/ftl/reports/JUnitReport.kt +++ b/test_runner/src/main/kotlin/ftl/reports/JUnitReport.kt @@ -1,5 +1,6 @@ package ftl.reports +import ftl.args.IArgs import ftl.json.MatrixMap import ftl.reports.util.IReport import ftl.reports.xml.model.JUnitTestResult @@ -10,18 +11,18 @@ import ftl.util.Utils.write object JUnitReport : IReport { override val extension = ".xml" - private fun write(matrices: MatrixMap, output: String) { - val reportPath = reportPath(matrices) + private fun write(matrices: MatrixMap, output: String, args: IArgs) { + val reportPath = reportPath(matrices, args) reportPath.write(output) } - override fun run(matrices: MatrixMap, testSuite: JUnitTestResult?, printToStdout: Boolean) { + override fun run(matrices: MatrixMap, testSuite: JUnitTestResult?, printToStdout: Boolean, args: IArgs) { val output = testSuite.xmlToString() if (printToStdout) { print(output) } else { - write(matrices, output) + write(matrices, output, args) } } } diff --git a/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt b/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt index f992bb3690..bd6c507347 100644 --- a/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt +++ b/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt @@ -1,5 +1,6 @@ package ftl.reports +import ftl.args.IArgs import ftl.config.FtlConstants.indent import ftl.json.MatrixMap import ftl.json.SavedMatrix @@ -62,14 +63,14 @@ object MatrixResultsReport : IReport { } } - private fun write(matrices: MatrixMap, output: String) { - val reportPath = reportPath(matrices) + private fun write(matrices: MatrixMap, output: String, args: IArgs) { + val reportPath = reportPath(matrices, args) reportPath.write(output) } - override fun run(matrices: MatrixMap, testSuite: JUnitTestResult?, printToStdout: Boolean) { + override fun run(matrices: MatrixMap, testSuite: JUnitTestResult?, printToStdout: Boolean, args: IArgs) { val output = generate(matrices) if (printToStdout) print(output) - write(matrices, output) + write(matrices, output, args) } } diff --git a/test_runner/src/main/kotlin/ftl/reports/util/IReport.kt b/test_runner/src/main/kotlin/ftl/reports/util/IReport.kt index 6a1c41995d..8f358b0fd5 100644 --- a/test_runner/src/main/kotlin/ftl/reports/util/IReport.kt +++ b/test_runner/src/main/kotlin/ftl/reports/util/IReport.kt @@ -1,12 +1,13 @@ package ftl.reports.util +import ftl.args.IArgs import ftl.json.MatrixMap import ftl.reports.xml.model.JUnitTestResult import ftl.util.resolveLocalRunPath import java.nio.file.Paths interface IReport { - fun run(matrices: MatrixMap, testSuite: JUnitTestResult?, printToStdout: Boolean = false) + fun run(matrices: MatrixMap, testSuite: JUnitTestResult?, printToStdout: Boolean = false, args: IArgs) fun reportName(): String { return this::class.java.simpleName @@ -14,8 +15,8 @@ interface IReport { val extension: String - fun reportPath(matrices: MatrixMap): String { - val path = resolveLocalRunPath(matrices) + fun reportPath(matrices: MatrixMap, args: IArgs): String { + val path = resolveLocalRunPath(matrices, args) return Paths.get(path, reportName() + extension).toString() } } diff --git a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt index b84f1b0f2f..b662e06897 100644 --- a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt +++ b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt @@ -20,9 +20,9 @@ import kotlin.math.roundToInt object ReportManager { - private fun findXmlFiles(matrices: MatrixMap): List { + private fun findXmlFiles(matrices: MatrixMap, args: IArgs): List { val xmlFiles = mutableListOf() - val rootFolder = File(resolveLocalRunPath(matrices)) + val rootFolder = File(resolveLocalRunPath(matrices, args)) rootFolder.walk().forEach { if (it.name.matches(Artifacts.testResultRgx)) { @@ -53,10 +53,10 @@ object ReportManager { return matchResult?.groupValues?.last().orEmpty() } - private fun processXml(matrices: MatrixMap, process: (file: File) -> JUnitTestResult): JUnitTestResult? { + private fun processXml(matrices: MatrixMap, args: IArgs, process: (file: File) -> JUnitTestResult): JUnitTestResult? { var mergedXml: JUnitTestResult? = null - findXmlFiles(matrices).forEach { xmlFile -> + findXmlFiles(matrices, args).forEach { xmlFile -> val parsedXml = process(xmlFile) val webLink = getWebLink(matrices, xmlFile) val suiteName = getDeviceString(xmlFile.parentFile.name) @@ -77,9 +77,9 @@ object ReportManager { private fun parseTestSuite(matrices: MatrixMap, args: IArgs): JUnitTestResult? { val iosXml = args is IosArgs return if (iosXml) { - processXml(matrices, ::parseAllSuitesXml) + processXml(matrices, args, ::parseAllSuitesXml) } else { - processXml(matrices, ::parseOneSuiteXml) + processXml(matrices, args, ::parseOneSuiteXml) } } @@ -96,16 +96,16 @@ object ReportManager { CostReport, MatrixResultsReport ).map { - it.run(matrices, testSuite, printToStdout = true) + it.run(matrices, testSuite, printToStdout = true, args = args) } if (!testSuccessful) { listOf( HtmlErrorReport - ).map { it.run(matrices, testSuite) } + ).map { it.run(matrices, testSuite, printToStdout = false, args = args) } } - JUnitReport.run(matrices, testSuite) + JUnitReport.run(matrices, testSuite, printToStdout = false, args = args) processJunitXml(testSuite, args) // FTL has a bug with matrix roll-up when using flakyTestAttempts diff --git a/test_runner/src/main/kotlin/ftl/run/GenericTestRunner.kt b/test_runner/src/main/kotlin/ftl/run/GenericTestRunner.kt index c878bc81ef..548eaa51fd 100644 --- a/test_runner/src/main/kotlin/ftl/run/GenericTestRunner.kt +++ b/test_runner/src/main/kotlin/ftl/run/GenericTestRunner.kt @@ -2,11 +2,14 @@ package ftl.run import com.google.api.services.testing.model.TestMatrix import ftl.args.IArgs +import ftl.args.yml.FlankYmlParams import ftl.config.FtlConstants +import ftl.config.FtlConstants.useMock import ftl.json.MatrixMap import ftl.json.SavedMatrix import ftl.util.StopWatch import ftl.util.Utils +import java.io.File object GenericTestRunner { fun beforeRunTests(args: IArgs): Pair { @@ -14,7 +17,22 @@ object GenericTestRunner { val stopwatch = StopWatch().start() TestRunner.assertMockUrl() - val resultsDir = args.resultsDir ?: Utils.uniqueObjectName() + val resultsDir = if (args.localResultDir != FlankYmlParams.defaultLocalResultDir) { + // Only one result is stored when using --local-result-dir + // Delete any old results if they exist before storing new ones. + File(args.localResultDir).deleteRecursively() + "../${args.localResultDir}" + } else { + args.resultsDir ?: Utils.uniqueObjectName() + } + + // Avoid spamming the results/ dir with temporary files from running the test suite. + if (useMock) { + Runtime.getRuntime().addShutdownHook(Thread { + File(args.localResultDir, resultsDir).deleteRecursively() + }) + } + return stopwatch to resultsDir } @@ -32,13 +50,13 @@ object GenericTestRunner { } val matrixMap = MatrixMap(savedMatrices, runGcsPath) - TestRunner.updateMatrixFile(matrixMap) + TestRunner.updateMatrixFile(matrixMap, config) TestRunner.saveConfigFile(matrixMap.runPath, config) println(FtlConstants.indent + "${savedMatrices.size} matrix ids created in ${stopwatch.check()}") val gcsBucket = "https://console.developers.google.com/storage/browser/" + - config.resultsBucket + "/" + matrixMap.runPath + config.resultsBucket + "/" + matrixMap.runPath println(FtlConstants.indent + gcsBucket) println() diff --git a/test_runner/src/main/kotlin/ftl/run/TestRunner.kt b/test_runner/src/main/kotlin/ftl/run/TestRunner.kt index 7352396e68..fad9de4575 100644 --- a/test_runner/src/main/kotlin/ftl/run/TestRunner.kt +++ b/test_runner/src/main/kotlin/ftl/run/TestRunner.kt @@ -12,7 +12,6 @@ import ftl.config.FtlConstants import ftl.config.FtlConstants.defaultAndroidConfig import ftl.config.FtlConstants.defaultIosConfig import ftl.config.FtlConstants.indent -import ftl.config.FtlConstants.localResultsDir import ftl.config.FtlConstants.localhost import ftl.config.FtlConstants.matrixIdsFile import ftl.gc.GcStorage @@ -55,8 +54,8 @@ object TestRunner { } } - fun updateMatrixFile(matrixMap: MatrixMap): Path { - val matrixIdsPath = Paths.get(FtlConstants.localResultsDir, matrixMap.runPath, FtlConstants.matrixIdsFile) + fun updateMatrixFile(matrixMap: MatrixMap, args: IArgs): Path { + val matrixIdsPath = Paths.get(args.localResultDir, matrixMap.runPath, FtlConstants.matrixIdsFile) matrixIdsPath.parent.toFile().mkdirs() Files.write(matrixIdsPath, gson.toJson(matrixMap.map).toByteArray()) return matrixIdsPath @@ -64,7 +63,7 @@ object TestRunner { fun saveConfigFile(matrixRunPath: String, args: IArgs): Path? { val configFilePath = - Paths.get(FtlConstants.localResultsDir, matrixRunPath, FtlConstants.configFileName(args)) + Paths.get(args.localResultDir, matrixRunPath, FtlConstants.configFileName(args)) configFilePath.parent.toFile().mkdirs() Files.write(configFilePath, args.data.toByteArray()) return configFilePath @@ -100,7 +99,7 @@ object TestRunner { if (dirty) { println(FtlConstants.indent + "Updating matrix file") - updateMatrixFile(matrixMap) + updateMatrixFile(matrixMap, args) } println() } @@ -131,8 +130,8 @@ object TestRunner { println() } - private fun lastGcsPath(): String? { - val resultsFile = Paths.get(FtlConstants.localResultsDir).toFile() + private fun lastGcsPath(args: IArgs): String? { + val resultsFile = Paths.get(args.localResultDir).toFile() if (!resultsFile.exists()) return null val scheduledRuns = resultsFile.listFiles().filter { it.isDirectory }.sortedBy { it.lastModified() } @@ -141,16 +140,16 @@ object TestRunner { return scheduledRuns.first().name } - /** Reads in the last matrices from the localResultsDir folder **/ - private fun lastArgs(): IArgs { - val lastRun = lastGcsPath() + /** Reads in the last matrices from the localResultDir folder **/ + private fun lastArgs(args: IArgs): IArgs { + val lastRun = lastGcsPath(args) if (lastRun == null) { fatalError("no runs found in results/ folder") } - val iosConfig = Paths.get(localResultsDir, lastRun, defaultIosConfig) - val androidConfig = Paths.get(localResultsDir, lastRun, defaultAndroidConfig) + val iosConfig = Paths.get(args.localResultDir, lastRun, defaultIosConfig) + val androidConfig = Paths.get(args.localResultDir, lastRun, defaultAndroidConfig) when { iosConfig.toFile().exists() -> return IosArgs.load(iosConfig) @@ -161,9 +160,9 @@ object TestRunner { throw RuntimeException("should not happen") } - /** Reads in the last matrices from the localResultsDir folder **/ - private fun lastMatrices(): MatrixMap { - val lastRun = lastGcsPath() + /** Reads in the last matrices from the localResultDir folder **/ + private fun lastMatrices(args: IArgs): MatrixMap { + val lastRun = lastGcsPath(args) if (lastRun == null) { fatalError("no runs found in results/ folder") @@ -171,14 +170,14 @@ object TestRunner { } println("Loading run $lastRun") - return matrixPathToObj(lastRun) + return matrixPathToObj(lastRun, args) } /** Creates MatrixMap from matrix_ids.json file */ - fun matrixPathToObj(path: String): MatrixMap { + fun matrixPathToObj(path: String, args: IArgs): MatrixMap { var filePath = Paths.get(path, matrixIdsFile).toFile() if (!filePath.exists()) { - filePath = Paths.get(localResultsDir, path, matrixIdsFile).toFile() + filePath = Paths.get(args.localResultDir, path, matrixIdsFile).toFile() } val json = filePath.readText() @@ -210,7 +209,7 @@ object TestRunner { result.iterateAll().forEach { blob -> val blobPath = blob.blobId.name if (artifactsList.any { blobPath.matches(it) }) { - val downloadFile = Paths.get(FtlConstants.localResultsDir, blobPath) + val downloadFile = Paths.get(args.localResultDir, blobPath) print(".") if (!downloadFile.toFile().exists()) { downloadFile.parent.toFile().mkdirs() @@ -228,7 +227,7 @@ object TestRunner { if (dirty) { println(FtlConstants.indent + "Updating matrix file") - updateMatrixFile(matrixMap) + updateMatrixFile(matrixMap, args) println() } } @@ -250,7 +249,7 @@ object TestRunner { } println() - updateMatrixFile(matrices) + updateMatrixFile(matrices, args) } // Used for when the matrix has exactly one test. Polls for detailed progress @@ -324,25 +323,25 @@ object TestRunner { } // used to update and poll the results from an async run - suspend fun refreshLastRun() { - val matrixMap = lastMatrices() - val args = lastArgs() + suspend fun refreshLastRun(currentArgs: IArgs) { + val matrixMap = lastMatrices(currentArgs) + val lastArgs = lastArgs(currentArgs) - refreshMatrices(matrixMap, args) - pollMatrices(matrixMap, args) - fetchArtifacts(matrixMap, args) + refreshMatrices(matrixMap, lastArgs) + pollMatrices(matrixMap, lastArgs) + fetchArtifacts(matrixMap, lastArgs) // Must generate reports *after* fetching xml artifacts since reports require xml - val exitCode = ReportManager.generate(matrixMap, args) + val exitCode = ReportManager.generate(matrixMap, lastArgs) System.exit(exitCode) } // used to cancel and update results from an async run - fun cancelLastRun() { - val matrixMap = lastMatrices() - val args = lastArgs() + fun cancelLastRun(args: IArgs) { + val matrixMap = lastMatrices(args) + val lastArgs = lastArgs(args) - cancelMatrices(matrixMap, args) + cancelMatrices(matrixMap, lastArgs) } suspend fun newRun(args: IArgs) { diff --git a/test_runner/src/main/kotlin/ftl/util/MatrixUtil.kt b/test_runner/src/main/kotlin/ftl/util/MatrixUtil.kt index d041a46958..2cf5130629 100644 --- a/test_runner/src/main/kotlin/ftl/util/MatrixUtil.kt +++ b/test_runner/src/main/kotlin/ftl/util/MatrixUtil.kt @@ -1,15 +1,14 @@ package ftl.util import ftl.args.IArgs -import ftl.config.FtlConstants import ftl.json.MatrixMap import java.io.File import java.nio.file.Paths import java.util.concurrent.TimeUnit -fun resolveLocalRunPath(matrices: MatrixMap): String { +fun resolveLocalRunPath(matrices: MatrixMap, args: IArgs): String { var runPath = File(matrices.runPath) - if (!runPath.exists()) runPath = Paths.get(FtlConstants.localResultsDir, runPath.name).toFile() + if (!runPath.exists()) runPath = Paths.get(args.localResultDir, runPath.name).toFile() return runPath.toString() } diff --git a/test_runner/src/main/kotlin/ftl/util/Utils.kt b/test_runner/src/main/kotlin/ftl/util/Utils.kt index 1139280479..eddf20c556 100644 --- a/test_runner/src/main/kotlin/ftl/util/Utils.kt +++ b/test_runner/src/main/kotlin/ftl/util/Utils.kt @@ -50,7 +50,7 @@ object Utils { throw RuntimeException(e) } System.err.println(e) - exitProcess(-1) + exitProcess(3) } fun assertNotEmpty(str: String, e: String) { diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index c9bec657e9..7ed0541c24 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -226,6 +226,7 @@ AndroidArgs - class example.Test#grantPermission2 disable-sharding: true project: projectFoo + local-result-dir: results """.trimIndent() ) } @@ -788,11 +789,28 @@ AndroidArgs gcloud: app: $appApk test: $testApk - results-dir: a """ assertThat(AndroidArgs.load(yaml).flakyTestAttempts).isEqualTo(0) val androidArgs = AndroidArgs.load(yaml, cli) assertThat(androidArgs.flakyTestAttempts).isEqualTo(3) } + + @Test + fun `cli local-result-dir`() { + val cli = AndroidRunCommand() + CommandLine(cli).parse("--local-result-dir=b") + + val yaml = """ + gcloud: + app: $appApk + test: $testApk + flank: + local-result-dir: a + """ + assertThat(AndroidArgs.load(yaml).localResultDir).isEqualTo("a") + + val androidArgs = AndroidArgs.load(yaml, cli) + assertThat(androidArgs.localResultDir).isEqualTo("b") + } } diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index df5d9460fd..7453a11b6f 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -178,6 +178,7 @@ IosArgs - b/testBasicSelection2 disable-sharding: true project: projectFoo + local-result-dir: results """.trimIndent() ) } @@ -617,6 +618,22 @@ IosArgs assertThat(androidArgs.flakyTestAttempts).isEqualTo(3) } + @Test + fun `cli local-results-dir`() { + val cli = IosRunCommand() + CommandLine(cli).parse("--local-result-dir=a") + + val yaml = """ + gcloud: + test: $testPath + xctestrun-file: $testPath + """ + assertThat(IosArgs.load(yaml).localResultDir).isEqualTo("results") + + val androidArgs = IosArgs.load(yaml, cli) + assertThat(androidArgs.localResultDir).isEqualTo("a") + } + private fun getValidTestsSample() = listOf( "ClassOneTest/testOne", "ClassOneTest/testTwo", diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/CancelCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/CancelCommandTest.kt index 032837b3bf..21cd2f6a53 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/CancelCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/CancelCommandTest.kt @@ -48,7 +48,9 @@ class CancelCommandTest { @Test fun cancelCommandRuns() { exit.expectSystemExit() - AndroidRunCommand().run() + val runCmd = AndroidRunCommand() + runCmd.configPath = "./src/test/kotlin/ftl/fixtures/android.yml" + runCmd.run() CancelCommand().run() val output = systemOutRule.log Truth.assertThat(output).contains("No matrices to cancel") diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/RefreshCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/RefreshCommandTest.kt index ecaffb9f15..a83b4a5e92 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/RefreshCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/RefreshCommandTest.kt @@ -3,6 +3,7 @@ package ftl.cli.firebase import com.google.common.truth.Truth import com.google.common.truth.Truth.assertThat import ftl.test.util.FlankTestRunner +import java.io.File import java.nio.file.Files import java.nio.file.Paths import org.junit.Rule @@ -28,6 +29,10 @@ class RefreshCommandTest { val yamlCfg = Paths.get(parent, "flank.yml") matrixIds.parent.toFile().mkdirs() + Runtime.getRuntime().addShutdownHook(Thread { + File(parent).deleteRecursively() + }) + if (matrixIds.toFile().exists() && yamlCfg.toFile().exists()) return Files.write( @@ -43,14 +48,16 @@ class RefreshCommandTest { "billablePhysicalMinutes": 0, "outcome": "success" } } - """.trimIndent().toByteArray()) + """.trimIndent().toByteArray() + ) Files.write( yamlCfg, """ gcloud: app: ../test_app/apks/app-debug.apk test: ../test_app/apks/app-debug-androidTest.apk - """.trimIndent().toByteArray()) + """.trimIndent().toByteArray() + ) } @Test @@ -82,7 +89,9 @@ class RefreshCommandTest { fun refreshCommandRuns() { exit.expectSystemExit() setupResultsDir() - RefreshCommand().run() + val cmd = RefreshCommand() + cmd.usageHelpRequested + cmd.run() val output = systemOutRule.log Truth.assertThat(output).contains("1 / 1 (100.00%)") } diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt index 5b2e9bc559..bcc251aa56 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt @@ -37,7 +37,9 @@ class AndroidRunCommandTest { @Test fun androidRunCommandRuns() { exit.expectSystemExit() - AndroidRunCommand().run() + val runCmd = AndroidRunCommand() + runCmd.configPath = "./src/test/kotlin/ftl/fixtures/android.yml" + runCmd.run() val output = systemOutRule.log assertThat(output).contains("1 / 1 (100.00%)") } @@ -85,6 +87,7 @@ class AndroidRunCommandTest { assertThat(cmd.resultsDir).isNull() assertThat(cmd.flakyTestAttempts).isNull() assertThat(cmd.disableSharding).isNull() + assertThat(cmd.localResultDir).isNull() } @Test @@ -309,4 +312,12 @@ class AndroidRunCommandTest { assertThat(cmd.disableSharding).isEqualTo(true) } + + @Test + fun `localResultsDir parse`() { + val cmd = AndroidRunCommand() + CommandLine(cmd).parse("--local-result-dir=a") + + assertThat(cmd.localResultDir).isEqualTo("a") + } } diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt index 0924e8cbeb..36672621ae 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt @@ -37,7 +37,10 @@ class IosRunCommandTest { @Test fun iosRunCommandRuns() { exit.expectSystemExit() - IosRunCommand().run() + val runCmd = IosRunCommand() + runCmd.configPath = "./src/test/kotlin/ftl/fixtures/ios.yml" + runCmd.run() + val output = systemOutRule.log Truth.assertThat(output).contains("1 / 1 (100.00%)") } @@ -78,6 +81,7 @@ class IosRunCommandTest { assertThat(cmd.device).isNull() assertThat(cmd.resultsDir).isNull() assertThat(cmd.flakyTestAttempts).isNull() + assertThat(cmd.localResultsDir).isNull() } @Test @@ -243,4 +247,12 @@ class IosRunCommandTest { assertThat(cmd.disableSharding).isEqualTo(true) } + + @Test + fun `local-results-dir parse`() { + val cmd = IosRunCommand() + CommandLine(cmd).parse("--local-result-dir=a") + + assertThat(cmd.localResultsDir).isEqualTo("a") + } } diff --git a/test_runner/src/test/kotlin/ftl/fixtures/android.yml b/test_runner/src/test/kotlin/ftl/fixtures/android.yml new file mode 100644 index 0000000000..1a1bcf50e1 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/android.yml @@ -0,0 +1,3 @@ +gcloud: + app: "../test_app/apks/app-debug.apk" + test: "../test_app/apks/error-androidTest.apk" diff --git a/test_runner/src/test/kotlin/ftl/fixtures/ios.yml b/test_runner/src/test/kotlin/ftl/fixtures/ios.yml new file mode 100644 index 0000000000..f28de4a4ad --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/ios.yml @@ -0,0 +1,3 @@ +gcloud: + test: "./src/test/kotlin/ftl/fixtures/tmp/EarlGreyExample.zip" + xctestrun-file: "./src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos12.1-arm64e.xctestrun" diff --git a/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt b/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt index 03434ac04f..bb2a9d8709 100644 --- a/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt @@ -29,7 +29,7 @@ class ReportManagerTest { @Test fun generate_fromErrorResult() { - val matrix = TestRunner.matrixPathToObj("./src/test/kotlin/ftl/fixtures/error_result") + val matrix = TestRunner.matrixPathToObj("./src/test/kotlin/ftl/fixtures/error_result", AndroidArgs.default()) val mockArgs = mock(AndroidArgs::class.java) `when`(mockArgs.smartFlankGcsPath).thenReturn("") ReportManager.generate(matrix, mockArgs) @@ -37,7 +37,7 @@ class ReportManagerTest { @Test fun generate_fromSuccessResult() { - val matrix = TestRunner.matrixPathToObj("./src/test/kotlin/ftl/fixtures/success_result") + val matrix = TestRunner.matrixPathToObj("./src/test/kotlin/ftl/fixtures/success_result", AndroidArgs.default()) val mockArgs = mock(AndroidArgs::class.java) `when`(mockArgs.smartFlankGcsPath).thenReturn("") ReportManager.generate(matrix, mockArgs) diff --git a/test_runner/src/test/kotlin/ftl/util/MatrixUtilTest.kt b/test_runner/src/test/kotlin/ftl/util/MatrixUtilTest.kt index 9109050ecc..d918d3dafc 100644 --- a/test_runner/src/test/kotlin/ftl/util/MatrixUtilTest.kt +++ b/test_runner/src/test/kotlin/ftl/util/MatrixUtilTest.kt @@ -1,6 +1,7 @@ package ftl.util import com.google.common.truth.Truth.assertThat +import ftl.args.AndroidArgs import ftl.args.IArgs import ftl.json.MatrixMap import org.junit.Test @@ -30,14 +31,14 @@ class MatrixUtilTest { fun resolveLocalRunPath_validInput() { val matrixMap = mock(MatrixMap::class.java) `when`(matrixMap.runPath).thenReturn("a/b") - assertThat(resolveLocalRunPath(matrixMap)).isEqualTo("results/b") + assertThat(resolveLocalRunPath(matrixMap, AndroidArgs.default())).isEqualTo("results/b") } @Test fun resolveLocalRunPath_pathExists() { val matrixMap = mock(MatrixMap::class.java) `when`(matrixMap.runPath).thenReturn("/tmp") - assertThat(resolveLocalRunPath(matrixMap)).isEqualTo("/tmp") + assertThat(resolveLocalRunPath(matrixMap, AndroidArgs.default())).isEqualTo("/tmp") } @Test