From 6f1819db20aaa99cc0e3b509d9fdfbd2324b68c1 Mon Sep 17 00:00:00 2001 From: Eli Hart Date: Thu, 4 Apr 2019 15:25:51 -0500 Subject: [PATCH 1/5] Add option to prevent uploading results for Smart Flank --- README.md | 10 ++++++++++ docs/smart_flank.md | 1 - release_notes.md | 2 +- test_runner/flank.ios.yml | 5 +++++ test_runner/flank.yml | 5 +++++ .../src/main/kotlin/ftl/args/AndroidArgs.kt | 2 ++ test_runner/src/main/kotlin/ftl/args/IArgs.kt | 1 + .../src/main/kotlin/ftl/args/IosArgs.kt | 2 ++ .../src/main/kotlin/ftl/args/yml/FlankYml.kt | 7 +++++-- .../test/android/AndroidRunCommand.kt | 6 ++++++ .../cli/firebase/test/ios/IosRunCommand.kt | 6 ++++++ .../src/main/kotlin/ftl/gc/GcStorage.kt | 2 +- .../test/kotlin/ftl/args/AndroidArgsTest.kt | 19 +++++++++++++++++++ .../src/test/kotlin/ftl/args/IosArgsTest.kt | 18 ++++++++++++++++++ .../test/android/AndroidRunCommandTest.kt | 9 +++++++++ .../firebase/test/ios/IosRunCommandTest.kt | 9 +++++++++ 16 files changed, 99 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 572c887ec9..b50c4f4986 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,11 @@ flank: ## Google cloud storage path to store the JUnit XML results from the last run. # smart-flank-gcs-path: gs://tmp_flank/flank/test_app_ios.xml + ## Whether the JUnit XML results should be uploaded for Smart Flank. + ## This can be disabled to prevent new results from overriding previous results. + ## Default: true + # smart-flank-upload-enabled: true + ## Disables sharding. Useful for parameterized tests. # disable-sharding: false @@ -274,6 +279,11 @@ flank: ## Google cloud storage path to store the JUnit XML results from the last run. # smart-flank-gcs-path: gs://tmp_flank/flank/test_app_android.xml + ## Whether the JUnit XML results should be uploaded for Smart Flank. + ## This can be disabled to prevent new results from overriding previous results. + ## Default: true + # smart-flank-upload-enabled: true + ## Disables sharding. Useful for parameterized tests. # disable-sharding: false diff --git a/docs/smart_flank.md b/docs/smart_flank.md index a3cc867a16..98428aabb0 100644 --- a/docs/smart_flank.md +++ b/docs/smart_flank.md @@ -65,4 +65,3 @@ Example: * Keep a user configurable rolling number of aggregated xmls (1.xml, 2.xml, 3.xml) and shard based on the average time. Average time is expected to be more reliable than always using the last time in isolation. * Identify a way of translating app binary to a default xml name (bundle id/package name) so that smart flank works out of the box for users. Talk with Firebase on how to do this locally and/or expose an API. -* Provide an option to disable uploading so that local runs don't update time information for CI/pull request runs. diff --git a/release_notes.md b/release_notes.md index 19dc5b386c..1fa2769154 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,6 +1,6 @@ ## next (unreleased) -- +- [#537](https://github.com/TestArmada/flank/pull/537) Add `smart-flank-upload-enabled` yml option to prevent new results from overriding previous results. ([elihart](https://github.com/elihart)) ## v5.0.1 diff --git a/test_runner/flank.ios.yml b/test_runner/flank.ios.yml index 879f5011ad..0d9753679a 100644 --- a/test_runner/flank.ios.yml +++ b/test_runner/flank.ios.yml @@ -82,6 +82,11 @@ flank: ## Google cloud storage path to store the JUnit XML results from the last run. # smart-flank-gcs-path: gs://tmp_flank/flank/test_app_ios.xml + ## Whether the JUnit XML results should be uploaded for Smart Flank. + ## This can be disabled to prevent new results from overriding previous results. + ## Default: true + # smart-flank-upload-enabled: true + ## Disables sharding. Useful for parameterized tests. # disable-sharding: false diff --git a/test_runner/flank.yml b/test_runner/flank.yml index fa88e67848..7ec33f832b 100644 --- a/test_runner/flank.yml +++ b/test_runner/flank.yml @@ -107,6 +107,11 @@ flank: ## Google cloud storage path to store the JUnit XML results from the last run. # smart-flank-gcs-path: gs://tmp_flank/flank/test_app_android.xml + ## Whether the JUnit XML results should be uploaded for Smart Flank. + ## This can be disabled to prevent new results from overriding previous results. + ## Default: true + # smart-flank-upload-enabled: true + ## Disables sharding. Useful for parameterized tests. # disable-sharding: false diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index be321bc3d7..6d2d3dfce1 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -69,6 +69,7 @@ class AndroidArgs( override val shardTime = cli?.shardTime ?: flank.shardTime override val repeatTests = cli?.repeatTests ?: flank.repeatTests override val smartFlankGcsPath = flank.smartFlankGcsPath + override val smartFlankUploadEnabled = cli?.smartFlankUploadEnabled ?: flank.smartFlankUploadEnabled override val testTargetsAlwaysRun = cli?.testTargetsAlwaysRun ?: flank.testTargetsAlwaysRun override val filesToDownload = cli?.filesToDownload ?: flank.filesToDownload override val disableSharding = cli?.disableSharding ?: flank.disableSharding @@ -169,6 +170,7 @@ ${devicesToString(devices)} shard-time: $shardTime repeat-tests: $repeatTests smart-flank-gcs-path: $smartFlankGcsPath + smart-flank-upload-enabled: $smartFlankUploadEnabled files-to-download: ${listToString(filesToDownload)} test-targets-always-run: diff --git a/test_runner/src/main/kotlin/ftl/args/IArgs.kt b/test_runner/src/main/kotlin/ftl/args/IArgs.kt index 4de174e8c4..b2d0f89f61 100644 --- a/test_runner/src/main/kotlin/ftl/args/IArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IArgs.kt @@ -21,6 +21,7 @@ interface IArgs { val shardTime: Int val repeatTests: Int val smartFlankGcsPath: String + val smartFlankUploadEnabled: Boolean val testTargetsAlwaysRun: List val filesToDownload: List val disableSharding: Boolean diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index 3dc74891e5..0a48ee9d64 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -52,6 +52,7 @@ class IosArgs( override val shardTime = cli?.shardTime ?: flank.shardTime override val repeatTests = cli?.repeatTests ?: flank.repeatTests override val smartFlankGcsPath = flank.smartFlankGcsPath + override val smartFlankUploadEnabled = cli?.smartFlankUploadEnabled ?: flank.smartFlankUploadEnabled override val testTargetsAlwaysRun = cli?.testTargetsAlwaysRun ?: flank.testTargetsAlwaysRun override val filesToDownload = cli?.filesToDownload ?: flank.filesToDownload override val disableSharding = cli?.disableSharding ?: flank.disableSharding @@ -124,6 +125,7 @@ ${devicesToString(devices)} shard-time: $shardTime repeat-tests: $repeatTests smart-flank-gcs-path: $smartFlankGcsPath + smart-flank-upload-enabled: $smartFlankUploadEnabled test-targets-always-run: ${listToString(testTargetsAlwaysRun)} files-to-download: diff --git a/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt index edfe4be219..c3a9d4c963 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt @@ -23,6 +23,9 @@ class FlankYmlParams( @field:JsonProperty("smart-flank-gcs-path") val smartFlankGcsPath: String = "", + @field:JsonProperty("smart-flank-upload-enabled") + val smartFlankUploadEnabled: Boolean = true, + @field:JsonProperty("disable-sharding") val disableSharding: Boolean = false, @@ -39,8 +42,8 @@ class FlankYmlParams( ) { companion object : IYmlKeys { override val keys = listOf( - "max-test-shards", "shard-time", "repeat-tests", "smart-flank-gcs-path", "disable-sharding", - "test-targets-always-run", "files-to-download", "project", "local-result-dir" + "max-test-shards", "shard-time", "repeat-tests", "smart-flank-gcs-path", "smart-flank-upload-enabled", + "disable-sharding", "test-targets-always-run", "files-to-download", "project", "local-result-dir" ) const val defaultLocalResultDir = "results" diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt index 7ad768a039..1334ae0a72 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt @@ -279,4 +279,10 @@ class AndroidRunCommand : Runnable { description = ["Saves test result to this local folder. Deleted before each run."] ) var localResultDir: String? = null + + @Option( + names = ["--smart-flank-upload-enabled"], + description = ["Whether the JUnit XML results should be uploaded to the Smart Flank cloud storage."] + ) + var smartFlankUploadEnabled: Boolean? = null } diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt index a9966a4aa9..1ddc565c2c 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt @@ -220,4 +220,10 @@ class IosRunCommand : Runnable { description = ["Saves test result to this local folder. Deleted before each run."] ) var localResultsDir: String? = null + + @Option( + names = ["--smart-flank-upload-enabled"], + description = ["Whether the JUnit XML results should be uploaded to the Smart Flank cloud storage."] + ) + var smartFlankUploadEnabled: Boolean? = null } diff --git a/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt b/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt index 697543a287..b16ada204d 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt @@ -54,7 +54,7 @@ object GcStorage { ) fun uploadJunitXml(testResult: JUnitTestResult, args: IArgs) { - if (args.smartFlankGcsPath.isEmpty()) return + if (args.smartFlankGcsPath.isEmpty() || !args.smartFlankUploadEnabled) return // bucket/path/to/object val rawPath = args.smartFlankGcsPath.drop(GCS_PREFIX.length) diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 7ed0541c24..3cbc979b40 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -227,6 +227,7 @@ AndroidArgs disable-sharding: true project: projectFoo local-result-dir: results + smart-flank-upload-enabled: true """.trimIndent() ) } @@ -813,4 +814,22 @@ AndroidArgs val androidArgs = AndroidArgs.load(yaml, cli) assertThat(androidArgs.localResultDir).isEqualTo("b") } + + @Test + fun `cli smart-flank-upload-enabled`() { + val cli = AndroidRunCommand() + CommandLine(cli).parse("--smart-flank-upload-enabled=false") + + val yaml = """ + gcloud: + app: $appApk + test: $testApk + flank: + smart-flank-upload-enabled: true + """ + assertThat(AndroidArgs.load(yaml).smartFlankUploadEnabled).isEqualTo(true) + + val androidArgs = AndroidArgs.load(yaml, cli) + assertThat(androidArgs.smartFlankUploadEnabled).isEqualTo(false) + } } diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index a82c74d539..bc77f92b6d 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -6,6 +6,7 @@ import ftl.args.yml.GcloudYml import ftl.args.yml.IosFlankYml import ftl.args.yml.IosGcloudYml import ftl.args.yml.IosGcloudYmlParams +import ftl.cli.firebase.test.android.AndroidRunCommand import ftl.cli.firebase.test.ios.IosRunCommand import ftl.config.Device import ftl.config.FtlConstants.defaultIosModel @@ -179,6 +180,7 @@ IosArgs disable-sharding: true project: projectFoo local-result-dir: results + smart-flank-upload-enabled: true """.trimIndent() ) } @@ -634,6 +636,22 @@ IosArgs assertThat(androidArgs.localResultDir).isEqualTo("a") } + @Test + fun `cli smart-flank-upload-enabled`() { + val cli = IosRunCommand() + CommandLine(cli).parse("--smart-flank-upload-enabled=false") + + val yaml = """ + gcloud: + test: $testPath + xctestrun-file: $testPath + """ + assertThat(IosArgs.load(yaml).smartFlankUploadEnabled).isEqualTo(true) + + val androidArgs = IosArgs.load(yaml, cli) + assertThat(androidArgs.smartFlankUploadEnabled).isEqualTo(false) + } + private fun getValidTestsSample() = listOf( "ClassOneTest/testOne", "ClassOneTest/testTwo", diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt index bcc251aa56..5f69a7716e 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt @@ -88,6 +88,7 @@ class AndroidRunCommandTest { assertThat(cmd.flakyTestAttempts).isNull() assertThat(cmd.disableSharding).isNull() assertThat(cmd.localResultDir).isNull() + assertThat(cmd.smartFlankUploadEnabled).isNull() } @Test @@ -320,4 +321,12 @@ class AndroidRunCommandTest { assertThat(cmd.localResultDir).isEqualTo("a") } + + @Test + fun `smartFlankUploadEnabled parse`() { + val cmd = AndroidRunCommand() + CommandLine(cmd).parse("--smart-flank-upload-enabled=false") + + assertThat(cmd.smartFlankUploadEnabled).isEqualTo(false) + } } diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt index 36672621ae..5e79c2125b 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt @@ -82,6 +82,7 @@ class IosRunCommandTest { assertThat(cmd.resultsDir).isNull() assertThat(cmd.flakyTestAttempts).isNull() assertThat(cmd.localResultsDir).isNull() + assertThat(cmd.smartFlankUploadEnabled).isNull() } @Test @@ -255,4 +256,12 @@ class IosRunCommandTest { assertThat(cmd.localResultsDir).isEqualTo("a") } + + @Test + fun `smart-flank-upload-enabled parse`() { + val cmd = IosRunCommand() + CommandLine(cmd).parse("--smart-flank-upload-enabled=false") + + assertThat(cmd.smartFlankUploadEnabled).isEqualTo(false) + } } From b5ee5972ae5e9cf01d6e93e3c95197364da16a8f Mon Sep 17 00:00:00 2001 From: Eli Hart Date: Thu, 4 Apr 2019 17:41:54 -0500 Subject: [PATCH 2/5] Fix unused import --- test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index bc77f92b6d..3c2b07ea6b 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -6,7 +6,6 @@ import ftl.args.yml.GcloudYml import ftl.args.yml.IosFlankYml import ftl.args.yml.IosGcloudYml import ftl.args.yml.IosGcloudYmlParams -import ftl.cli.firebase.test.android.AndroidRunCommand import ftl.cli.firebase.test.ios.IosRunCommand import ftl.config.Device import ftl.config.FtlConstants.defaultIosModel From b5419b64eb022956e6c0770b3b7d4d033e71ae37 Mon Sep 17 00:00:00 2001 From: Eli Hart Date: Thu, 4 Apr 2019 18:11:48 -0500 Subject: [PATCH 3/5] fix args tests --- test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt | 2 +- test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 3cbc979b40..6196927eeb 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -218,6 +218,7 @@ AndroidArgs shard-time: 60 repeat-tests: 8 smart-flank-gcs-path:${' '} + smart-flank-upload-enabled: true files-to-download: - /sdcard/screenshots - /sdcard/screenshots2 @@ -227,7 +228,6 @@ AndroidArgs disable-sharding: true project: projectFoo local-result-dir: results - smart-flank-upload-enabled: true """.trimIndent() ) } diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index 3c2b07ea6b..a8c64b5686 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -167,6 +167,7 @@ IosArgs shard-time: 60 repeat-tests: 8 smart-flank-gcs-path:${' '} + smart-flank-upload-enabled: true test-targets-always-run: - a/testGrantPermissions - a/testGrantPermissions2 @@ -179,7 +180,6 @@ IosArgs disable-sharding: true project: projectFoo local-result-dir: results - smart-flank-upload-enabled: true """.trimIndent() ) } From aa5b3682daaf3cb0f9c1c3ac4b6da6cb782ed35d Mon Sep 17 00:00:00 2001 From: Eli Hart Date: Fri, 5 Apr 2019 09:05:01 -0500 Subject: [PATCH 4/5] Change name and description of flag --- README.md | 14 ++++++-------- release_notes.md | 2 +- test_runner/flank.ios.yml | 7 +++---- test_runner/flank.yml | 7 +++---- .../src/main/kotlin/ftl/args/AndroidArgs.kt | 4 ++-- test_runner/src/main/kotlin/ftl/args/IArgs.kt | 2 +- test_runner/src/main/kotlin/ftl/args/IosArgs.kt | 4 ++-- .../src/main/kotlin/ftl/args/yml/FlankYml.kt | 6 +++--- .../cli/firebase/test/android/AndroidRunCommand.kt | 6 +++--- .../ftl/cli/firebase/test/ios/IosRunCommand.kt | 6 +++--- test_runner/src/main/kotlin/ftl/gc/GcStorage.kt | 2 +- .../src/test/kotlin/ftl/args/AndroidArgsTest.kt | 12 ++++++------ .../src/test/kotlin/ftl/args/IosArgsTest.kt | 10 +++++----- .../firebase/test/android/AndroidRunCommandTest.kt | 8 ++++---- .../ftl/cli/firebase/test/ios/IosRunCommandTest.kt | 8 ++++---- 15 files changed, 47 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index b50c4f4986..bfe8cd12be 100644 --- a/README.md +++ b/README.md @@ -131,10 +131,9 @@ flank: ## Google cloud storage path to store the JUnit XML results from the last run. # smart-flank-gcs-path: gs://tmp_flank/flank/test_app_ios.xml - ## Whether the JUnit XML results should be uploaded for Smart Flank. - ## This can be disabled to prevent new results from overriding previous results. - ## Default: true - # smart-flank-upload-enabled: true + ## Disables smart flank JUnit XML uploading. Useful for preventing timing data from being updated. + ## Default: false + # smart-flank-disable-upload: false ## Disables sharding. Useful for parameterized tests. # disable-sharding: false @@ -279,10 +278,9 @@ flank: ## Google cloud storage path to store the JUnit XML results from the last run. # smart-flank-gcs-path: gs://tmp_flank/flank/test_app_android.xml - ## Whether the JUnit XML results should be uploaded for Smart Flank. - ## This can be disabled to prevent new results from overriding previous results. - ## Default: true - # smart-flank-upload-enabled: true + ## Disables smart flank JUnit XML uploading. Useful for preventing timing data from being updated. + ## Default: false + # smart-flank-disable-upload: false ## Disables sharding. Useful for parameterized tests. # disable-sharding: false diff --git a/release_notes.md b/release_notes.md index 1fa2769154..a02b5a3ad7 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,6 +1,6 @@ ## next (unreleased) -- [#537](https://github.com/TestArmada/flank/pull/537) Add `smart-flank-upload-enabled` yml option to prevent new results from overriding previous results. ([elihart](https://github.com/elihart)) +- [#537](https://github.com/TestArmada/flank/pull/537) Add `smart-flank-disable-upload` yml option to prevent new results from overriding previous results. ([elihart](https://github.com/elihart)) ## v5.0.1 diff --git a/test_runner/flank.ios.yml b/test_runner/flank.ios.yml index 0d9753679a..13da74843b 100644 --- a/test_runner/flank.ios.yml +++ b/test_runner/flank.ios.yml @@ -82,10 +82,9 @@ flank: ## Google cloud storage path to store the JUnit XML results from the last run. # smart-flank-gcs-path: gs://tmp_flank/flank/test_app_ios.xml - ## Whether the JUnit XML results should be uploaded for Smart Flank. - ## This can be disabled to prevent new results from overriding previous results. - ## Default: true - # smart-flank-upload-enabled: true + ## Disables smart flank JUnit XML uploading. Useful for preventing timing data from being updated. + ## Default: false + # smart-flank-disable-upload: false ## Disables sharding. Useful for parameterized tests. # disable-sharding: false diff --git a/test_runner/flank.yml b/test_runner/flank.yml index 7ec33f832b..69fdd4f06c 100644 --- a/test_runner/flank.yml +++ b/test_runner/flank.yml @@ -107,10 +107,9 @@ flank: ## Google cloud storage path to store the JUnit XML results from the last run. # smart-flank-gcs-path: gs://tmp_flank/flank/test_app_android.xml - ## Whether the JUnit XML results should be uploaded for Smart Flank. - ## This can be disabled to prevent new results from overriding previous results. - ## Default: true - # smart-flank-upload-enabled: true + ## Disables smart flank JUnit XML uploading. Useful for preventing timing data from being updated. + ## Default: false + # smart-flank-disable-upload: false ## Disables sharding. Useful for parameterized tests. # disable-sharding: false diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index 6d2d3dfce1..cd8b133249 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -69,7 +69,7 @@ class AndroidArgs( override val shardTime = cli?.shardTime ?: flank.shardTime override val repeatTests = cli?.repeatTests ?: flank.repeatTests override val smartFlankGcsPath = flank.smartFlankGcsPath - override val smartFlankUploadEnabled = cli?.smartFlankUploadEnabled ?: flank.smartFlankUploadEnabled + override val smartFlankDisableUpload = cli?.smartFlankDisableUpload ?: flank.smartFlankDisableUpload override val testTargetsAlwaysRun = cli?.testTargetsAlwaysRun ?: flank.testTargetsAlwaysRun override val filesToDownload = cli?.filesToDownload ?: flank.filesToDownload override val disableSharding = cli?.disableSharding ?: flank.disableSharding @@ -170,7 +170,7 @@ ${devicesToString(devices)} shard-time: $shardTime repeat-tests: $repeatTests smart-flank-gcs-path: $smartFlankGcsPath - smart-flank-upload-enabled: $smartFlankUploadEnabled + smart-flank-disable-upload: $smartFlankDisableUpload files-to-download: ${listToString(filesToDownload)} test-targets-always-run: diff --git a/test_runner/src/main/kotlin/ftl/args/IArgs.kt b/test_runner/src/main/kotlin/ftl/args/IArgs.kt index b2d0f89f61..6d83ea22b6 100644 --- a/test_runner/src/main/kotlin/ftl/args/IArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IArgs.kt @@ -21,7 +21,7 @@ interface IArgs { val shardTime: Int val repeatTests: Int val smartFlankGcsPath: String - val smartFlankUploadEnabled: Boolean + val smartFlankDisableUpload: Boolean val testTargetsAlwaysRun: List val filesToDownload: List val disableSharding: Boolean diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index 0a48ee9d64..615615bb0f 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -52,7 +52,7 @@ class IosArgs( override val shardTime = cli?.shardTime ?: flank.shardTime override val repeatTests = cli?.repeatTests ?: flank.repeatTests override val smartFlankGcsPath = flank.smartFlankGcsPath - override val smartFlankUploadEnabled = cli?.smartFlankUploadEnabled ?: flank.smartFlankUploadEnabled + override val smartFlankDisableUpload = cli?.smartFlankDisableUpload ?: flank.smartFlankDisableUpload override val testTargetsAlwaysRun = cli?.testTargetsAlwaysRun ?: flank.testTargetsAlwaysRun override val filesToDownload = cli?.filesToDownload ?: flank.filesToDownload override val disableSharding = cli?.disableSharding ?: flank.disableSharding @@ -125,7 +125,7 @@ ${devicesToString(devices)} shard-time: $shardTime repeat-tests: $repeatTests smart-flank-gcs-path: $smartFlankGcsPath - smart-flank-upload-enabled: $smartFlankUploadEnabled + smart-flank-disable-upload: $smartFlankDisableUpload test-targets-always-run: ${listToString(testTargetsAlwaysRun)} files-to-download: diff --git a/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt index c3a9d4c963..57814a4702 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt @@ -23,8 +23,8 @@ class FlankYmlParams( @field:JsonProperty("smart-flank-gcs-path") val smartFlankGcsPath: String = "", - @field:JsonProperty("smart-flank-upload-enabled") - val smartFlankUploadEnabled: Boolean = true, + @field:JsonProperty("smart-flank-disable-upload") + val smartFlankDisableUpload: Boolean = false, @field:JsonProperty("disable-sharding") val disableSharding: Boolean = false, @@ -42,7 +42,7 @@ class FlankYmlParams( ) { companion object : IYmlKeys { override val keys = listOf( - "max-test-shards", "shard-time", "repeat-tests", "smart-flank-gcs-path", "smart-flank-upload-enabled", + "max-test-shards", "shard-time", "repeat-tests", "smart-flank-gcs-path", "smart-flank-disable-upload", "disable-sharding", "test-targets-always-run", "files-to-download", "project", "local-result-dir" ) diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt index 1334ae0a72..623f5ba044 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt @@ -281,8 +281,8 @@ class AndroidRunCommand : Runnable { var localResultDir: String? = null @Option( - names = ["--smart-flank-upload-enabled"], - description = ["Whether the JUnit XML results should be uploaded to the Smart Flank cloud storage."] + names = ["--smart-flank-disable-upload"], + description = ["Disables smart flank JUnit XML uploading. Useful for preventing timing data from being updated."] ) - var smartFlankUploadEnabled: Boolean? = null + var smartFlankDisableUpload: Boolean? = null } diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt index 1ddc565c2c..1217a009f9 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt @@ -222,8 +222,8 @@ class IosRunCommand : Runnable { var localResultsDir: String? = null @Option( - names = ["--smart-flank-upload-enabled"], - description = ["Whether the JUnit XML results should be uploaded to the Smart Flank cloud storage."] + names = ["--smart-flank-disable-upload"], + description = ["Disables smart flank JUnit XML uploading. Useful for preventing timing data from being updated."] ) - var smartFlankUploadEnabled: Boolean? = null + var smartFlankDisableUpload: Boolean? = null } diff --git a/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt b/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt index b16ada204d..1c413e8029 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt @@ -54,7 +54,7 @@ object GcStorage { ) fun uploadJunitXml(testResult: JUnitTestResult, args: IArgs) { - if (args.smartFlankGcsPath.isEmpty() || !args.smartFlankUploadEnabled) return + if (args.smartFlankGcsPath.isEmpty() || !args.smartFlankDisableUpload) return // bucket/path/to/object val rawPath = args.smartFlankGcsPath.drop(GCS_PREFIX.length) diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 6196927eeb..cc49fcfc26 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -218,7 +218,7 @@ AndroidArgs shard-time: 60 repeat-tests: 8 smart-flank-gcs-path:${' '} - smart-flank-upload-enabled: true + smart-flank-disable-upload: false files-to-download: - /sdcard/screenshots - /sdcard/screenshots2 @@ -816,20 +816,20 @@ AndroidArgs } @Test - fun `cli smart-flank-upload-enabled`() { + fun `cli smart-flank-disable-upload`() { val cli = AndroidRunCommand() - CommandLine(cli).parse("--smart-flank-upload-enabled=false") + CommandLine(cli).parse("--smart-flank-disable-upload=true") val yaml = """ gcloud: app: $appApk test: $testApk flank: - smart-flank-upload-enabled: true + smart-flank-disable-upload: false """ - assertThat(AndroidArgs.load(yaml).smartFlankUploadEnabled).isEqualTo(true) + assertThat(AndroidArgs.load(yaml).smartFlankDisableUpload).isEqualTo(false) val androidArgs = AndroidArgs.load(yaml, cli) - assertThat(androidArgs.smartFlankUploadEnabled).isEqualTo(false) + assertThat(androidArgs.smartFlankDisableUpload).isEqualTo(true) } } diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index a8c64b5686..a5fd397143 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -167,7 +167,7 @@ IosArgs shard-time: 60 repeat-tests: 8 smart-flank-gcs-path:${' '} - smart-flank-upload-enabled: true + smart-flank-disable-upload: false test-targets-always-run: - a/testGrantPermissions - a/testGrantPermissions2 @@ -636,19 +636,19 @@ IosArgs } @Test - fun `cli smart-flank-upload-enabled`() { + fun `cli smart-flank-disable-upload`() { val cli = IosRunCommand() - CommandLine(cli).parse("--smart-flank-upload-enabled=false") + CommandLine(cli).parse("--smart-flank-disable-upload=true") val yaml = """ gcloud: test: $testPath xctestrun-file: $testPath """ - assertThat(IosArgs.load(yaml).smartFlankUploadEnabled).isEqualTo(true) + assertThat(IosArgs.load(yaml).smartFlankDisableUpload).isEqualTo(false) val androidArgs = IosArgs.load(yaml, cli) - assertThat(androidArgs.smartFlankUploadEnabled).isEqualTo(false) + assertThat(androidArgs.smartFlankDisableUpload).isEqualTo(true) } private fun getValidTestsSample() = listOf( diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt index 5f69a7716e..91f5849cce 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt @@ -88,7 +88,7 @@ class AndroidRunCommandTest { assertThat(cmd.flakyTestAttempts).isNull() assertThat(cmd.disableSharding).isNull() assertThat(cmd.localResultDir).isNull() - assertThat(cmd.smartFlankUploadEnabled).isNull() + assertThat(cmd.smartFlankDisableUpload).isNull() } @Test @@ -323,10 +323,10 @@ class AndroidRunCommandTest { } @Test - fun `smartFlankUploadEnabled parse`() { + fun `smartFlankDisableUpload parse`() { val cmd = AndroidRunCommand() - CommandLine(cmd).parse("--smart-flank-upload-enabled=false") + CommandLine(cmd).parse("--smart-flank-disable-upload=true") - assertThat(cmd.smartFlankUploadEnabled).isEqualTo(false) + assertThat(cmd.smartFlankDisableUpload).isEqualTo(true) } } diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt index 5e79c2125b..10ccb05f51 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt @@ -82,7 +82,7 @@ class IosRunCommandTest { assertThat(cmd.resultsDir).isNull() assertThat(cmd.flakyTestAttempts).isNull() assertThat(cmd.localResultsDir).isNull() - assertThat(cmd.smartFlankUploadEnabled).isNull() + assertThat(cmd.smartFlankDisableUpload).isNull() } @Test @@ -258,10 +258,10 @@ class IosRunCommandTest { } @Test - fun `smart-flank-upload-enabled parse`() { + fun `smart-flank-disable-upload parse`() { val cmd = IosRunCommand() - CommandLine(cmd).parse("--smart-flank-upload-enabled=false") + CommandLine(cmd).parse("--smart-flank-disable-upload=true") - assertThat(cmd.smartFlankUploadEnabled).isEqualTo(false) + assertThat(cmd.smartFlankDisableUpload).isEqualTo(true) } } From 3aa66491abcf969f3346cf9631f8fdc3f398fbc8 Mon Sep 17 00:00:00 2001 From: Eli Hart Date: Fri, 5 Apr 2019 09:22:47 -0500 Subject: [PATCH 5/5] Fix inverted boolean check --- test_runner/src/main/kotlin/ftl/gc/GcStorage.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt b/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt index 1c413e8029..89ededc6c1 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt @@ -54,7 +54,7 @@ object GcStorage { ) fun uploadJunitXml(testResult: JUnitTestResult, args: IArgs) { - if (args.smartFlankGcsPath.isEmpty() || !args.smartFlankDisableUpload) return + if (args.smartFlankGcsPath.isEmpty() || args.smartFlankDisableUpload) return // bucket/path/to/object val rawPath = args.smartFlankGcsPath.drop(GCS_PREFIX.length)