From ad4389578beafd38c48776037643569c56301f12 Mon Sep 17 00:00:00 2001 From: Michael Wright Date: Fri, 18 Jun 2021 13:41:18 +0200 Subject: [PATCH] feat: Added new Parameterized Test Option (#2035) --- build.gradle.kts | 3 +- docs/index.md | 9 ++++ test_runner/flank.yml | 9 ++++ .../src/main/kotlin/ftl/args/AndroidArgs.kt | 4 ++ .../main/kotlin/ftl/args/CreateAndroidArgs.kt | 3 +- .../kotlin/ftl/args/ValidateAndroidArgs.kt | 16 +++++- .../main/kotlin/ftl/config/FlankDefaults.kt | 1 + .../ftl/config/android/AndroidGcloudConfig.kt | 8 +++ .../test/kotlin/ftl/args/AndroidArgsTest.kt | 53 +++++++++++++++++++ 9 files changed, 103 insertions(+), 3 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 04dc218cda..d3aa216f98 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,6 +1,7 @@ + import com.github.benmanes.gradle.versions.updates.DependencyUpdatesTask -import org.jmailen.gradle.kotlinter.tasks.LintTask import org.gradle.nativeplatform.platform.internal.DefaultNativePlatform +import org.jmailen.gradle.kotlinter.tasks.LintTask import java.nio.file.Paths // Fix Exception in thread "main" java.lang.NoSuchMethodError: com.google.common.hash.Hashing.crc32c()Lcom/google/common/hash/HashFunction; diff --git a/docs/index.md b/docs/index.md index fc9a31462f..6a045e8244 100644 --- a/docs/index.md +++ b/docs/index.md @@ -587,6 +587,15 @@ gcloud: # - package com.package1.for.shard1 # - class com.package2.for.shard2.Class + ### parameterized-tests + ## Specifies how to handle tests which contain Parameterization. + ## 3 options are available + ## default: treat Parameterized tests as normal and shard accordingly + ## ignore-all: Parameterized tests are ignored and not sharded + ## collect-all-single: Parameterized tests are collected and put into a single shard + ## Note: if left blank default is used. + # parameterized-tests: default + flank: # -- FlankYml -- diff --git a/test_runner/flank.yml b/test_runner/flank.yml index 80c3899b83..cb02b7b7e9 100644 --- a/test_runner/flank.yml +++ b/test_runner/flank.yml @@ -219,6 +219,15 @@ gcloud: # - package com.package1.for.shard1 # - package com.package1.for.shard2 + ### parameterized-tests + ## Specifies how to handle tests which contain the parameterization annotation. + ## 3 options are available + ## default: treat Parameterized tests as normal and shard accordingly + ## ignore-all: Parameterized tests are ignored and not sharded + ## collect-all-single: Parameterized tests are collected and put into a single shard + ## Note: if left blank default is used. + # parameterized-tests: default + flank: # -- FlankYml -- diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index d9738d84f6..a1b4274a33 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -59,6 +59,9 @@ data class AndroidArgs( @property:AnonymizeInStatistics val testTargetsForShard: ShardChunks, + @property:IgnoreInStatistics + val parameterizedTests: String?, + @property:IgnoreInStatistics val customSharding: Map ) : IArgs by commonArgs { @@ -100,6 +103,7 @@ AndroidArgs num-flaky-test-attempts: $flakyTestAttempts test-targets-for-shard:${ArgsToString.listOfListToString(testTargetsForShard)} fail-fast: $failFast + parameterized-tests: $parameterizedTests flank: max-test-shards: $maxTestShards diff --git a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt index 7ce43699b4..1d4a135872 100644 --- a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt @@ -44,7 +44,8 @@ fun createAndroidArgs( obbNames = gcloud::obbnames.require(), grantPermissions = gcloud.grantPermissions, testTargetsForShard = gcloud.testTargetsForShard?.normalizeToTestTargets().orEmpty(), - customSharding = createCustomShards(commonArgs.customShardingJson) + customSharding = createCustomShards(commonArgs.customShardingJson), + parameterizedTests = gcloud.parameterizedTests ) private fun createCustomShards(shardingJsonPath: String) = diff --git a/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt index bd3ddc6f33..bef6030601 100644 --- a/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt @@ -36,6 +36,7 @@ fun AndroidArgs.validate() = if (shouldValidateConfig) apply { checkEnvironmentVariables() checkFilesToDownload() checkNumUniformShards() + assertParameterizedTests() } else this private fun AndroidArgs.assertTestTargetForShards() { @@ -147,7 +148,11 @@ private fun AndroidArgs.assertTestTypes() { private fun AndroidArgs.assertDirectoriesToPull() { val correctNameRegex = "(/[a-zA-Z0-9_\\-.+]+)+/?".toRegex() directoriesToPull - .filter { !it.startsWith("/sdcard") && !it.startsWith("/data/local/tmp") && !it.startsWith("/storage") || !correctNameRegex.matches(it) } + .filter { + !it.startsWith("/sdcard") && !it.startsWith("/data/local/tmp") && !it.startsWith("/storage") || !correctNameRegex.matches( + it + ) + } .takeIf { it.isNotEmpty() } ?.also { throw FlankConfigurationError( @@ -265,3 +270,12 @@ private fun AndroidArgs.checkNumUniformShards() { if ((numUniformShards ?: 0) > 0 && disableSharding) logLn("WARNING: disable-sharding is enabled with num-uniform-shards = $numUniformShards, Flank will ignore num-uniform-shards and disable sharding.") } + +private fun AndroidArgs.assertParameterizedTests() { + if (parameterizedTests.isNullOrEmpty() || parameterizedTests !in listOf( + "ignore-all", + "default", + "shard-into-single" + ) + ) throw FlankConfigurationError("Parameterized test flag must be one of the following: `default`, `ignore-all`, `shard-into-single`, leaving it blank will result in `default` sharding.") +} diff --git a/test_runner/src/main/kotlin/ftl/config/FlankDefaults.kt b/test_runner/src/main/kotlin/ftl/config/FlankDefaults.kt index 840d6eb326..7ff5a0e92b 100644 --- a/test_runner/src/main/kotlin/ftl/config/FlankDefaults.kt +++ b/test_runner/src/main/kotlin/ftl/config/FlankDefaults.kt @@ -10,4 +10,5 @@ object FlankDefaults { const val DISABLE_AUTO_LOGIN = false const val DISABLE_PERFORMANCE_METRICS = false const val GRANT_PERMISSIONS_ALL = "all" + const val DEFAULT_PARAMETERIZED_TESTS = "default" } diff --git a/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt b/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt index 0af3d99e70..7870785ae9 100644 --- a/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt +++ b/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt @@ -248,6 +248,13 @@ data class AndroidGcloudConfig @JsonIgnore constructor( @set:JsonProperty("test-targets-for-shard") var testTargetsForShard: List? by data + @set:CommandLine.Option( + names = ["--parameterized-tests"], + description = ["Specifies how to handle tests which contain the parameterization annotation."] + ) + @set:JsonProperty("parameterized-tests") + var parameterizedTests: String? by data + constructor() : this(mutableMapOf().withDefault { null }) companion object : IYmlKeys { @@ -276,6 +283,7 @@ data class AndroidGcloudConfig @JsonIgnore constructor( roboDirectives = emptyMap() roboScript = null testTargetsForShard = emptyList() + parameterizedTests = FlankDefaults.DEFAULT_PARAMETERIZED_TESTS } } } diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index e487827a2b..86901a7ab5 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -356,6 +356,7 @@ AndroidArgs num-flaky-test-attempts: 3 test-targets-for-shard: fail-fast: false + parameterized-tests: default flank: max-test-shards: 7 @@ -445,6 +446,7 @@ AndroidArgs num-flaky-test-attempts: 0 test-targets-for-shard: fail-fast: false + parameterized-tests: default flank: max-test-shards: 1 @@ -2658,6 +2660,57 @@ AndroidArgs """.trimIndent() assertEquals(expectedMessage, errorMessage) } + + @Test + fun `should throw exception if paramterized test is incorrect`() { + val yaml = """ + gcloud: + # Android gcloud + app: $appApk + test: $testApk + parameterized-tests: error + """.trimIndent() + val errorMessage = getThrowable { AndroidArgs.load(yaml).validate() }.message ?: "" + val expectedMessage = """ + Parameterized test flag must be one of the following: `default`, `ignore-all`, `shard-into-single`, leaving it blank will result in `default` sharding. + """.trimIndent() + assertEquals(expectedMessage, errorMessage) + } + + @Test + fun `should throw NOT throw an exception if parameterized test is left to be blank`() { + val yaml = """ + gcloud: + # Android gcloud + app: $appApk + test: $testApk + """.trimIndent() + AndroidArgs.load(yaml).validate() + } + + @Test + fun `should throw NOT throw an exception if paramterized test is ignore-all`() { + val yaml = """ + gcloud: + # Android gcloud + app: $appApk + test: $testApk + parameterized-tests: ignore-all + """.trimIndent() + AndroidArgs.load(yaml).validate() + } + + @Test + fun `should throw NOT throw an exception if paramterized test is shard-into-single`() { + val yaml = """ + gcloud: + # Android gcloud + app: $appApk + test: $testApk + parameterized-tests: shard-into-single + """.trimIndent() + AndroidArgs.load(yaml).validate() + } } private fun AndroidArgs.Companion.load(