feat: Added new Parameterized Test Option (#2035)

build.gradle.kts

@@ -1,6 +1,7 @@
+
 import com.github.benmanes.gradle.versions.updates.DependencyUpdatesTask
-import org.jmailen.gradle.kotlinter.tasks.LintTask
 import org.gradle.nativeplatform.platform.internal.DefaultNativePlatform
+import org.jmailen.gradle.kotlinter.tasks.LintTask
 import java.nio.file.Paths
 
 // Fix Exception in thread "main" java.lang.NoSuchMethodError: com.google.common.hash.Hashing.crc32c()Lcom/google/common/hash/HashFunction;

docs/index.md

@@ -587,6 +587,15 @@ gcloud:
   # - package com.package1.for.shard1
   # - class com.package2.for.shard2.Class
 
+  ### parameterized-tests
+  ## Specifies how to handle tests which contain Parameterization.
+  ## 3 options are available
+  ## default: treat Parameterized tests as normal and shard accordingly
+  ## ignore-all: Parameterized tests are ignored and not sharded
+  ## collect-all-single: Parameterized tests are collected and put into a single shard
+  ## Note: if left blank default is used.
+  # parameterized-tests: default
+
 flank:
   # -- FlankYml --
 

test_runner/flank.yml

@@ -219,6 +219,15 @@ gcloud:
   # - package com.package1.for.shard1
   # - package com.package1.for.shard2
 
+  ### parameterized-tests
+  ## Specifies how to handle tests which contain the parameterization annotation.
+  ## 3 options are available
+  ## default: treat Parameterized tests as normal and shard accordingly
+  ## ignore-all: Parameterized tests are ignored and not sharded
+  ## collect-all-single: Parameterized tests are collected and put into a single shard
+  ## Note: if left blank default is used.
+  # parameterized-tests: default
+
 flank:
   # -- FlankYml --
 

test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt

@@ -59,6 +59,9 @@ data class AndroidArgs(
     @property:AnonymizeInStatistics
     val testTargetsForShard: ShardChunks,
 
+    @property:IgnoreInStatistics
+    val parameterizedTests: String?,
+
     @property:IgnoreInStatistics
     val customSharding: Map<String, AndroidTestShards>
 ) : IArgs by commonArgs {
@@ -100,6 +103,7 @@ AndroidArgs
       num-flaky-test-attempts: $flakyTestAttempts
       test-targets-for-shard:${ArgsToString.listOfListToString(testTargetsForShard)}
       fail-fast: $failFast
+      parameterized-tests: $parameterizedTests
 
     flank:
       max-test-shards: $maxTestShards

test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt

@@ -44,7 +44,8 @@ fun createAndroidArgs(
     obbNames = gcloud::obbnames.require(),
     grantPermissions = gcloud.grantPermissions,
     testTargetsForShard = gcloud.testTargetsForShard?.normalizeToTestTargets().orEmpty(),
-    customSharding = createCustomShards(commonArgs.customShardingJson)
+    customSharding = createCustomShards(commonArgs.customShardingJson),
+    parameterizedTests = gcloud.parameterizedTests
 )
 
 private fun createCustomShards(shardingJsonPath: String) =

test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt

@@ -36,6 +36,7 @@ fun AndroidArgs.validate() = if (shouldValidateConfig) apply {
     checkEnvironmentVariables()
     checkFilesToDownload()
     checkNumUniformShards()
+    assertParameterizedTests()
 } else this
 
 private fun AndroidArgs.assertTestTargetForShards() {
@@ -147,7 +148,11 @@ private fun AndroidArgs.assertTestTypes() {
 private fun AndroidArgs.assertDirectoriesToPull() {
     val correctNameRegex = "(/[a-zA-Z0-9_\\-.+]+)+/?".toRegex()
     directoriesToPull
-        .filter { !it.startsWith("/sdcard") && !it.startsWith("/data/local/tmp") && !it.startsWith("/storage") || !correctNameRegex.matches(it) }
+        .filter {
+            !it.startsWith("/sdcard") && !it.startsWith("/data/local/tmp") && !it.startsWith("/storage") || !correctNameRegex.matches(
+                it
+            )
+        }
         .takeIf { it.isNotEmpty() }
         ?.also {
             throw FlankConfigurationError(
@@ -265,3 +270,12 @@ private fun AndroidArgs.checkNumUniformShards() {
     if ((numUniformShards ?: 0) > 0 && disableSharding)
         logLn("WARNING: disable-sharding is enabled with num-uniform-shards = $numUniformShards, Flank will ignore num-uniform-shards and disable sharding.")
 }
+
+private fun AndroidArgs.assertParameterizedTests() {
+    if (parameterizedTests.isNullOrEmpty() || parameterizedTests !in listOf(
+            "ignore-all",
+            "default",
+            "shard-into-single"
+        )
+    ) throw FlankConfigurationError("Parameterized test flag must be one of the following: `default`, `ignore-all`, `shard-into-single`, leaving it blank will result in `default` sharding.")
+}

test_runner/src/main/kotlin/ftl/config/FlankDefaults.kt

@@ -10,4 +10,5 @@ object FlankDefaults {
     const val DISABLE_AUTO_LOGIN = false
     const val DISABLE_PERFORMANCE_METRICS = false
     const val GRANT_PERMISSIONS_ALL = "all"
+    const val DEFAULT_PARAMETERIZED_TESTS = "default"
 }

test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt

@@ -248,6 +248,13 @@ data class AndroidGcloudConfig @JsonIgnore constructor(
     @set:JsonProperty("test-targets-for-shard")
     var testTargetsForShard: List<String>? by data
 
+    @set:CommandLine.Option(
+        names = ["--parameterized-tests"],
+        description = ["Specifies how to handle tests which contain the parameterization annotation."]
+    )
+    @set:JsonProperty("parameterized-tests")
+    var parameterizedTests: String? by data
+
     constructor() : this(mutableMapOf<String, Any?>().withDefault { null })
 
     companion object : IYmlKeys {
@@ -276,6 +283,7 @@ data class AndroidGcloudConfig @JsonIgnore constructor(
             roboDirectives = emptyMap()
             roboScript = null
             testTargetsForShard = emptyList()
+            parameterizedTests = FlankDefaults.DEFAULT_PARAMETERIZED_TESTS
         }
     }
 }

test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt

@@ -356,6 +356,7 @@ AndroidArgs
       num-flaky-test-attempts: 3
       test-targets-for-shard:
       fail-fast: false
+      parameterized-tests: default
 
     flank:
       max-test-shards: 7
@@ -445,6 +446,7 @@ AndroidArgs
       num-flaky-test-attempts: 0
       test-targets-for-shard:
       fail-fast: false
+      parameterized-tests: default
 
     flank:
       max-test-shards: 1
@@ -2658,6 +2660,57 @@ AndroidArgs
         """.trimIndent()
         assertEquals(expectedMessage, errorMessage)
     }
+
+    @Test
+    fun `should throw exception if paramterized test is incorrect`() {
+        val yaml = """
+        gcloud:
+            # Android gcloud
+            app: $appApk
+            test: $testApk
+            parameterized-tests: error
+        """.trimIndent()
+        val errorMessage = getThrowable { AndroidArgs.load(yaml).validate() }.message ?: ""
+        val expectedMessage = """
+            Parameterized test flag must be one of the following: `default`, `ignore-all`, `shard-into-single`, leaving it blank will result in `default` sharding.
+        """.trimIndent()
+        assertEquals(expectedMessage, errorMessage)
+    }
+
+    @Test
+    fun `should throw NOT throw an exception if parameterized test is left to be blank`() {
+        val yaml = """
+        gcloud:
+            # Android gcloud
+            app: $appApk
+            test: $testApk
+        """.trimIndent()
+        AndroidArgs.load(yaml).validate()
+    }
+
+    @Test
+    fun `should throw NOT throw an exception if paramterized test is ignore-all`() {
+        val yaml = """
+        gcloud:
+            # Android gcloud
+            app: $appApk
+            test: $testApk
+            parameterized-tests: ignore-all
+        """.trimIndent()
+        AndroidArgs.load(yaml).validate()
+    }
+
+    @Test
+    fun `should throw NOT throw an exception if paramterized test is shard-into-single`() {
+        val yaml = """
+        gcloud:
+            # Android gcloud
+            app: $appApk
+            test: $testApk
+            parameterized-tests: shard-into-single
+        """.trimIndent()
+        AndroidArgs.load(yaml).validate()
+    }
 }
 
 private fun AndroidArgs.Companion.load(