Allow runtime test discovery when sharding is disabled by not setting test-targets. This unblocks cucumber testing

release_notes.md

@@ -10,8 +10,7 @@
 - [#812](https://github.com/Flank/flank/issues/812) Convert bitrise macOS workflow to github action. ([piotradamczyk5](https://github.com/piotradamczyk5))
 - [#799](https://github.com/Flank/flank/pull/799) Refactor Shared object by splitting it into smaller functions. ([piotradamczyk5](https://github.com/piotradamczyk5))
 - [#798](https://github.com/Flank/flank/pull/798) Remove failure nodes from tests that passed on retry so that Jenkins JUnit plugin marks them as successful. ([adamfilipow92](https://github.com/adamfilipow92))
-- 
-- 
+- [#822](https://github.com/Flank/flank/pull/822) Allow runtime test discovery when sharding is disabled by not setting test-targets. This unblocks cucumber testing. ([adamfilipow92](https://github.com/adamfilipow92))
 
 ## v20.05.2
 

test_runner/src/main/kotlin/ftl/gc/android/CreateAndroidInstrumentationTest.kt

@@ -19,29 +19,37 @@ internal fun createAndroidInstrumentationTest(
 }.setupTestTargets(
     disableSharding = config.disableSharding,
     testShards = config.testShards,
-    numUniformShards = config.numUniformShards
+    numUniformShards = config.numUniformShards,
+    keepTestTargetsEmpty = config.keepTestTargetsEmpty
 )
 
 internal fun AndroidInstrumentationTest.setupTestTargets(
     disableSharding: Boolean,
     testShards: ShardChunks,
-    numUniformShards: Int?
+    numUniformShards: Int?,
+    keepTestTargetsEmpty: Boolean
 ) = apply {
-    if (disableSharding) {
-        testTargets = testShards.flatten()
-    } else {
-        shardingOption = ShardingOption().apply {
-            if (numUniformShards != null) {
-                testTargets = testShards.flatten()
-                val safeNumUniformShards = if (testTargets.size > numUniformShards) numUniformShards else {
-                    println("WARNING: num-uniform-shards ($numUniformShards) is higher than number of test cases (${testTargets.size}) from ${testApk.gcsPath}")
-                    testTargets.size
+    when {
+        keepTestTargetsEmpty -> {
+            testTargets = emptyList()
+        }
+        disableSharding -> {
+            testTargets = testShards.flatten()
+        }
+        else -> {
+            shardingOption = ShardingOption().apply {
+                if (numUniformShards != null) {
+                    testTargets = testShards.flatten()
+                    val safeNumUniformShards = if (testTargets.size > numUniformShards) numUniformShards else {
+                        println("WARNING: num-uniform-shards ($numUniformShards) is higher than number of test cases (${testTargets.size}) from ${testApk.gcsPath}")
+                        testTargets.size
+                    }
+                    uniformSharding = UniformSharding().setNumShards(safeNumUniformShards)
+                } else {
+                    manualSharding = ManualSharding().setTestTargetsForShard(testShards.map {
+                        TestTargetsForShard().setTestTargets(it)
+                    })
                 }
-                uniformSharding = UniformSharding().setNumShards(safeNumUniformShards)
-            } else {
-                manualSharding = ManualSharding().setTestTargetsForShard(testShards.map {
-                    TestTargetsForShard().setTestTargets(it)
-                })
             }
         }
     }

test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt

@@ -61,7 +61,8 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS
         val androidTestConfig = args.createAndroidTestConfig(
             uploadedApks = uploadedApks,
             testShards = testShards,
-            runGcsPath = runGcsPath
+            runGcsPath = runGcsPath,
+            keepTestTargetsEmpty = args.disableSharding && args.testTargets.isEmpty()
         )
 
         testMatrices += executeAndroidTestMatrix(runCount = args.repeatTests) {

test_runner/src/main/kotlin/ftl/run/platform/android/AndroidTestConfig.kt

@@ -13,7 +13,8 @@ sealed class AndroidTestConfig {
         // sharding
         val disableSharding: Boolean,
         val testShards: ShardChunks,
-        val numUniformShards: Int?
+        val numUniformShards: Int?,
+        val keepTestTargetsEmpty: Boolean
     ) : AndroidTestConfig()
 
     data class Robo(

test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestConfig.kt

@@ -8,13 +8,16 @@ import ftl.util.FlankFatalError
 internal fun AndroidArgs.createAndroidTestConfig(
     uploadedApks: UploadedApks,
     testShards: ShardChunks? = null,
-    runGcsPath: String? = null
+    runGcsPath: String? = null,
+    keepTestTargetsEmpty: Boolean = false
 ): AndroidTestConfig = when {
     isInstrumentationTest -> createInstrumentationConfig(
         uploadedApks = uploadedApks,
+        keepTestTargetsEmpty = keepTestTargetsEmpty,
         testShards = testShards ?: throw FlankFatalError("Arg testShards is required for instrumentation test.")
     )
-    isRoboTest -> createRoboConfig(
+    isRoboTest
+    -> createRoboConfig(
         uploadedApks = uploadedApks,
         runGcsPath = runGcsPath ?: throw FlankFatalError("Arg runGcsPath is required for robo test.")
     )

test_runner/src/main/kotlin/ftl/run/platform/android/CreateInstrumentationConfig.kt

@@ -6,6 +6,7 @@ import ftl.args.yml.UploadedApks
 
 internal fun AndroidArgs.createInstrumentationConfig(
     uploadedApks: UploadedApks,
+    keepTestTargetsEmpty: Boolean,
     testShards: ShardChunks
 ) = AndroidTestConfig.Instrumentation(
     appApkGcsPath = uploadedApks.app,
@@ -14,5 +15,6 @@ internal fun AndroidArgs.createInstrumentationConfig(
     orchestratorOption = "USE_ORCHESTRATOR".takeIf { useOrchestrator },
     disableSharding = disableSharding,
     numUniformShards = numUniformShards,
-    testShards = testShards
+    testShards = testShards,
+    keepTestTargetsEmpty = keepTestTargetsEmpty
 )

test_runner/src/test/kotlin/Debug.kt

@@ -11,7 +11,7 @@ fun main() {
     val projectId = System.getenv("GOOGLE_CLOUD_PROJECT")
         ?: "YOUR PROJECT ID"
     val quantity = "single"
-    val type = "errorFlaky"
+    val type = "success-disable-sharding"
 
     // Bugsnag keeps the process alive so we must call exitProcess
     // https://github.com/bugsnag/bugsnag-java/issues/151

test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt

@@ -1,12 +1,16 @@
 package ftl.args
 
+import com.google.api.services.testing.model.TestSpecification
 import com.google.common.truth.Truth.assertThat
 import ftl.args.yml.AppTestPair
+import ftl.args.yml.UploadedApks
 import ftl.cli.firebase.test.android.AndroidRunCommand
 import ftl.config.Device
 import ftl.config.FlankRoboDirective
 import ftl.config.FtlConstants.defaultAndroidModel
 import ftl.config.FtlConstants.defaultAndroidVersion
+import ftl.gc.android.setupAndroidTest
+import ftl.run.platform.android.createAndroidTestConfig
 import ftl.run.platform.runAndroidTests
 import ftl.run.platform.android.getAndroidShardChunks
 import ftl.run.status.OutputStyle
@@ -24,6 +28,7 @@ import org.junit.After
 import org.junit.Assert
 import org.junit.Assert.assertEquals
 import org.junit.Assert.assertFalse
+import org.junit.Assert.assertTrue
 import org.junit.Rule
 import org.junit.Test
 import org.junit.rules.ExpectedException
@@ -1362,6 +1367,40 @@ AndroidArgs
             )
         }
     }
+
+    @Test
+    fun `should return AndroidTestConfig without testTargets`() {
+        val yaml = """
+        gcloud:
+          app: $appApk
+          test: $testApk
+          use-orchestrator: false
+        flank:
+          disable-sharding: true
+        """.trimIndent()
+        val args = AndroidArgs.load(yaml)
+        val androidTestConfig = args.createAndroidTestConfig(UploadedApks("", ""), listOf(listOf("test")), null, true)
+        val testSpecification = TestSpecification().setupAndroidTest(androidTestConfig)
+        assertTrue(testSpecification.androidInstrumentationTest.testTargets.isEmpty())
+    }
+
+    @Test
+    fun `should return AndroidTestConfig with testTargets`() {
+        val yaml = """
+        gcloud:
+          app: $appApk
+          test: $testApk
+          use-orchestrator: false
+          test-targets:
+          - EarlGreyExampleSwiftTests/testWith.*${'$'}
+        flank:
+          disable-sharding: true
+        """.trimIndent()
+        val args = AndroidArgs.load(yaml)
+        val androidTestConfig = args.createAndroidTestConfig(UploadedApks("", ""), listOf(listOf("test"), listOf("test")))
+        val testSpecification = TestSpecification().setupAndroidTest(androidTestConfig)
+        assertTrue(testSpecification.androidInstrumentationTest.testTargets.isNotEmpty())
+    }
 }
 
 private fun AndroidArgs.Companion.load(yamlData: String, cli: AndroidRunCommand? = null): AndroidArgs = load(StringReader(yamlData), cli)

test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-single-success-disable-sharding.yml

@@ -0,0 +1,7 @@
+gcloud:
+  app: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk
+  test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk
+  use-orchestrator: false
+
+flank:
+  disable-sharding: true

test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt

@@ -30,7 +30,8 @@ class GcAndroidTestMatrixTest {
                 orchestratorOption = null,
                 numUniformShards = null,
                 disableSharding = false,
-                testRunnerClass = ""
+                testRunnerClass = "",
+                keepTestTargetsEmpty = false
             ),
             runGcsPath = "",
             otherFiles = emptyMap(),
@@ -53,7 +54,8 @@ class GcAndroidTestMatrixTest {
                 orchestratorOption = null,
                 numUniformShards = null,
                 disableSharding = false,
-                testRunnerClass = ""
+                testRunnerClass = "",
+                keepTestTargetsEmpty = false
             ),
             runGcsPath = "",
             otherFiles = emptyMap(),
@@ -80,7 +82,8 @@ class GcAndroidTestMatrixTest {
                 orchestratorOption = null,
                 numUniformShards = null,
                 disableSharding = false,
-                testRunnerClass = ""
+                testRunnerClass = "",
+                keepTestTargetsEmpty = false
             ),
             runGcsPath = "",
             otherFiles = emptyMap(),

test_runner/src/test/kotlin/ftl/gc/android/CreateAndroidInstrumentationTestTest.kt

@@ -19,7 +19,8 @@ class UtilsKtTest {
             .setupTestTargets(
                 disableSharding = true,
                 testShards = testShards,
-                numUniformShards = null
+                numUniformShards = null,
+                keepTestTargetsEmpty = false
             )
             .testTargets
 
@@ -39,7 +40,8 @@ class UtilsKtTest {
             .setupTestTargets(
                 disableSharding = false,
                 testShards = testShards,
-                numUniformShards = 50
+                numUniformShards = 50,
+                keepTestTargetsEmpty = false
             )
 
         // then
@@ -57,7 +59,8 @@ class UtilsKtTest {
             .setupTestTargets(
                 disableSharding = false,
                 testShards = shardChunks,
-                numUniformShards = null
+                numUniformShards = null,
+                keepTestTargetsEmpty = false
             )
             .shardingOption
             .manualSharding