Add additional-app-test-apks support

Flank · Apr 10, 2019 · f1ca31e · f1ca31e
1 parent a1750fe
commit f1ca31e
Show file tree

Hide file tree

Showing 27 changed files with 216 additions and 188 deletions.
diff --git a/README.md b/README.md
@@ -307,6 +307,12 @@ flank:
 
   ## Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved.
   # local-result-dir: flank
+
+  ## Include additional app/test apk pairs in the run. If app is omitted, then the top level app is used for that pair.
+  # additional-app-test-apks:
+  #  - app: ../test_app/apks/app-debug.apk
+  #    test: ../test_app/apks/app-debug-androidTest.apk
+  #  - test: ../test_app/apks/app-debug-androidTest.apk
 ```
 
 ### Android code coverage

diff --git a/release_notes.md b/release_notes.md
@@ -1,5 +1,6 @@
 ## next (unreleased)
 - [#541](https://github.com/TestArmada/flank/pull/541) Rename `--test-shards` CLI flag to `--max-test-shards`. Add `--smart-flank-gcs-path` CLI flag. ([bootstraponline](https://github.com/bootstraponline))
+- [#542](https://github.com/TestArmada/flank/pull/542) Add `additional-app-test-apks` to include multiple app/test apk pairs in a single run. ([bootstraponline](https://github.com/bootstraponline))
 
 ## v5.1.0
 - [#537](https://github.com/TestArmada/flank/pull/537) Add `smart-flank-disable-upload` yml option to prevent new results from overriding previous results. ([elihart](https://github.com/elihart))

diff --git a/test_runner/flank.yml b/test_runner/flank.yml
@@ -128,4 +128,9 @@ flank:
 
   ## Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved.
   # local-result-dir: flank
-
+
+  ## Include additional app/test apk pairs in the run. If app is omitted, then the top level app is used for that pair.
+  # additional-app-test-apks:
+  #  - app: ../test_app/apks/app-debug.apk
+  #    test: ../test_app/apks/app-debug-androidTest.apk
+  #  - test: ../test_app/apks/app-debug-androidTest.apk
diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt
@@ -1,15 +1,12 @@
 package ftl.args
 
-import com.linkedin.dex.parser.DexParser
-import com.linkedin.dex.parser.TestMethod
 import ftl.android.AndroidCatalog
 import ftl.android.IncompatibleModelVersion
 import ftl.android.SupportedDeviceConfig
 import ftl.android.UnsupportedModelId
 import ftl.android.UnsupportedVersionId
 import ftl.args.ArgsHelper.assertFileExists
 import ftl.args.ArgsHelper.assertGcsFileExists
-import ftl.args.ArgsHelper.calculateShards
 import ftl.args.ArgsHelper.createGcsBucket
 import ftl.args.ArgsHelper.createJunitBucket
 import ftl.args.ArgsHelper.evaluateFilePath
@@ -28,13 +25,8 @@ import ftl.args.yml.YamlDeprecated
 import ftl.cli.firebase.test.android.AndroidRunCommand
 import ftl.config.Device
 import ftl.config.FtlConstants
-import ftl.config.FtlConstants.useMock
-import ftl.filter.TestFilters
-import ftl.gc.GcStorage
-import ftl.util.Utils
 import java.nio.file.Files
 import java.nio.file.Path
-import kotlinx.coroutines.runBlocking
 
 // set default values, init properties, etc.
 class AndroidArgs(
@@ -82,22 +74,6 @@ class AndroidArgs(
     private val androidFlank = androidFlankYml.flank
     val additionalAppTestApks = cli?.additionalAppTestApks ?: androidFlank.additionalAppTestApks
 
-    // computed properties not specified in yaml
-    override val testShardChunks: List<List<String>> by lazy {
-        if (disableSharding) return@lazy listOf(emptyList<String>())
-
-        // Download test APK if necessary so it can be used to validate test methods
-        var testLocalApk = testApk
-        if (testApk.startsWith(FtlConstants.GCS_PREFIX)) {
-            runBlocking {
-                testLocalApk = GcStorage.downloadTestApk(this@AndroidArgs)
-            }
-        }
-
-        val filteredTests = getTestMethods(testLocalApk)
-        calculateShards(filteredTests, this)
-    }
-
     init {
         resultsBucket = createGcsBucket(project, cli?.resultsBucket ?: gcloud.resultsBucket)
         createJunitBucket(project, flank.smartFlankGcsPath)
@@ -119,21 +95,6 @@ class AndroidArgs(
         devices.forEach { device -> assertDeviceSupported(device) }
     }
 
-    private fun getTestMethods(testLocalApk: String): List<String> {
-        val allTestMethods = DexParser.findTestMethods(testLocalApk)
-        require(allTestMethods.isNotEmpty()) { Utils.fatalError("Test APK has no tests") }
-        val testFilter = TestFilters.fromTestTargets(testTargets)
-        val filteredTests = allTestMethods
-            .asSequence()
-            .distinct()
-            .filter(testFilter.shouldRun)
-            .map(TestMethod::testName)
-            .map { "class $it" }
-            .toList()
-        require(useMock || filteredTests.isNotEmpty()) { Utils.fatalError("All tests filtered out") }
-        return filteredTests
-    }
-
     private fun assertDeviceSupported(device: Device) {
         val deviceConfigTest = AndroidCatalog.supportedDeviceConfig(device.model, device.version)
         when (deviceConfigTest) {

diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidTestShard.kt b/test_runner/src/main/kotlin/ftl/args/AndroidTestShard.kt
@@ -0,0 +1,43 @@
+package ftl.args
+
+import com.linkedin.dex.parser.DexParser
+import com.linkedin.dex.parser.TestMethod
+import ftl.config.FtlConstants
+import ftl.filter.TestFilters
+import ftl.gc.GcStorage
+import ftl.util.Utils
+import kotlinx.coroutines.runBlocking
+
+object AndroidTestShard {
+
+    // computed properties not specified in yaml
+    fun getTestShardChunks(args: AndroidArgs, testApk: String): List<List<String>> {
+        if (args.disableSharding) return listOf(emptyList())
+
+        // Download test APK if necessary so it can be used to validate test methods
+        var testLocalApk = testApk
+        if (testApk.startsWith(FtlConstants.GCS_PREFIX)) {
+            runBlocking {
+                testLocalApk = GcStorage.download(testApk)
+            }
+        }
+
+        val filteredTests = getTestMethods(args, testLocalApk)
+        return ArgsHelper.calculateShards(filteredTests, args)
+    }
+
+    private fun getTestMethods(args: AndroidArgs, testLocalApk: String): List<String> {
+        val allTestMethods = DexParser.findTestMethods(testLocalApk)
+        require(allTestMethods.isNotEmpty()) { Utils.fatalError("Test APK has no tests") }
+        val testFilter = TestFilters.fromTestTargets(args.testTargets)
+        val filteredTests = allTestMethods
+            .asSequence()
+            .distinct()
+            .filter(testFilter.shouldRun)
+            .map(TestMethod::testName)
+            .map { "class $it" }
+            .toList()
+        require(FtlConstants.useMock || filteredTests.isNotEmpty()) { Utils.fatalError("All tests filtered out") }
+        return filteredTests
+    }
+}
diff --git a/test_runner/src/main/kotlin/ftl/args/IArgs.kt b/test_runner/src/main/kotlin/ftl/args/IArgs.kt
@@ -28,8 +28,5 @@ interface IArgs {
     val disableSharding: Boolean
     val localResultDir: String
 
-    // computed property
-    val testShardChunks: List<List<String>>
-
     fun useLocalResultDir() = localResultDir != FlankYmlParams.defaultLocalResultDir
 }
diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt
@@ -63,7 +63,7 @@ class IosArgs(
     val testTargets = cli?.testTargets ?: iosFlank.testTargets
 
     // computed properties not specified in yaml
-    override val testShardChunks: List<List<String>> by lazy {
+    val testShardChunks: List<List<String>> by lazy {
         if (disableSharding) return@lazy listOf(emptyList<String>())
 
         val validTestMethods = Xctestrun.findTestNames(xctestrunFile)

diff --git a/test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt
@@ -4,7 +4,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties
 import com.fasterxml.jackson.annotation.JsonProperty
 
 data class AppTestPair(
-    val app: String,
+    val app: String?,
     val test: String
 )
 

diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/RefreshCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/RefreshCommand.kt
@@ -22,7 +22,7 @@ Reads in the matrix_ids.json file. Refreshes any incomplete matrices.
 class RefreshCommand : Runnable {
     override fun run() {
         runBlocking {
-            TestRunner.refreshLastRun(AndroidArgs.default())
+            TestRunner.refreshLastRun(AndroidArgs.default(), emptyList())
         }
     }
 

diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt
@@ -9,10 +9,10 @@ import ftl.config.FtlConstants.defaultAndroidVersion
 import ftl.config.FtlConstants.defaultLocale
 import ftl.config.FtlConstants.defaultOrientation
 import ftl.run.TestRunner
+import java.nio.file.Paths
 import kotlinx.coroutines.runBlocking
 import picocli.CommandLine.Command
 import picocli.CommandLine.Option
-import java.nio.file.Paths
 
 @Command(
     name = "run",

diff --git a/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt b/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt
@@ -20,7 +20,6 @@ import ftl.util.ShardCounter
 import ftl.util.Utils.fatalError
 import ftl.util.Utils.join
 import ftl.util.testTimeoutToSeconds
-import ftl.util.validateTestShardIndex
 
 object GcAndroidTestMatrix {
 
@@ -34,12 +33,11 @@ object GcAndroidTestMatrix {
         testApkGcsPath: String,
         runGcsPath: String,
         androidDeviceList: AndroidDeviceList,
-        testShardsIndex: Int = -1,
+        testTargets: List<String>,
         args: AndroidArgs,
         shardCounter: ShardCounter,
         toolResultsHistory: ToolResultsHistory
     ): Testing.Projects.TestMatrices.Create {
-        validateTestShardIndex(testShardsIndex, args)
 
         // https://github.com/bootstraponline/studio-google-cloud-testing/blob/203ed2890c27a8078cd1b8f7ae12cf77527f426b/firebase-testing/src/com/google/gct/testing/launcher/CloudTestsLauncher.java#L120
         val clientInfo = ClientInfo().setName("Flank")
@@ -54,7 +52,7 @@ object GcAndroidTestMatrix {
             androidInstrumentation.orchestratorOption = "USE_ORCHESTRATOR"
         }
 
-        androidInstrumentation.testTargets = args.testShardChunks.elementAt(testShardsIndex).toList()
+        androidInstrumentation.testTargets = testTargets
 
         // --auto-google-login
         // https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run

diff --git a/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt b/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt
@@ -20,21 +20,19 @@ import ftl.util.ShardCounter
 import ftl.util.Utils.fatalError
 import ftl.util.Utils.join
 import ftl.util.testTimeoutToSeconds
-import ftl.util.validateTestShardIndex
 
 object GcIosTestMatrix {
 
     fun build(
         iosDeviceList: IosDeviceList,
         testZipGcsPath: String,
         runGcsPath: String,
-        testShardsIndex: Int,
         xcTestParsed: NSDictionary,
         args: IosArgs,
+        testTargets: List<String>,
         shardCounter: ShardCounter,
         toolResultsHistory: ToolResultsHistory
     ): Testing.Projects.TestMatrices.Create {
-        validateTestShardIndex(testShardsIndex, args)
         val clientInfo = ClientInfo().setName("Flank")
 
         val gcsBucket = args.resultsBucket
@@ -46,8 +44,7 @@ object GcIosTestMatrix {
         val generatedXctestrun = if (args.disableSharding) {
             xcTestParsed.toByteArray()
         } else {
-            val methods = args.testShardChunks.elementAt(testShardsIndex)
-            Xctestrun.rewrite(xcTestParsed, methods)
+            Xctestrun.rewrite(xcTestParsed, testTargets)
         }
 
         val xctestrunFileGcsPath = GcStorage.uploadXCTestFile(args, gcsBucket, matrixGcsSuffix, generatedXctestrun)

diff --git a/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt b/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt
@@ -5,7 +5,6 @@ import com.google.cloud.storage.BlobInfo
 import com.google.cloud.storage.Storage
 import com.google.cloud.storage.StorageOptions
 import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper
-import ftl.args.AndroidArgs
 import ftl.args.IArgs
 import ftl.args.IosArgs
 import ftl.config.FtlConstants
@@ -45,13 +44,16 @@ object GcStorage {
         }
     }
 
-    private fun upload(file: String, rootGcsBucket: String, runGcsPath: String): String =
-        upload(
+    fun upload(file: String, rootGcsBucket: String, runGcsPath: String): String {
+        if (file.startsWith(FtlConstants.GCS_PREFIX)) return file
+
+        return upload(
             file = file,
             fileBytes = Files.readAllBytes(Paths.get(file)),
             rootGcsBucket = rootGcsBucket,
             runGcsPath = runGcsPath
         )
+    }
 
     fun uploadJunitXml(testResult: JUnitTestResult, args: IArgs) {
         if (args.smartFlankGcsPath.isEmpty() || args.smartFlankDisableUpload) return
@@ -74,12 +76,6 @@ object GcStorage {
         }
     }
 
-    fun uploadAppApk(args: AndroidArgs, gcsBucket: String, runGcsPath: String): String =
-        upload(args.appApk, gcsBucket, runGcsPath)
-
-    fun uploadTestApk(args: AndroidArgs, gcsBucket: String, runGcsPath: String): String =
-        upload(args.testApk, gcsBucket, runGcsPath)
-
     fun uploadXCTestZip(args: IosArgs, runGcsPath: String): String =
         upload(args.xctestrunZip, args.resultsBucket, runGcsPath)
 
@@ -91,9 +87,6 @@ object GcStorage {
             runGcsPath = runGcsPath
         )
 
-    fun downloadTestApk(args: AndroidArgs): String =
-        download(args.testApk)
-
     // junit xml may not exist. ignore error if it doesn't exist
     fun downloadJunitXml(args: IArgs): JUnitTestResult? {
         val oldXmlPath = download(args.smartFlankGcsPath, ignoreError = true)
@@ -124,7 +117,7 @@ object GcStorage {
         return gcsFilePath
     }
 
-    private fun download(gcsUriString: String, ignoreError: Boolean = false): String {
+    fun download(gcsUriString: String, ignoreError: Boolean = false): String {
         val gcsURI = URI.create(gcsUriString)
         val bucket = gcsURI.authority
         val path = gcsURI.path.drop(1) // Drop leading slash

diff --git a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt
@@ -91,7 +91,7 @@ object ReportManager {
     }
 
     /** Returns true if there were no test failures */
-    fun generate(matrices: MatrixMap, args: IArgs): Int {
+    fun generate(matrices: MatrixMap, args: IArgs, testShardChunks: List<List<String>>): Int {
         val testSuite = parseTestSuite(matrices, args)
 
         val useFlakyTests = args.flakyTestAttempts > 0
@@ -111,7 +111,7 @@ object ReportManager {
         }
 
         JUnitReport.run(matrices, testSuite, printToStdout = false, args = args)
-        processJunitXml(testSuite, args)
+        processJunitXml(testSuite, args, testShardChunks)
 
         return matrices.exitCode()
     }
@@ -123,13 +123,18 @@ object ReportManager {
         val timeDiff: Double
     )
 
-    fun createShardEfficiencyList(oldResult: JUnitTestResult, newResult: JUnitTestResult, args: IArgs):
+    fun createShardEfficiencyList(
+        oldResult: JUnitTestResult,
+        newResult: JUnitTestResult,
+        args: IArgs,
+        testShardChunks: List<List<String>>
+    ):
             List<ShardEfficiency> {
         val oldJunitMap = Shard.createJunitMap(oldResult, args)
         val newJunitMap = Shard.createJunitMap(newResult, args)
 
         val timeList = mutableListOf<ShardEfficiency>()
-        args.testShardChunks.forEachIndexed { index, testSuite ->
+        testShardChunks.forEachIndexed { index, testSuite ->
 
             var expectedTime = 0.0
             var finalTime = 0.0
@@ -145,23 +150,32 @@ object ReportManager {
         return timeList
     }
 
-    private fun printActual(oldResult: JUnitTestResult, newResult: JUnitTestResult, args: IArgs) {
-        val list = createShardEfficiencyList(oldResult, newResult, args)
+    private fun printActual(
+        oldResult: JUnitTestResult,
+        newResult: JUnitTestResult,
+        args: IArgs,
+        testShardChunks: List<List<String>>
+    ) {
+        val list = createShardEfficiencyList(oldResult, newResult, args, testShardChunks)
 
         println("Actual shard times:\n" + list.joinToString("\n") {
             "  ${it.shard}: Expected: ${it.expectedTime.roundToInt()}s, Actual: ${it.finalTime.roundToInt()}s, Diff: ${it.timeDiff.roundToInt()}s"
         } + "\n")
     }
 
-    private fun processJunitXml(newTestResult: JUnitTestResult?, args: IArgs) {
+    private fun processJunitXml(
+        newTestResult: JUnitTestResult?,
+        args: IArgs,
+        testShardChunks: List<List<String>>
+    ) {
         if (newTestResult == null) return
 
         val oldTestResult = GcStorage.downloadJunitXml(args)
 
         newTestResult.mergeTestTimes(oldTestResult)
 
         if (oldTestResult != null) {
-            printActual(oldTestResult, newTestResult, args)
+            printActual(oldTestResult, newTestResult, args, testShardChunks)
         }
 
         GcStorage.uploadJunitXml(newTestResult, args)