Skip to content
This repository has been archived by the owner on Dec 7, 2019. It is now read-only.

Add coverage report #148

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Add coverage report #148

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b929938
Add coverage report
tokou Jun 3, 2018
fcb3edf
Add note and update README
Jun 6, 2018
75bc99e
Extract consts for dirs
Jun 6, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,12 @@ Composer shipped as jar, to run it you need JVM 1.8+: `java -jar composer-latest
* Default: `true`.
* `False` may be applicable when you run tests conditionally(via annotation/package filters) and empty suite is a valid outcome.
* Example: `--fail-if-no-tests false`

* `--coverage`
* Either `true` or `false` to enable/disable test coverage reports collection.
* Default: `false`.
* For this to work, your test APK should be built with instrumentation from EMMA or JaCoCo.
* Example: `--coverage true`

##### Example

Simplest :
Expand Down
13 changes: 11 additions & 2 deletions composer/src/main/kotlin/com/gojuno/composer/Args.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,16 @@ data class Args(
description = "Either `true` or `false` to enable/disable error on empty test suite. True by default.",
order = 11
)
var failIfNoTests: Boolean = true
var failIfNoTests: Boolean = true,

@Parameter(
names = arrayOf("--coverage"),
required = false,
arity = 1,
description = "Either `true` or `false` to enable/disable test coverage reports collection. `false` by default. For this to work, your test APK should be built with instrumentation from EMMA or JaCoCo.",
order = 12
)
var coverage: Boolean = false
)

// No way to share array both for runtime and annotation without reflection.
Expand All @@ -133,4 +142,4 @@ fun parseArgs(rawArgs: Array<String>) = Args().also { args ->

private class InstrumentationArgumentsConverter : IStringConverter<List<String>> {
override fun convert(argument: String): List<String> = listOf(argument)
}
}
5 changes: 5 additions & 0 deletions composer/src/main/kotlin/com/gojuno/composer/Consts.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
package com.gojuno.composer

val EXTERNAL_STORAGE = "/storage/emulated/0"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I meant retrieving the actual value from environment on device using something like adb shell printenv EXTERNAL_STORAGE.
AFAIR it is mandatory: https://source.android.com/devices/storage/config-example

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Btw I guess we can have this code in commander

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you need me to try and do this or will you add something more generic to commander ?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll add it to commander now

val COVERAGE_DIR = "$EXTERNAL_STORAGE/app_coverage"
val SCREENSHOTS_DIR = "$EXTERNAL_STORAGE/app_spoon-screenshots"
25 changes: 22 additions & 3 deletions composer/src/main/kotlin/com/gojuno/composer/Main.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
package com.gojuno.composer

import com.gojuno.commander.android.adb
import com.gojuno.commander.android.connectedAdbDevices
import com.gojuno.commander.android.installApk
import com.gojuno.commander.os.log
import com.gojuno.commander.os.nanosToHumanReadableTime
import com.gojuno.commander.os.process
import com.gojuno.composer.html.writeHtmlReport
import com.google.gson.Gson
import rx.Observable
Expand Down Expand Up @@ -111,15 +113,25 @@ private fun runAllTests(args: Args, testPackage: TestPackage.Valid, testRunner:
val installTimeout = Pair(args.installTimeoutSeconds, TimeUnit.SECONDS)
val installAppApk = device.installApk(pathToApk = args.appApkPath, timeout = installTimeout)
val installTestApk = device.installApk(pathToApk = args.testApkPath, timeout = installTimeout)
val coverageDir = "$COVERAGE_DIR/${testPackage.value}"
val makeCoverageDir = if (args.coverage) {
process(commandAndArgs = listOf(adb, "-s", device.id, "shell", "mkdir", "-p", coverageDir))
.map { Unit }
} else {
Observable.just(Unit)
}

Observable
.concat(installAppApk, installTestApk)
.concat(installAppApk, installTestApk, makeCoverageDir)
// Work with each device in parallel, but install apks sequentially on a device.
.subscribeOn(Schedulers.io())
.toList()
.flatMap {
val instrumentationArguments =
buildShardArguments(
buildCoverageArguments(
coverage = args.coverage,
coverageDir = coverageDir
) + buildShardArguments(
shardingOn = args.shard,
shardIndex = index,
devices = connectedAdbDevices.size
Expand All @@ -132,7 +144,8 @@ private fun runAllTests(args: Args, testPackage: TestPackage.Valid, testRunner:
instrumentationArguments = instrumentationArguments.formatInstrumentationArguments(),
outputDir = File(args.outputDirectory),
verboseOutput = args.verboseOutput,
keepOutput = args.keepOutputOnExit
keepOutput = args.keepOutputOnExit,
coverage = args.coverage
)
.flatMap { adbDeviceTestRun ->
writeJunit4Report(
Expand Down Expand Up @@ -210,6 +223,12 @@ private fun buildShardArguments(shardingOn: Boolean, shardIndex: Int, devices: I
else -> emptyList()
}

private fun buildCoverageArguments(coverage: Boolean, coverageDir: String): List<Pair<String, String>> =
if (coverage) listOf(
"coverage" to "true",
"coverageFile" to "$coverageDir/coverage.ec"
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This needs to be coverageFilePath when running with testOrchestrator and then we will have to pull down .ec for each test case.

) else emptyList()

private fun List<Pair<String, String>>.formatInstrumentationArguments(): String = when (isEmpty()) {
true -> ""
false -> " " + joinToString(separator = " ") { "-e ${it.first} ${it.second}" }
Expand Down
38 changes: 29 additions & 9 deletions composer/src/main/kotlin/com/gojuno/composer/TestRun.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ data class AdbDeviceTestRun(
val durationNanos: Long,
val timestampMillis: Long,
val logcat: File,
val instrumentationOutput: File
val instrumentationOutput: File,
val coverageReport: File
)

data class AdbDeviceTest(
Expand All @@ -44,12 +45,14 @@ fun AdbDevice.runTests(
instrumentationArguments: String,
outputDir: File,
verboseOutput: Boolean,
keepOutput: Boolean
keepOutput: Boolean,
coverage: Boolean
): Single<AdbDeviceTestRun> {

val adbDevice = this
val logsDir = File(File(outputDir, "logs"), adbDevice.id)
val instrumentationOutputFile = File(logsDir, "instrumentation.output")
val coverageReportDir = File(File(outputDir, "coverage"), adbDevice.id)

val runTests = process(
commandAndArgs = listOf(
Expand Down Expand Up @@ -88,13 +91,31 @@ fun AdbDevice.runTests(
}
.toList()

val testRunFinish = runTests.ofType(Notification.Exit::class.java).cache()

val pullCoverage = testRunFinish.toSingle()
.flatMap {
val coverageReport = File(coverageReportDir, "coverage.ec")
if (coverage) {
coverageReportDir.mkdirs()
adbDevice.pullFolder(
folderOnDevice = "$COVERAGE_DIR/$testPackageName/coverage.ec",
folderOnHostMachine = coverageReport,
logErrors = verboseOutput
).map { coverageReport }
} else {
Single.just(coverageReport)
}
}.toObservable()

val adbDeviceTestRun = Observable
.zip(
Observable.fromCallable { System.nanoTime() },
runningTests,
{ time, tests -> time to tests }
pullCoverage,
{ time, tests, coverageFile -> Triple(time, tests, coverageFile) }
)
.map { (startTimeNanos, testsWithPulledFiles) ->
.map { (startTimeNanos, testsWithPulledFiles, coverageFile) ->
val tests = testsWithPulledFiles.map { it.first }

AdbDeviceTestRun(
Expand All @@ -121,12 +142,11 @@ fun AdbDevice.runTests(
durationNanos = System.nanoTime() - startTimeNanos,
timestampMillis = System.currentTimeMillis(),
logcat = logcatFileForDevice(logsDir),
instrumentationOutput = instrumentationOutputFile
instrumentationOutput = instrumentationOutputFile,
coverageReport = coverageFile
)
}

val testRunFinish = runTests.ofType(Notification.Exit::class.java).cache()

val saveLogcat = saveLogcat(adbDevice, logsDir)
.map { Unit }
// TODO: Stop when all expected tests were parsed from logcat and not when instrumentation finishes.
Expand All @@ -135,7 +155,7 @@ fun AdbDevice.runTests(
.startWith(Unit) // To allow zip finish normally even if no tests were run.

return Observable
.zip(adbDeviceTestRun, saveLogcat, testRunFinish) { suite, _, _ -> suite }
.zip(adbDeviceTestRun, saveLogcat, pullCoverage, testRunFinish) { suite, _, _, _ -> suite }
.doOnSubscribe { adbDevice.log("Starting tests...") }
.doOnNext { testRun ->
adbDevice.log(
Expand Down Expand Up @@ -163,7 +183,7 @@ private fun pullTestFiles(adbDevice: AdbDevice, test: InstrumentationTest, outpu
adbDevice
.pullFolder(
// TODO: Add support for internal storage and external storage strategies.
folderOnDevice = "/storage/emulated/0/app_spoon-screenshots/${test.className}/${test.testName}",
folderOnDevice = "$SCREENSHOTS_DIR/${test.className}/${test.testName}",
folderOnHostMachine = screenshotsFolderOnHostMachine,
logErrors = verboseOutput
)
Expand Down