Document coverage with orchestrator

README.md

@@ -139,6 +139,113 @@ flank:
     - .*\.mp4$
 ```
 
+### Android code coverage
+
+<details>
+<summary>Update your app's build.gradle to build with coverage and use orchestrator.
+A custom gradle task is defined to generate the coverage report.</summary>
+
+```gradle
+def coverageEnabled = project.hasProperty('coverage')
+
+android {
+
+  defaultConfig {
+    testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
+    // runs pm clear after each test invocation
+    testInstrumentationRunnerArguments clearPackageData: 'true'
+  }
+
+  buildTypes {
+    debug {
+      testCoverageEnabled true
+    }
+  }
+
+  // https://google.github.io/android-gradle-dsl/current/com.android.build.gradle.internal.dsl.TestOptions.html#com.android.build.gradle.internal.dsl.TestOptions:animationsDisabled
+  testOptions {
+        execution 'ANDROIDX_TEST_ORCHESTRATOR'
+        animationsDisabled = true
+    }
+}
+
+dependencies {
+  androidTestUtil 'androidx.test:orchestrator:1.1.1'
+
+  androidTestImplementation("androidx.test:runner:1.1.1")
+  androidTestImplementation("androidx.test.ext:junit:1.1.0")
+  androidTestImplementation("androidx.test.ext:junit-ktx:1.1.0")
+  androidTestImplementation("androidx.test.ext:truth:1.1.0")
+  androidTestImplementation("androidx.test.espresso.idling:idling-concurrent:3.1.1")
+  androidTestImplementation("androidx.test.espresso.idling:idling-net:3.1.1")
+  androidTestImplementation("androidx.test.espresso:espresso-accessibility:3.1.1")
+  androidTestImplementation("androidx.test:rules:1.1.1")
+  androidTestImplementation("androidx.test.espresso:espresso-core:3.1.1")
+  androidTestImplementation("androidx.test.espresso:espresso-contrib:3.1.1")
+  androidTestImplementation("androidx.test.espresso:espresso-idling-resource:3.1.1")
+  androidTestImplementation("androidx.test.espresso:espresso-intents:3.1.1")
+  androidTestImplementation("androidx.test.espresso:espresso-web:3.1.1")
+}
+
+if (coverageEnabled) {
+    // gradle -Pcoverage firebaseJacoco
+    task firebaseJacoco(type: JacocoReport) {
+        group = "Reporting"
+        description = "Generate Jacoco coverage reports for Firebase test lab."
+
+        def excludes = [
+                '**/R.class',
+                '**/R$*.class',
+                '**/BuildConfig.*',
+                "**/androidx"]
+        def javaClasses = fileTree(dir: "${project.buildDir}/intermediates/javac/debug/compileDebugJavaWithJavac/classes", excludes: excludes)
+        def kotlinClasses = fileTree(dir: "${project.buildDir}/tmp/kotlin-classes/debug", excludes: excludes)
+        getClassDirectories().setFrom(files([javaClasses, kotlinClasses]))
+
+        getSourceDirectories().setFrom(files([
+                'src/main/java', 'src/main/kotlin',
+                'src/androidTest/java', 'src/androidTest/kotlin']))
+
+        def ecFiles = project.fileTree(dir: '..', include: 'results/coverage_ec/**/sdcard/*.ec')
+        ecFiles.forEach { println("Reading in $it") }
+        getExecutionData().setFrom(ecFiles)
+
+        reports {
+            html { enabled true }
+            xml { enabled false }
+        }
+    }
+}
+```
+
+Here's an example flank.yml. Note that `coverage` and `coverageFilePath` must be set when using orchestrator with coverage.
+`coverageFile` is not used. Orchestrator will generate one coverage file per test. `coverageFilePath` must be a directory, not a file.
+
+```yaml
+gcloud:
+  app: ./app/build/outputs/apk/debug/app-debug.apk
+  test: ./app/build/outputs/apk/androidTest/debug/app-debug-androidTest.apk
+  environment-variables: 
+    coverage: true
+    coverageFilePath: /sdcard/
+    clearPackageData: true
+  directories-to-pull: 
+    - /sdcard/
+  # use a named results dir that's used by the gradle task
+  results-dir: coverage_ec
+
+flank:
+  disableSharding: true
+  files-to-download:
+    - .*/sdcard/[^/]+\.ec$
+```
+
+- Build the app with coverage: `./gradlew -Pcoverage build`
+- Run flank `flank android run`
+- Generate the report `./gradlew -Pcoverage firebaseJacoco`
+- Open the report in `./build/reports/jacoco/firebaseJacoco/html/index.html`
+</details>
+
 ### CI integration
 
 Download Flank from GitHub releases.

test_app/.gitignore

@@ -1,9 +1,14 @@
 *.iml
 .gradle
 /local.properties
-/.idea/workspace.xml
+/.idea/caches
 /.idea/libraries
+/.idea/modules.xml
+/.idea/workspace.xml
+/.idea/navEditor.xml
+/.idea/assetWizardSettings.xml
 .DS_Store
 /build
 /captures
 .externalNativeBuild
+results/

test_app/app/build.gradle

@@ -0,0 +1,91 @@
+apply plugin: 'com.android.application'
+apply plugin: 'kotlin-android'
+apply plugin: 'kotlin-android-extensions'
+apply plugin: 'jacoco'
+
+def coverageEnabled = project.hasProperty('coverage')
+
+android {
+    compileSdkVersion 28
+    defaultConfig {
+        applicationId "com.example.test_app"
+        minSdkVersion 26
+        targetSdkVersion 28
+        versionCode 1
+        versionName "1.0"
+        testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
+        testInstrumentationRunnerArguments(clearPackageData: 'true')
+
+    }
+
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
+        }
+
+        debug {
+            testCoverageEnabled coverageEnabled
+        }
+    }
+
+    testOptions {
+        animationsDisabled = true
+        execution = "ANDROIDX_TEST_ORCHESTRATOR"
+    }
+}
+
+dependencies {
+    implementation"org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version"
+    implementation 'androidx.appcompat:appcompat:1.1.0-alpha01'
+    implementation 'androidx.core:core-ktx:1.1.0-alpha03'
+    implementation 'androidx.constraintlayout:constraintlayout:1.1.3'
+
+    // Espresso.
+    // https://developer.android.com/jetpack/androidx/releases/test
+    androidTestUtil 'androidx.test:orchestrator:1.1.1'
+
+    androidTestImplementation("androidx.test:runner:1.1.1")
+    androidTestImplementation("androidx.test.ext:junit:1.1.0")
+    androidTestImplementation("androidx.test.ext:junit-ktx:1.1.0")
+    androidTestImplementation("androidx.test.ext:truth:1.1.0")
+    androidTestImplementation("androidx.test.espresso.idling:idling-concurrent:3.1.1")
+    androidTestImplementation("androidx.test.espresso.idling:idling-net:3.1.1")
+    androidTestImplementation("androidx.test.espresso:espresso-accessibility:3.1.1")
+    androidTestImplementation("androidx.test:rules:1.1.1")
+    androidTestImplementation("androidx.test.espresso:espresso-core:3.1.1")
+    androidTestImplementation("androidx.test.espresso:espresso-contrib:3.1.1")
+    androidTestImplementation("androidx.test.espresso:espresso-idling-resource:3.1.1")
+    androidTestImplementation("androidx.test.espresso:espresso-intents:3.1.1")
+    androidTestImplementation("androidx.test.espresso:espresso-web:3.1.1")
+}
+
+if (coverageEnabled) {
+    // gradle -Pcoverage firebaseJacoco
+    task firebaseJacoco(type: JacocoReport) {
+        group = "Reporting"
+        description = "Generate Jacoco coverage reports for Firebase test lab."
+
+        def excludes = [
+                '**/R.class',
+                '**/R$*.class',
+                '**/BuildConfig.*',
+                "**/androidx"]
+        def javaClasses = fileTree(dir: "${project.buildDir}/intermediates/javac/debug/compileDebugJavaWithJavac/classes", excludes: excludes)
+        def kotlinClasses = fileTree(dir: "${project.buildDir}/tmp/kotlin-classes/debug", excludes: excludes)
+        getClassDirectories().setFrom(files([javaClasses, kotlinClasses]))
+
+        getSourceDirectories().setFrom(files([
+                'src/main/java', 'src/main/kotlin',
+                'src/androidTest/java', 'src/androidTest/kotlin']))
+
+        def ecFiles = project.fileTree(dir: '..', include: 'results/coverage_ec/**/sdcard/*.ec')
+        ecFiles.forEach { println("Reading in $it") }
+        getExecutionData().setFrom(ecFiles)
+
+        reports {
+            html { enabled true }
+            xml { enabled false }
+        }
+    }
+}

test_app/app/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile